diff --git a/README.md b/README.md index d08a683..1f8fee3 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ Quorum is a child process inside the hive container, like the `spektacular` bina 1. Hive writes a fresh random bearer token to a file at boot, and the tenant's Vexa API key (and, if used, the Discord bot token) from its keystore to files. 2. Hive starts `node quorum.mjs serve` with the `QUORUM_*` variables below and supervises it. `GET /healthz` answers `{"ok":true,...}` once it is listening. -3. Hive is the MCP client: `POST http://127.0.0.1:8765/mcp` with `Authorization: Bearer ` (streamable HTTP, stateless, JSON responses). Tools: `meeting.join`, `meeting.leave`, `meeting.status`, `transcript.read`, `chat.post`, `say`, `say.tone`, `say.prewarm`, `meeting.delete`. Every result is `{ok:true,...}` or `{ok:false,error:{code,message}}` in `structuredContent` and as JSON text. +3. Hive is the MCP client: `POST http://127.0.0.1:8765/mcp` with `Authorization: Bearer ` (streamable HTTP, stateless, JSON responses). Tools: `meeting.join`, `meeting.leave`, `meeting.status`, `transcript.read`, `chat.post`, `say`, `say.tone` (the soft listening chime: played by the Vexa bot, and by quorum itself on Discord), `say.prewarm`, `meeting.delete`. Every result is `{ok:true,...}` or `{ok:false,error:{code,message}}` in `structuredContent` and as JSON text. 4. Hive follows each meeting on `GET /events?meeting_id=` (Server-Sent Events, same bearer): `segment`, `chat` and `state` events, one JSON object per `data:` line, `id:` = `seq`. Reconnect with `after_seq=` to replay what was missed. `GET /events` without `meeting_id` carries every meeting, including Discord `/hive join` requests. 5. On SIGTERM quorum leaves Discord channels and exits. Vexa bots keep running until hive calls `meeting.leave`; a restarted quorum that is asked to join the same meeting adopts the running bot. diff --git a/docs/contract.md b/docs/contract.md index 80c5c82..7261415 100644 --- a/docs/contract.md +++ b/docs/contract.md @@ -30,7 +30,7 @@ All tools return `{ok: true, ...}` or `{ok: false, error: {code, message}}`, as | `transcript.read` | `{meeting_id, after_seq?}` | `{segments:[Segment], next_seq}` final segments only, oldest first; `next_seq` is the cursor to pass back as `after_seq` | | `chat.post` | `{meeting_id, text, confirm?}` text max `QUORUM_CHAT_MAX_CHARS`=2000 | `{posted_at, delivery}`. `delivery` is `confirmed` when the platform reports the line in the meeting, else `unconfirmed`. Without `confirm` quorum does not wait: Discord answers `confirmed` (its send returns once the message exists), Vexa `unconfirmed` (the bot only received the line). With `confirm:true` on Vexa, quorum reads the send's delivery record (`GET /bots/{platform}/{native_id}/chat/{request_id}`) every `QUORUM_CHAT_CONFIRM_POLL_MS`=500 for up to `QUORUM_CHAT_CONFIRM_TIMEOUT_MS`=30000 (the Teams bot alone may wait 10 s for the chat composer after admission): `confirmed` once the bot reports it posted, `platform_unavailable` (message names the bot's reason, for example `no_composer` when the meeting chat is not open to the bot) when it reports the send failed, and `unconfirmed` when no verdict came in time or Vexa has no delivery route. A caller that must know the line was seen (the consent notice) passes `confirm:true` and treats `unconfirmed` as not posted. | | `say` | `{meeting_id, text}` text max `QUORUM_SAY_MAX_CHARS`=1000 | `{queued:true, duration_ms?, first_audio_ms?, synth_ms?, cached?}`. Lines of one meeting play one after another, never over each other. On Discord a line is spoken sentence by sentence: the text is cut at `.`, `!`, `?` or `:` followed by whitespace (a closing quote or bracket stays with its sentence; pieces under 24 characters are joined to a neighbour; text with no such break is one sentence), the sentences are synthesized one after another, and each is queued for playback in order as soon as it is ready, so the first sound comes after one sentence's synthesis. The call resolves once the first sentence is ready (playback is queued behind earlier lines). If the first sentence cannot be synthesized the call answers `upstream_error` and nothing of that line is spoken; a later sentence that fails is skipped and logged, and the sentences after it still play. Each sentence's clip gets a 5 ms fade in and out and is cached by voice and text. `duration_ms` (whole milliseconds) is the line's real audio length when known: on Discord only when every sentence was ready by the time the call resolved (a one-sentence line, or a line whose sentences were all cached), on Vexa when the bot reports it. `first_audio_ms` is the time from the request to the first audio: on Discord until the first sentence was handed to the playback queue; on Vexa the bot's own `first_audio_ms` when the playback record has it, else the time until quorum first read the record as `speaking` or `spoken` (late by at most one poll), present only when the line was reported `spoken`. `synth_ms` is the synthesis time before the first audio: on Discord 0 when the whole line came from the cache, else equal to `first_audio_ms`; on Vexa the record's `synth_ms` when it has one. `cached` says every sentence of the line came from the clip cache (Discord), or is the record's `cached` when it has one (Vexa). On Vexa each say is sent only after the meeting's previous say settled; when the speak answer carries a `request_id`, quorum reads the line's playback record (`GET /bots/{platform}/{native_id}/speak/{request_id}`, statuses `queued`, `speaking`, `spoken` with `duration_ms` and, from a bot that reports them, `first_audio_ms`, `synth_ms` and `cached`, `failed` with `reason`) every `QUORUM_SAY_POLL_MS`=200 for up to `QUORUM_SAY_CONFIRM_TIMEOUT_MS`=60000 and resolves once it is `spoken` (or with no verdict in time, or no such route). `platform_unavailable` when the bot was started without voice or Vexa refuses speech (404/409, message `speak is not available: `), when the bot reports the line `failed` (message `the line was not spoken: `, for example `mic_muted`), or when `QUORUM_TTS_URL` is unset (Discord); `upstream_error` when the TTS service fails on the line's first sentence (Discord) | -| `say.tone` | `{meeting_id}`; the meeting must be live (else `not_live`) | `{played}`. Plays the listening tone: a short, soft chime, not speech, that tells the room the bot heard its name and is about to answer. It never waits for the sound and never queues behind speech: `played` is `false` when it was dropped on purpose (a `say` is in progress, the bot has no voice, the bot refused it, or the platform has no tone: Discord and the companion). On Vexa the bot synthesizes the chime itself and plays it at once; it is requested through the speak route with the reserved line `[[hive:listening-tone]]`, so a person never has that line spoken. | +| `say.tone` | `{meeting_id}`; the meeting must be live (else `not_live`) | `{played}`. Plays the listening tone: a short, soft chime, not speech, that tells the room the bot heard its name and is about to answer. It never waits for the sound and never queues behind speech: `played` is `false` when it was dropped on purpose (a `say` is in progress, the bot has no voice, the bot refused it, or the platform has no tone: the companion). On Discord quorum renders the chime in code (two sine notes, G5 then C6, about 200 ms, peak at 18% of full scale with a smooth attack and release, so it cannot click) and plays it into the voice channel at once, asynchronously; it is dropped (`played:false`) when a `say` is queued or playing, when a tone is already playing, when `QUORUM_TTS_URL` is unset (the bot has no voice) or when the bot is not in the channel, and a `say` that starts during the chime cuts it. On Vexa the bot synthesizes the chime itself and plays it at once; it is requested through the speak route with the reserved line `[[hive:listening-tone]]`, so a person never has that line spoken. | | `say.prewarm` | `{meeting_id, texts}` `texts` is 1 to `QUORUM_SAY_PREWARM_MAX_LINES`=32 non-empty strings, each at most `QUORUM_SAY_MAX_CHARS` (else `invalid_argument`); the meeting must be live (else `not_live`) | `{queued}`. Gets fixed lines (an introduction, acknowledgements, a goodbye) ready so a later `say` of the same text starts at once. On Discord each line is cut into sentences as `say` does and every sentence not yet cached is synthesized in the background with the meeting's voice, one request at a time; `queued` is the number of sentences sent for synthesis (0 when all were ready or already on their way). A sentence that fails is skipped and synthesized when it is said. Nothing is played. `queued` is 0 on platforms without their own synthesis in quorum: Vexa (its bot synthesizes), the companion, and Discord without `QUORUM_TTS_URL`. | | `meeting.delete` | `{meeting_id}` | `{deleted:true}` erases transcript upstream | diff --git a/package-lock.json b/package-lock.json index f0f5b1a..ed4ce93 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@hivecommons/quorum", - "version": "0.5.1", + "version": "0.6.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@hivecommons/quorum", - "version": "0.5.1", + "version": "0.6.0", "license": "Apache-2.0", "dependencies": { "@discordjs/voice": "^0.19.2", diff --git a/package.json b/package.json index 00babbd..55197b6 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@hivecommons/quorum", - "version": "0.5.1", + "version": "0.6.0", "description": "Quorum: AI agents as live meeting participants, exposed as MCP tools", "license": "Apache-2.0", "type": "module", diff --git a/src/discord/adapter.ts b/src/discord/adapter.ts index 45a0e10..5886f8a 100644 --- a/src/discord/adapter.ts +++ b/src/discord/adapter.ts @@ -18,6 +18,7 @@ import { Utterance, } from "./audio.js"; import { SpeechSynth, splitSentences } from "./speech.js"; +import { renderListeningTone } from "../tone.js"; import { nameHints, type Transcriber } from "./transcribe.js"; import type { TtsClient } from "./tts.js"; import type { @@ -81,6 +82,8 @@ interface Live { /** Clips in `say` order; each resolves to 48 kHz stereo PCM, or undefined if synthesis failed. */ clips: Promise[]; playing: boolean; + /** The listening tone is playing now; a `say` cuts it and a second tone is dropped. */ + toning: boolean; /** Settles when the queued speech has all played; undefined before the first clip. */ drained?: Promise; /** Per speaker: segments waiting for transcription, and whether one is being transcribed. */ @@ -137,6 +140,8 @@ export class DiscordAdapter implements PlatformAdapter { private readonly now: () => number; /** Synthesis with the clip cache, shared by every meeting; undefined without QUORUM_TTS_URL. */ private readonly synth: SpeechSynth | undefined; + /** The rendered listening tone (24 kHz mono), made on first use. */ + private toneClip: Buffer | undefined; constructor(private readonly opts: DiscordAdapterOptions) { this.client = opts.runtime.createClient({ messageContent: true }); @@ -330,6 +335,7 @@ export class DiscordAdapter implements PlatformAdapter { player: undefined, clips: [], playing: false, + toning: false, stt: new Map(), }; if (this.opts.tts) { @@ -675,6 +681,7 @@ export class DiscordAdapter implements PlatformAdapter { for (const clip of clips) { live.clips.push(clip.catch(() => undefined)); } + this.cutTone(live); this.drain(meeting, live); try { await clips[0]; @@ -695,6 +702,48 @@ export class DiscordAdapter implements PlatformAdapter { }; } + /** + * Plays the listening tone into the channel at once and does not wait for it. Dropped (false) + * when the bot cannot speak (no TTS configured, so no player) or is not in the channel, when a + * line is queued or playing, or when a tone is already playing. A `say` that starts during the + * tone cuts it (cutTone), so the tone never delays speech. Never throws. + */ + async tone(meeting: Meeting): Promise { + const live = this.live.get(meeting.id); + if (!live?.player || live.playing || live.clips.length > 0 || live.toning) { + return false; + } + try { + live.toning = true; + void this.play(live, upsampleToStereo48k(this.tonePcm())).then( + () => { + live.toning = false; + }, + () => { + live.toning = false; + }, + ); + return true; + } catch (err) { + live.toning = false; + this.opts.log.warn("discord: listening tone not played", { error: String(err) }); + return false; + } + } + + /** The chime is rendered on first use and kept: it is the same bytes every time. */ + private tonePcm(): Buffer { + this.toneClip ??= renderListeningTone(); + return this.toneClip; + } + + /** Stops a playing tone so speech starts clean; a no-op when none is playing. */ + private cutTone(live: Live): void { + if (live.toning) { + live.player?.stop(true); + } + } + /** Synthesizes fixed lines into the clip cache in the background (say.prewarm). */ async prewarm(meeting: Meeting, lines: string[]): Promise { if (!this.synth) { diff --git a/src/tone.ts b/src/tone.ts new file mode 100644 index 0000000..4538530 --- /dev/null +++ b/src/tone.ts @@ -0,0 +1,56 @@ +/** + * The listening tone: a short, soft, rising two-note chime a voice agent plays when it heard its + * name and is about to answer (like a smart speaker's wake sound), so the room knows it is + * listening. It is generated here from two sine waves, so there is no sample file and no licence + * to carry, and the same design as the Vexa bot's own tone (listening-tone.ts in hivecommons/vexa), + * so Betty chimes alike on every platform. Pure and deterministic: no I/O, no randomness. + */ + +/** 24 kHz mono s16le, the same format as the TTS service's PCM, so speech helpers apply to it. */ +export const TONE_RATE_HZ = 24_000; +const BYTES_PER_SAMPLE = 2; +const MS_PER_S = 1000; +const INT16_MAX = 32767; + +/** The two notes, rising a perfect fourth: G5 then C6. */ +export const TONE_NOTE_HZ: readonly [number, number] = [783.99, 1046.5]; +/** Each note's length. Two notes make a chime about 200 ms long. */ +export const TONE_NOTE_MS = 100; +/** Attack: a fast, smooth rise (no click at the start). */ +export const TONE_ATTACK_MS = 8; +/** Release: the last of each note fades to exactly zero (no click at the end). */ +export const TONE_RELEASE_MS = 25; +/** How far each note dies away over its length: the level falls to this fraction of its start. */ +export const TONE_DECAY_RATIO = 0.12; +/** The loudest sample as a fraction of full scale: a quiet cue under speech, which peaks higher. */ +export const TONE_PEAK = 0.18; +/** Hard ceiling on the loudest sample as a fraction of full scale, however the above change. */ +export const TONE_PEAK_CAP = 0.25; + +/** The chime as 24 kHz mono signed 16-bit little-endian PCM. */ +export function renderListeningTone(): Buffer { + const noteSamples = Math.round((TONE_RATE_HZ * TONE_NOTE_MS) / MS_PER_S); + const attack = Math.round((TONE_RATE_HZ * TONE_ATTACK_MS) / MS_PER_S); + const release = Math.round((TONE_RATE_HZ * TONE_RELEASE_MS) / MS_PER_S); + const out = Buffer.alloc(noteSamples * TONE_NOTE_HZ.length * BYTES_PER_SAMPLE); + const peak = Math.round(Math.min(TONE_PEAK, TONE_PEAK_CAP) * INT16_MAX); + const decayPerSample = Math.log(TONE_DECAY_RATIO) / noteSamples; + TONE_NOTE_HZ.forEach((hz, n) => { + for (let i = 0; i < noteSamples; i++) { + const rise = i < attack ? Math.sin((Math.PI / 2) * (i / attack)) ** 2 : 1; + const fall = + i >= noteSamples - release + ? Math.sin((Math.PI / 2) * ((noteSamples - i) / release)) ** 2 + : 1; + const level = rise * fall * Math.exp(decayPerSample * i); + const v = Math.round(peak * level * Math.sin((2 * Math.PI * hz * i) / TONE_RATE_HZ)); + out.writeInt16LE(v, (n * noteSamples + i) * BYTES_PER_SAMPLE); + } + }); + return out; +} + +/** The length of rendered tone PCM in milliseconds. */ +export function toneDurationMs(pcm: Buffer): number { + return Math.round((pcm.length / BYTES_PER_SAMPLE / TONE_RATE_HZ) * MS_PER_S); +} diff --git a/src/version.ts b/src/version.ts index e8e8638..1f4fcca 100644 --- a/src/version.ts +++ b/src/version.ts @@ -1,5 +1,5 @@ /** The package version, kept in step with package.json by the release workflow check. */ -export const VERSION = "0.5.1"; +export const VERSION = "0.6.0"; /** The contract this build implements (docs/contract.md). */ export const CONTRACT_VERSION = "quorum.contract/v1"; diff --git a/test/e2e-discord.test.ts b/test/e2e-discord.test.ts index 9d727ca..0377761 100644 --- a/test/e2e-discord.test.ts +++ b/test/e2e-discord.test.ts @@ -21,6 +21,7 @@ import { } from "./helpers/fakeDiscord.js"; import { pcmFrames, upsampleToStereo48k } from "../src/discord/audio.js"; import { fadeEdges } from "../src/discord/speech.js"; +import { renderListeningTone } from "../src/tone.js"; import { openSse, startHarness, @@ -1107,6 +1108,75 @@ describe("DiscordAdapter lifecycle", () => { }); }); +describe("Discord listening tone", () => { + const toneFrames = (): Buffer[] => pcmFrames(upsampleToStereo48k(renderListeningTone())); + + it("plays the chime at once as 48 kHz stereo Opus and answers played:true", async () => { + await boot(ttsEnv()); + const id = await joinDiscord(); + expect(await H().call("say.tone", { meeting_id: id })).toEqual({ ok: true, played: true }); + expect(player().played.length).toBe(1); + const got = await packets(player().played[0] as never); + expect(got.length).toBe(toneFrames().length); + expect(tts.requests.length).toBe(0); + expect(rt.encodersFreed).toBe(1); + // While it plays a second request is dropped; once it finished, the next one plays. + expect(await H().call("say.tone", { meeting_id: id })).toEqual({ ok: true, played: false }); + player().finish(); + await new Promise((r) => setImmediate(r)); + expect(await H().call("say.tone", { meeting_id: id })).toEqual({ ok: true, played: true }); + expect(player().played.length).toBe(2); + player().finish(); + }); + + it("is dropped while a line is queued or playing, and never queues behind it", async () => { + await boot(ttsEnv()); + const id = await joinDiscord(); + tts.replies.push({ status: 200, body: clip(1) }); + await H().call("say", { meeting_id: id, text: "Hello, I am Betty." }); + await until(() => player().played.length === 1); + expect(await H().call("say.tone", { meeting_id: id })).toEqual({ ok: true, played: false }); + expect(player().played.length).toBe(1); + player().finish(); + }); + + it("is cut cleanly when a line starts during it, and the line plays", async () => { + await boot(ttsEnv()); + const id = await joinDiscord(); + expect(await H().call("say.tone", { meeting_id: id })).toMatchObject({ played: true }); + tts.replies.push({ status: 200, body: clip(7) }); + await H().call("say", { meeting_id: id, text: "Hello, I am Betty." }); + expect(player().stops).toBe(1); + await until(() => player().played.length === 2); + expect(await playedValue(1)).toBe(7); + player().finish(); + }); + + it("is dropped when the bot has no voice (no TTS) and when the bot is no longer in the channel", async () => { + await boot(); + const id = await joinDiscord(); + expect(await H().call("say.tone", { meeting_id: id })).toEqual({ ok: true, played: false }); + await H().call("meeting.leave", { meeting_id: id }); + expect(await H().call("say.tone", { meeting_id: id })).toMatchObject({ + ok: false, + error: { code: "not_live" }, + }); + const meeting = H().q.quorum.registry.get(id); + const adapter = H().q.quorum["adapter"]("discord"); + expect(await adapter.tone?.(meeting as never)).toBe(false); + }); + + it("answers without an error when the player fails, and frees the player for the next tone", async () => { + await boot(ttsEnv()); + const id = await joinDiscord(); + player().playError = new Error("boom"); + expect(await H().call("say.tone", { meeting_id: id })).toEqual({ ok: true, played: true }); + // The failed play frees the player: a tone right after is allowed again. + await new Promise((r) => setImmediate(r)); + expect(await H().call("say.tone", { meeting_id: id })).toEqual({ ok: true, played: true }); + }); +}); + describe("discordAdapters factory", () => { const ctx = (env: Record) => ({ config: loadConfig(env), diff --git a/test/tone.test.ts b/test/tone.test.ts new file mode 100644 index 0000000..76255b4 --- /dev/null +++ b/test/tone.test.ts @@ -0,0 +1,54 @@ +import { describe, expect, it } from "vitest"; +import { + renderListeningTone, + toneDurationMs, + TONE_ATTACK_MS, + TONE_NOTE_HZ, + TONE_NOTE_MS, + TONE_PEAK_CAP, + TONE_RATE_HZ, +} from "../src/tone.js"; + +const INT16_MAX = 32767; +const MS_PER_S = 1000; + +function samples(pcm: Buffer): number[] { + return Array.from({ length: pcm.length / 2 }, (_, i) => pcm.readInt16LE(i * 2)); +} + +describe("listening tone", () => { + const pcm = renderListeningTone(); + const s = samples(pcm); + const note = (TONE_RATE_HZ * TONE_NOTE_MS) / MS_PER_S; + + it("is short: two notes, a fifth of a second, 24 kHz mono s16le", () => { + expect(s.length).toBe(note * TONE_NOTE_HZ.length); + expect(toneDurationMs(pcm)).toBe(TONE_NOTE_MS * TONE_NOTE_HZ.length); + }); + + it("is soft: never louder than the peak cap", () => { + const peak = Math.max(...s.map(Math.abs)); + expect(peak).toBeGreaterThan(0); + expect(peak).toBeLessThanOrEqual(Math.round(TONE_PEAK_CAP * INT16_MAX)); + }); + + it("starts and ends at silence, and each note fades to zero, so it cannot click", () => { + expect(s[0]).toBe(0); + expect(s[note]).toBe(0); + expect(s[s.length - 1]).toBe(0); + const attack = (TONE_RATE_HZ * TONE_ATTACK_MS) / MS_PER_S; + const early = Math.max(...s.slice(0, attack / 4).map(Math.abs)); + const later = Math.max(...s.slice(attack * 2, attack * 3).map(Math.abs)); + expect(early).toBeLessThan(later); + }); + + it("is the same bytes every time", () => { + expect(renderListeningTone().equals(pcm)).toBe(true); + }); + + it("plays two notes: the second half is a higher pitch than the first", () => { + const crossings = (part: number[]): number => + part.slice(1).filter((v, i) => (part[i] ?? 0) < 0 !== v < 0).length; + expect(crossings(s.slice(note, note * 2))).toBeGreaterThan(crossings(s.slice(0, note))); + }); +});