From 7394c3d388bb3964cd35ab267e21cb892e12f5da Mon Sep 17 00:00:00 2001 From: "@mrubens" <2600+mrubens@users.noreply.github.com> Date: Sat, 12 Sep 2026 01:05:01 +0000 Subject: [PATCH 1/2] feat: add concise voice progress bridges --- apps/web/src/lib/server/voice.test.ts | 24 +++++++-- apps/web/src/lib/server/voice.ts | 6 ++- .../__tests__/fast-agent-prompt.test.ts | 49 ++++++++++++++++--- .../__tests__/fast-agent-service.test.ts | 35 +++++++++++++ .../server/fast-agent/fast-agent-prompt.ts | 14 +++--- .../server/fast-agent/fast-agent-service.ts | 5 +- 6 files changed, 114 insertions(+), 19 deletions(-) diff --git a/apps/web/src/lib/server/voice.test.ts b/apps/web/src/lib/server/voice.test.ts index 473eae634..f30b211df 100644 --- a/apps/web/src/lib/server/voice.test.ts +++ b/apps/web/src/lib/server/voice.test.ts @@ -109,9 +109,24 @@ describe('createVoiceLiveSession', () => { expect(body.session.instructions).toContain( 'Integrations the backend can use: GitHub.', ); - // The voice acknowledges, delegates real work, and reports results - // faithfully in its own words. - expect(body.session.instructions).toContain('Backchannel policy'); + // The voice bridges likely silence, delegates real work, and reports + // results faithfully in its own words. + expect(body.session.instructions).toContain('Bridge policy'); + expect(body.session.instructions).toContain( + 'Before delegating a request whose backend work is likely to create noticeable silence', + ); + expect(body.session.instructions).toContain( + 'Just a second while I read this Notion document', + ); + expect(body.session.instructions).toContain( + 'Thirty seconds or more without useful speech is generally too long', + ); + expect(body.session.instructions).toContain( + 'Do not repeat equivalent updates', + ); + expect(body.session.instructions).toContain( + 'Stop speaking the moment the person starts talking', + ); expect(body.session.instructions).toContain('Delegate to the backend when'); expect(body.session.instructions).toContain( 'Do not delegate to the backend when', @@ -120,6 +135,9 @@ describe('createVoiceLiveSession', () => { expect(body.session.instructions).toContain( 'keep every number, name, path, and link label exactly as given', ); + expect(body.session.instructions).toContain( + 'Never claim work finished or a result exists before commentary says so', + ); }); it('creates a short preview through OpenAI speech with the selected voice', async () => { diff --git a/apps/web/src/lib/server/voice.ts b/apps/web/src/lib/server/voice.ts index a742ae4b1..546c5cec6 100644 --- a/apps/web/src/lib/server/voice.ts +++ b/apps/web/src/lib/server/voice.ts @@ -177,7 +177,9 @@ The person will mostly talk about their code repositories, pull requests, issues ${formatVoiceWorkspaceContext(context)} -Backchannel policy: Acknowledge each request in a few words right away ("Sure.", "I'll check.") and then wait for the backend. Do not narrate while waiting; if the wait runs long, one brief "still working on it" is enough. +Bridge policy: Before delegating a request whose backend work is likely to create noticeable silence, say one short sentence naming the user-visible activity, then delegate immediately. For example: "Just a second while I read this Notion document" or "I'm starting the coding work now." This spoken bridge is the request's acknowledgement. Do not add a generic acknowledgement before or after it, narrate every backend action, mention tool names or orchestration, or claim an outcome before commentary confirms it. + +Waiting policy: Thirty seconds or more without useful speech is generally too long. While delegated work is still pending, if roughly 30 seconds have passed since the last useful spoken bridge or update, give one concise sentence grounded in the same user-visible work. Continue with a new sentence only for a meaningful phase change or another prolonged silent interval. Do not repeat equivalent updates, invent progress, describe low-level activity, or speak merely to fill space. Interruption policy: Stop speaking the moment the person starts talking, and listen. @@ -198,7 +200,7 @@ Grounding policy: Reporting policy: - Commentary is the backend's result. Report it in your own words, faithfully and completely: keep every number, name, path, and link label exactly as given, and do not add conclusions the backend did not state. Never claim work finished or a result exists before commentary says so. -- Commentary may arrive in pieces; start speaking as soon as the first piece arrives and continue smoothly.`; +- Commentary may arrive in pieces; start speaking as soon as the first useful piece arrives and continue smoothly. Treat meaningful changes, blockers, decisions, and confirmed completion as immediately useful, but do not repeat an equivalent bridge or update.`; } export async function createVoiceLiveSession(options: { diff --git a/packages/cloud-agents/src/server/fast-agent/__tests__/fast-agent-prompt.test.ts b/packages/cloud-agents/src/server/fast-agent/__tests__/fast-agent-prompt.test.ts index 9e653ce3d..05dd1d4cb 100644 --- a/packages/cloud-agents/src/server/fast-agent/__tests__/fast-agent-prompt.test.ts +++ b/packages/cloud-agents/src/server/fast-agent/__tests__/fast-agent-prompt.test.ts @@ -478,7 +478,7 @@ describe('buildFastAgentSystemPrompt', () => { expect(turnStartupIndex).toBeLessThan(prompt.indexOf(laterSection)); } expect(prompt).toContain( - 'the first model-selected action must communicate with the user before substantive model-invoked work', + "the first model-selected action must communicate with the user before substantive model-invoked work, except that a marked voice turn's activity-specific spoken bridge already satisfies this requirement", ); expect(prompt).toContain( 'use `send_chat_reply` with purpose `ack` before calling `launch_task`', @@ -497,7 +497,7 @@ describe('buildFastAgentSystemPrompt', () => { 'If launch fails, explain the failure through the normal closeout or clarification path', ); expect(prompt).toContain( - 'Before Brain recall, integrations, subagents, task steering, skills, result recovery, widgets, memory, custom automation management, or any other model-invoked work, communicate first', + "Before Brain recall, integrations, subagents, task steering, skills, result recovery, widgets, memory, custom automation management, or any other model-invoked work, communicate first unless the marked voice turn's spoken bridge already did so", ); expect(prompt).toContain( 'Trusted platform events follow their dedicated rules instead of this startup contract', @@ -841,7 +841,10 @@ describe('buildFastAgentSystemPrompt', () => { 'post one brief consolidated factual status for the Session when either inspection finds a genuinely notable new development', ); expect(prompt).toContain( - 'or the user has received no useful user-visible work update during the current automatic-check interval', + 'or the user has received no useful work update for roughly 30 seconds on a voice-marked check or roughly 10 minutes otherwise', + ); + expect(prompt).toContain( + 'These reporting windows are separate from the automatic check cadence', ); expect(prompt).toContain( 'Important news is immediate and has no minimum wait', @@ -881,7 +884,7 @@ describe('buildFastAgentSystemPrompt', () => { ); expect(prompt).toContain('automatic monitoring must never reactivate it'); expect(prompt).toContain( - 'report notable new developments immediately or one factual consolidated status when there has been no useful visible work update during the current automatic-check interval', + 'report notable new developments immediately or one factual consolidated status when there has been no useful work update for roughly 30 seconds on a voice-marked check or roughly 10 minutes otherwise', ); expect(prompt).toContain( 'otherwise stay silent while still rearming if work runs', @@ -899,7 +902,7 @@ describe('buildFastAgentSystemPrompt', () => { ); }); - it('keeps one cache-stable follow-through contract for voice and text cadence', () => { + it('separates voice check cadence from deduplicated spoken reporting', () => { const prompt = buildFastAgentSystemPrompt({ availableEnvironments: [], turnSource: 'platform_event', @@ -907,14 +910,20 @@ describe('buildFastAgentSystemPrompt', () => { }); expect(prompt).toContain( - 'user-visible work update during the current automatic-check interval', + 'no useful work update for roughly 30 seconds on a voice-marked check or roughly 10 minutes otherwise', ); expect(prompt).toContain('stable nominal schedule "in 10m"'); expect(prompt).toContain( 'the server replaces that nominal delay with "in 1m" while voice is currently active and otherwise keeps "in 10m"', ); expect(prompt).toContain( - 'Keep routine spoken updates especially concise, applying these same reporting and repetition rules rather than inventing another suppression policy', + "Check the conversation's actual visible updates: a recent useful update suppresses only a routine cadence status", + ); + expect(prompt).toContain( + 'When neither reporting condition is met, call "ignore_event" after ensuring the next check', + ); + expect(prompt).toContain( + "Keep every spoken update to one concise sentence about the user's work", ); expect(prompt).toContain( 'do not narrate routine logs, invent progress, repeat an already reported development', @@ -1084,6 +1093,32 @@ describe('buildFastAgentSystemPrompt', () => { ); }); + it('bridges likely voice silence without narrating tools or duplicating acknowledgements', () => { + const prompt = buildFastAgentSystemPrompt({ availableEnvironments: [] }); + + expect(prompt).toContain( + 'The voice already gave one activity-specific spoken bridge before handing you the request', + ); + expect(prompt).toContain( + 'the runtime counts it as startup communication. Do not send another opening acknowledgement', + ); + expect(prompt).toContain( + 'Before a later operation likely to create another noticeable silence', + ); + expect(prompt).toContain( + 'reading a Notion document, inspecting relevant files, starting coding work, validating a fix, or waiting on tests', + ); + expect(prompt).toContain( + 'Treat roughly 30 seconds without useful speech during active work as a prolonged silent interval', + ); + expect(prompt).toContain( + 'Deduplicate equivalent updates and otherwise stay silent', + ); + expect(prompt).toContain( + 'Do not name tools, integrations as machinery, orchestration, wakeups, or routine logs', + ); + }); + it('provides repository-focused coding task acknowledgement guidance', () => { const prompt = buildFastAgentSystemPrompt({ availableEnvironments: [] }); diff --git a/packages/cloud-agents/src/server/fast-agent/__tests__/fast-agent-service.test.ts b/packages/cloud-agents/src/server/fast-agent/__tests__/fast-agent-service.test.ts index cfa9561f8..4ba11b1c2 100644 --- a/packages/cloud-agents/src/server/fast-agent/__tests__/fast-agent-service.test.ts +++ b/packages/cloud-agents/src/server/fast-agent/__tests__/fast-agent-service.test.ts @@ -7278,6 +7278,41 @@ describe('answerFastAgentQuestion native OpenCode tools', () => { tools: [{ name: 'search_code', inputSchema: { type: 'object' } }], }; + it('counts the voice bridge as startup communication without posting a duplicate acknowledgement', async () => { + mocks.listIntegrations.mockResolvedValue([githubIntegration]); + let result: unknown; + mocks.generateText.mockImplementation( + async (_params, _session, options) => { + await options.onSessionReady('opencode-session-1'); + result = await invokeMcpTool('github', 'search_code', { + query: 'voice progress', + }); + await invokeTool(nativeToolNames.sendChatReply, { + purpose: 'closeout', + message: 'I found it.', + }); + return ''; + }, + ); + const adapter = callbacks(); + + await answerFastAgentQuestion({ + ...baseParams, + voiceMode: true, + adapter, + }); + + expect(result).toEqual({ + success: true, + result: { matches: ['fast-agent.ts'] }, + }); + expect(adapter.postReply).toHaveBeenCalledOnce(); + expect(adapter.postReply).toHaveBeenCalledWith({ + purpose: 'closeout', + message: 'I found it.', + }); + }); + it('does not let a reaction unlock work, but a text acknowledgement does', async () => { mocks.listIntegrations.mockResolvedValue([githubIntegration]); const results: unknown[] = []; diff --git a/packages/cloud-agents/src/server/fast-agent/fast-agent-prompt.ts b/packages/cloud-agents/src/server/fast-agent/fast-agent-prompt.ts index 065961bbe..0e1bd5d46 100644 --- a/packages/cloud-agents/src/server/fast-agent/fast-agent-prompt.ts +++ b/packages/cloud-agents/src/server/fast-agent/fast-agent-prompt.ts @@ -34,8 +34,10 @@ function buildVoiceModeInstructions(): string { When the current input begins with the platform-generated \`\` marker, that message was spoken on a voice call and your reply will be reported aloud by the call's voice rather than shown as a chat message. Apply these rules only on a marked turn: - Write for the ear: short plain-prose sentences. No Markdown, headings, bullet lists, tables, code blocks, or emoji. - Lead with the answer or outcome. Include every number, name, branch, file path, and link label the person needs, exactly; the voice keeps them verbatim. Prefer "the pull request Fix login redirect" to a raw URL. -- Do not open with an acknowledgement; the voice already said one. Do not describe what you are about to do; do it and report. -- When you launch a task, say so in one sentence and say what the person will hear when it finishes. Progress narration stays in the transcript's tool activity, not in the reply. +- The voice already gave one activity-specific spoken bridge before handing you the request, and the runtime counts it as startup communication. Do not send another opening acknowledgement. Start the work directly. +- Before a later operation likely to create another noticeable silence, send one concise progress sentence naming only the user-visible activity, such as reading a Notion document, inspecting relevant files, starting coding work, validating a fix, or waiting on tests. Do not name tools, integrations as machinery, orchestration, wakeups, or routine logs. Do not send another bridge when the existing one still explains the wait. +- Treat roughly 30 seconds without useful speech during active work as a prolonged silent interval. Report a meaningful phase change, concrete blocker or needed decision, confirmed completion, or one brief factual update when that interval passes. Deduplicate equivalent updates and otherwise stay silent; never invent progress or speak merely to fill space. +- When you launch a task, the opening bridge already covers startup. Report later work only under the same phase-change, silence, blocker, decision, and completion rules. - If you need a decision from the person, ask one clear question.`; } @@ -292,11 +294,11 @@ ${ return `You are ${PRODUCT_NAME} in fast mode on ${surfaceName}. You are the conversational orchestrator for this conversation, not a router and not a transparent relay to a sandbox task. You own the conversation, answer directly when possible, and deliberately delegate execution work when useful. ${releaseIdentifier}## Turn Startup (Highest Priority) -- On every response-required human turn, the first model-selected action must communicate with the user before substantive model-invoked work. +- On every response-required human turn, the first model-selected action must communicate with the user before substantive model-invoked work, except that a marked voice turn's activity-specific spoken bridge already satisfies this requirement. - When work will continue in a coding task, use \`send_chat_reply\` with purpose \`ack\` before calling \`launch_task\`. A reaction never satisfies this startup requirement, including an "eyes" reaction. - A direct closeout or clarification that fully handles the turn is already the first communication; do not prepend a separate acknowledgement. - The acknowledgement streams independently of coding-task startup, so send it first and do not wait for launch or workspace provisioning. Do not repeat it after \`launch_task\`. If launch fails, explain the failure through the normal closeout or clarification path. -- Before Brain recall, integrations, subagents, task steering, skills, result recovery, widgets, memory, custom automation management, or any other model-invoked work, communicate first. Brain recall remains the first context or work call when its instructions require one, but it comes after the acknowledgement. +- Before Brain recall, integrations, subagents, task steering, skills, result recovery, widgets, memory, custom automation management, or any other model-invoked work, communicate first unless the marked voice turn's spoken bridge already did so. Brain recall remains the first context or work call when its instructions require one, but it comes after that communication. - After acknowledging, continue the same turn through the needed work and finish with a closeout or clarification. Do not stop at the acknowledgement. - An eligible ambient message or optional human reaction may use \`ignore_event\` under its narrow rule below. Trusted platform events follow their dedicated rules instead of this startup contract. @@ -439,7 +441,7 @@ ${emailCadenceGuidance}- Prefer one direct closeout over an acknowledgement foll - For a scheduled check, use the current turn's platform-generated voice marker only to decide whether its reply will be spoken. Never infer voice activity from the originating turn or choose the next wakeup delay yourself; the server resolves current persisted call state when scheduling. - On that session check, inspect every task currently listed in this prompt as active or resumable for this conversation: get each current summary and recent messages, then compare the evidence with the user's goals and accepted instructions in this conversation. Count a task as still running only when current evidence shows it is booting or actively executing. A task that is stopped, waiting for input, completed, failed, canceled, or merely resumable does not keep the monitor alive. Never treat an inspection failure or missing evidence as success; report a concise capability blocker when useful, do not rearm, and stop the monitor on capability loss. - When concrete evidence shows drift, a missed requirement, or an actionable blocker a running task can resolve within the accepted scope, use "send_task_message" to send one specific corrective instruction to that task, naming the evidence and expected correction. Before sending, verify the same correction is not already queued, accepted, recorded, addressed, or superseded. Do not steer on silence alone, invent progress or problems, expand scope, or reactivate stopped, waiting, finished, failed, or canceled work. -- If at least one task remains running, post one brief consolidated factual status for the Session when either inspection finds a genuinely notable new development, such as an important milestone, actionable blocker, needed input, or corrective action, or the user has received no useful user-visible work update during the current automatic-check interval. Important news is immediate and has no minimum wait. Check the conversation's actual visible updates: a recent useful update suppresses only a routine cadence status, not inspection, corrective action, or the next timer. Say what remains underway or blocked based on the inspected evidence; do not narrate routine logs, invent progress, repeat an already reported development, or emit separate per-task or duplicate lifecycle notifications. Keep routine spoken updates especially concise, applying these same reporting and repetition rules rather than inventing another suppression policy. When neither reporting condition is met, call "ignore_event" after ensuring the next check. In all cases with running work, list active wakeups and ensure exactly one equivalent next one-shot check exists by creating it with the stable nominal schedule "in 10m", the same name, prompt, and reportPolicy, passing "internal": true; the server replaces that nominal delay with "in 1m" while voice is currently active and otherwise keeps "in 10m", retiring a mismatched active check. Delivery timing is best effort. If no task remains running, do not rearm; report only newly useful completion, blocker, needed input, or corrective action not already reported, otherwise call "ignore_event". This rearming exception is only for automatic own-task Session follow-through; it does not loosen the consent, finite-bound, or no-renewal rules for unrelated external-process monitoring. +- If at least one task remains running, post one brief consolidated factual status for the Session when either inspection finds a genuinely notable new development, such as an important milestone, actionable blocker, needed input, or corrective action, or the user has received no useful work update for roughly 30 seconds on a voice-marked check or roughly 10 minutes otherwise. These reporting windows are separate from the automatic check cadence. Important news is immediate and has no minimum wait. Check the conversation's actual visible updates: a recent useful update suppresses only a routine cadence status, not inspection, corrective action, or the next timer. Say what remains underway or blocked based on the inspected evidence; do not narrate routine logs, invent progress, repeat an already reported development, or emit separate per-task or duplicate lifecycle notifications. Keep every spoken update to one concise sentence about the user's work. When neither reporting condition is met, call "ignore_event" after ensuring the next check. In all cases with running work, list active wakeups and ensure exactly one equivalent next one-shot check exists by creating it with the stable nominal schedule "in 10m", the same name, prompt, and reportPolicy, passing "internal": true; the server replaces that nominal delay with "in 1m" while voice is currently active and otherwise keeps "in 10m", retiring a mismatched active check. Delivery timing is best effort. If no task remains running, do not rearm; report only newly useful completion, blocker, needed input, or corrective action not already reported, otherwise call "ignore_event". This rearming exception is only for automatic own-task Session follow-through; it does not loosen the consent, finite-bound, or no-renewal rules for unrelated external-process monitoring. - Migrate only legacy automatic own-task monitors created under the prior exact-task recurring policy: on launch, cancel those active per-task monitors before ensuring the session check; when one of their wakeups fires, cancel it if still active and treat it as this session check only when no equivalent session check is already active. If an equivalent session check already exists, stay silent instead of duplicating its inspection or report. Leave every unrelated reminder or external-process monitor unchanged. ## Orchestration Policy @@ -502,7 +504,7 @@ ${ - Do the work the prompt asks for. Apply the same scope-based exploration and execution delegation rules as human turns. - \`reportPolicy\` governs whether to speak. With "always", finish with one closeout addressed to the user. With "only_when_notable", post a closeout only when there is news, a result, a blocker, or a required decision; otherwise call "ignore_event". - When the monitored condition has resolved or the wakeup is no longer relevant, cancel it with "manage_wakeups" (action "cancel", the event's \`wakeupId\`) and say so in the closeout. \`nextRunAt\` is null when this was the final run; a finished wakeup needs no cancel. -- For the own-task session check above, follow its reporting and rearming rules instead: report notable new developments immediately or one factual consolidated status when there has been no useful visible work update during the current automatic-check interval; otherwise stay silent while still rearming if work runs. A check with no running work stays silent unless it found newly useful completion, blocker, input, or corrective-action news. This overrides the generic instruction to announce a resolved monitor. The session check's prompt explicitly authorizes creating its next one-shot only while running work remains. +- For the own-task session check above, follow its reporting and rearming rules instead: report notable new developments immediately or one factual consolidated status when there has been no useful work update for roughly 30 seconds on a voice-marked check or roughly 10 minutes otherwise; otherwise stay silent while still rearming if work runs. A check with no running work stays silent unless it found newly useful completion, blocker, input, or corrective-action news. This overrides the generic instruction to announce a resolved monitor. The session check's prompt explicitly authorizes creating its next one-shot only while running work remains. - Do not create another wakeup from a wakeup turn unless the prompt explicitly asks you to schedule the next check. ` : '' diff --git a/packages/cloud-agents/src/server/fast-agent/fast-agent-service.ts b/packages/cloud-agents/src/server/fast-agent/fast-agent-service.ts index d197bc4bd..aad7ecc02 100644 --- a/packages/cloud-agents/src/server/fast-agent/fast-agent-service.ts +++ b/packages/cloud-agents/src/server/fast-agent/fast-agent-service.ts @@ -3315,7 +3315,10 @@ export async function answerFastAgentQuestion({ ), ); let visibleUpdatePosted = resumedWithDeliveredAcknowledgement; - let substantiveWorkAcknowledged = resumedWithDeliveredAcknowledgement; + // GPT-Live speaks an activity-specific bridge before delegating a voice + // request, so requiring another Fast acknowledgement would repeat it. + let substantiveWorkAcknowledged = + voiceMode || resumedWithDeliveredAcknowledgement; let nativeToolInvoked = false; let retriedTaskStart = false; From 7808103a79d93e2af1c5439f4f3043f1b2e402ef Mon Sep 17 00:00:00 2001 From: "@mrubens" <2600+mrubens@users.noreply.github.com> Date: Sat, 12 Sep 2026 01:25:47 +0000 Subject: [PATCH 2/2] fix: enforce voice progress timing --- apps/web/src/lib/server/voice.test.ts | 4 +-- apps/web/src/lib/server/voice.ts | 2 +- .../__tests__/fast-agent-prompt.test.ts | 20 ++++++------ .../__tests__/fast-agent-service.test.ts | 32 +++++++++++++------ .../server/fast-agent/fast-agent-prompt.ts | 12 +++---- .../server/fast-agent/fast-agent-service.ts | 5 +-- .../server/session-wakeups/service.test.ts | 20 ++++++------ .../src/server/session-wakeups/service.ts | 4 +-- 8 files changed, 55 insertions(+), 44 deletions(-) diff --git a/apps/web/src/lib/server/voice.test.ts b/apps/web/src/lib/server/voice.test.ts index f30b211df..4ed6f02fe 100644 --- a/apps/web/src/lib/server/voice.test.ts +++ b/apps/web/src/lib/server/voice.test.ts @@ -113,10 +113,10 @@ describe('createVoiceLiveSession', () => { // results faithfully in its own words. expect(body.session.instructions).toContain('Bridge policy'); expect(body.session.instructions).toContain( - 'Before delegating a request whose backend work is likely to create noticeable silence', + 'Delegate substantive work immediately without acknowledging it yourself', ); expect(body.session.instructions).toContain( - 'Just a second while I read this Notion document', + 'The backend will return one short activity-specific bridge before it starts work likely to create noticeable silence', ); expect(body.session.instructions).toContain( 'Thirty seconds or more without useful speech is generally too long', diff --git a/apps/web/src/lib/server/voice.ts b/apps/web/src/lib/server/voice.ts index 546c5cec6..22f18b5cc 100644 --- a/apps/web/src/lib/server/voice.ts +++ b/apps/web/src/lib/server/voice.ts @@ -177,7 +177,7 @@ The person will mostly talk about their code repositories, pull requests, issues ${formatVoiceWorkspaceContext(context)} -Bridge policy: Before delegating a request whose backend work is likely to create noticeable silence, say one short sentence naming the user-visible activity, then delegate immediately. For example: "Just a second while I read this Notion document" or "I'm starting the coding work now." This spoken bridge is the request's acknowledgement. Do not add a generic acknowledgement before or after it, narrate every backend action, mention tool names or orchestration, or claim an outcome before commentary confirms it. +Bridge policy: Delegate substantive work immediately without acknowledging it yourself. The backend will return one short activity-specific bridge before it starts work likely to create noticeable silence. Speak that commentary as soon as it arrives; do not add a generic acknowledgement before or after it, narrate every backend action, mention tool names or orchestration, or claim an outcome before commentary confirms it. Waiting policy: Thirty seconds or more without useful speech is generally too long. While delegated work is still pending, if roughly 30 seconds have passed since the last useful spoken bridge or update, give one concise sentence grounded in the same user-visible work. Continue with a new sentence only for a meaningful phase change or another prolonged silent interval. Do not repeat equivalent updates, invent progress, describe low-level activity, or speak merely to fill space. diff --git a/packages/cloud-agents/src/server/fast-agent/__tests__/fast-agent-prompt.test.ts b/packages/cloud-agents/src/server/fast-agent/__tests__/fast-agent-prompt.test.ts index 05dd1d4cb..1c895dae2 100644 --- a/packages/cloud-agents/src/server/fast-agent/__tests__/fast-agent-prompt.test.ts +++ b/packages/cloud-agents/src/server/fast-agent/__tests__/fast-agent-prompt.test.ts @@ -478,7 +478,7 @@ describe('buildFastAgentSystemPrompt', () => { expect(turnStartupIndex).toBeLessThan(prompt.indexOf(laterSection)); } expect(prompt).toContain( - "the first model-selected action must communicate with the user before substantive model-invoked work, except that a marked voice turn's activity-specific spoken bridge already satisfies this requirement", + 'the first model-selected action must communicate with the user before substantive model-invoked work', ); expect(prompt).toContain( 'use `send_chat_reply` with purpose `ack` before calling `launch_task`', @@ -497,7 +497,7 @@ describe('buildFastAgentSystemPrompt', () => { 'If launch fails, explain the failure through the normal closeout or clarification path', ); expect(prompt).toContain( - "Before Brain recall, integrations, subagents, task steering, skills, result recovery, widgets, memory, custom automation management, or any other model-invoked work, communicate first unless the marked voice turn's spoken bridge already did so", + 'Before Brain recall, integrations, subagents, task steering, skills, result recovery, widgets, memory, custom automation management, or any other model-invoked work, communicate first', ); expect(prompt).toContain( 'Trusted platform events follow their dedicated rules instead of this startup contract', @@ -776,7 +776,7 @@ describe('buildFastAgentSystemPrompt', () => { 'the runtime silently ensures this conversation has exactly one internal session-wide one-shot check', ); expect(prompt).toContain( - '"in 1m" while a voice call is active, otherwise "in 10m"', + '"in 30s" while a voice call is active, otherwise "in 10m"', ); expect(prompt).toContain('Do not create another wakeup for this purpose'); expect(prompt).toContain('passing "internal": true'); @@ -843,9 +843,6 @@ describe('buildFastAgentSystemPrompt', () => { expect(prompt).toContain( 'or the user has received no useful work update for roughly 30 seconds on a voice-marked check or roughly 10 minutes otherwise', ); - expect(prompt).toContain( - 'These reporting windows are separate from the automatic check cadence', - ); expect(prompt).toContain( 'Important news is immediate and has no minimum wait', ); @@ -914,7 +911,7 @@ describe('buildFastAgentSystemPrompt', () => { ); expect(prompt).toContain('stable nominal schedule "in 10m"'); expect(prompt).toContain( - 'the server replaces that nominal delay with "in 1m" while voice is currently active and otherwise keeps "in 10m"', + 'the server replaces that nominal delay with "in 30s" while voice is currently active and otherwise keeps "in 10m"', ); expect(prompt).toContain( "Check the conversation's actual visible updates: a recent useful update suppresses only a routine cadence status", @@ -1093,14 +1090,17 @@ describe('buildFastAgentSystemPrompt', () => { ); }); - it('bridges likely voice silence without narrating tools or duplicating acknowledgements', () => { + it('requires Fast to bridge likely voice silence before opening the work gate', () => { const prompt = buildFastAgentSystemPrompt({ availableEnvironments: [] }); expect(prompt).toContain( - 'The voice already gave one activity-specific spoken bridge before handing you the request', + 'Before the first integration, task, or other substantive operation likely to create noticeable silence', + ); + expect(prompt).toContain( + 'send one short activity-specific bridge with purpose "ack"', ); expect(prompt).toContain( - 'the runtime counts it as startup communication. Do not send another opening acknowledgement', + 'This delivered Fast reply opens the runtime work-start gate; do not rely on the voice client having spoken first', ); expect(prompt).toContain( 'Before a later operation likely to create another noticeable silence', diff --git a/packages/cloud-agents/src/server/fast-agent/__tests__/fast-agent-service.test.ts b/packages/cloud-agents/src/server/fast-agent/__tests__/fast-agent-service.test.ts index 4ba11b1c2..2ca0cf39e 100644 --- a/packages/cloud-agents/src/server/fast-agent/__tests__/fast-agent-service.test.ts +++ b/packages/cloud-agents/src/server/fast-agent/__tests__/fast-agent-service.test.ts @@ -7278,15 +7278,26 @@ describe('answerFastAgentQuestion native OpenCode tools', () => { tools: [{ name: 'search_code', inputSchema: { type: 'object' } }], }; - it('counts the voice bridge as startup communication without posting a duplicate acknowledgement', async () => { + it('keeps voice work gated until Fast posts the activity bridge', async () => { mocks.listIntegrations.mockResolvedValue([githubIntegration]); - let result: unknown; + const results: unknown[] = []; mocks.generateText.mockImplementation( async (_params, _session, options) => { await options.onSessionReady('opencode-session-1'); - result = await invokeMcpTool('github', 'search_code', { - query: 'voice progress', + results.push( + await invokeMcpTool('github', 'search_code', { + query: 'before bridge', + }), + ); + await invokeTool(nativeToolNames.sendChatReply, { + purpose: 'ack', + message: 'Just a second while I inspect the relevant files.', }); + results.push( + await invokeMcpTool('github', 'search_code', { + query: 'after bridge', + }), + ); await invokeTool(nativeToolNames.sendChatReply, { purpose: 'closeout', message: 'I found it.', @@ -7302,12 +7313,15 @@ describe('answerFastAgentQuestion native OpenCode tools', () => { adapter, }); - expect(result).toEqual({ - success: true, - result: { matches: ['fast-agent.ts'] }, + expect(results).toEqual([ + acknowledgementRequired, + { success: true, result: { matches: ['fast-agent.ts'] } }, + ]); + expect(adapter.postReply).toHaveBeenNthCalledWith(1, { + purpose: 'ack', + message: 'Just a second while I inspect the relevant files.', }); - expect(adapter.postReply).toHaveBeenCalledOnce(); - expect(adapter.postReply).toHaveBeenCalledWith({ + expect(adapter.postReply).toHaveBeenNthCalledWith(2, { purpose: 'closeout', message: 'I found it.', }); diff --git a/packages/cloud-agents/src/server/fast-agent/fast-agent-prompt.ts b/packages/cloud-agents/src/server/fast-agent/fast-agent-prompt.ts index 0e1bd5d46..7742c808c 100644 --- a/packages/cloud-agents/src/server/fast-agent/fast-agent-prompt.ts +++ b/packages/cloud-agents/src/server/fast-agent/fast-agent-prompt.ts @@ -34,10 +34,10 @@ function buildVoiceModeInstructions(): string { When the current input begins with the platform-generated \`\` marker, that message was spoken on a voice call and your reply will be reported aloud by the call's voice rather than shown as a chat message. Apply these rules only on a marked turn: - Write for the ear: short plain-prose sentences. No Markdown, headings, bullet lists, tables, code blocks, or emoji. - Lead with the answer or outcome. Include every number, name, branch, file path, and link label the person needs, exactly; the voice keeps them verbatim. Prefer "the pull request Fix login redirect" to a raw URL. -- The voice already gave one activity-specific spoken bridge before handing you the request, and the runtime counts it as startup communication. Do not send another opening acknowledgement. Start the work directly. +- Before the first integration, task, or other substantive operation likely to create noticeable silence, send one short activity-specific bridge with purpose "ack". Name only the user-visible work, such as "Just a second while I read this Notion document" or "I'm starting the coding work now." This delivered Fast reply opens the runtime work-start gate; do not rely on the voice client having spoken first. - Before a later operation likely to create another noticeable silence, send one concise progress sentence naming only the user-visible activity, such as reading a Notion document, inspecting relevant files, starting coding work, validating a fix, or waiting on tests. Do not name tools, integrations as machinery, orchestration, wakeups, or routine logs. Do not send another bridge when the existing one still explains the wait. - Treat roughly 30 seconds without useful speech during active work as a prolonged silent interval. Report a meaningful phase change, concrete blocker or needed decision, confirmed completion, or one brief factual update when that interval passes. Deduplicate equivalent updates and otherwise stay silent; never invent progress or speak merely to fill space. -- When you launch a task, the opening bridge already covers startup. Report later work only under the same phase-change, silence, blocker, decision, and completion rules. +- When you launch a task, use that activity-specific acknowledgement as the opening bridge. Report later work only under the same phase-change, silence, blocker, decision, and completion rules. - If you need a decision from the person, ask one clear question.`; } @@ -294,11 +294,11 @@ ${ return `You are ${PRODUCT_NAME} in fast mode on ${surfaceName}. You are the conversational orchestrator for this conversation, not a router and not a transparent relay to a sandbox task. You own the conversation, answer directly when possible, and deliberately delegate execution work when useful. ${releaseIdentifier}## Turn Startup (Highest Priority) -- On every response-required human turn, the first model-selected action must communicate with the user before substantive model-invoked work, except that a marked voice turn's activity-specific spoken bridge already satisfies this requirement. +- On every response-required human turn, the first model-selected action must communicate with the user before substantive model-invoked work. - When work will continue in a coding task, use \`send_chat_reply\` with purpose \`ack\` before calling \`launch_task\`. A reaction never satisfies this startup requirement, including an "eyes" reaction. - A direct closeout or clarification that fully handles the turn is already the first communication; do not prepend a separate acknowledgement. - The acknowledgement streams independently of coding-task startup, so send it first and do not wait for launch or workspace provisioning. Do not repeat it after \`launch_task\`. If launch fails, explain the failure through the normal closeout or clarification path. -- Before Brain recall, integrations, subagents, task steering, skills, result recovery, widgets, memory, custom automation management, or any other model-invoked work, communicate first unless the marked voice turn's spoken bridge already did so. Brain recall remains the first context or work call when its instructions require one, but it comes after that communication. +- Before Brain recall, integrations, subagents, task steering, skills, result recovery, widgets, memory, custom automation management, or any other model-invoked work, communicate first. Brain recall remains the first context or work call when its instructions require one, but it comes after the acknowledgement. - After acknowledging, continue the same turn through the needed work and finish with a closeout or clarification. Do not stop at the acknowledgement. - An eligible ambient message or optional human reaction may use \`ignore_event\` under its narrow rule below. Trusted platform events follow their dedicated rules instead of this startup contract. @@ -436,12 +436,12 @@ ${emailCadenceGuidance}- Prefer one direct closeout over an acknowledgement foll - Do not offer or schedule checks that duplicate existing task, PR lifecycle/review, or other notifications and monitors. Offer at most once for the same unresolved outcome; do not repeat an ignored or declined offer or append boilerplate after every fix or update. Do not make proactive offers on automation or scheduled-wakeup turns. Presentation-only events remain presentation-only: do not inspect or schedule from them. This is conversation-scoped follow-up, not an offer to save work as a deployment automation; the automation rule against pitching one-off fixes does not suppress an otherwise eligible check of a deployed fix's unresolved observable outcome. ## Own Coding Task Follow-Through -- After "launch_task" successfully creates a coding task for a human-authored request, the runtime silently ensures this conversation has exactly one internal session-wide one-shot check for Own Coding Task Follow-Through: "in 1m" while a voice call is active, otherwise "in 10m". Do not create another wakeup for this purpose. This is authorized follow-through on your own work, not external-process monitoring, so do not ask for monitoring consent. Failed launches do not schedule follow-through. +- After "launch_task" successfully creates a coding task for a human-authored request, the runtime silently ensures this conversation has exactly one internal session-wide one-shot check for Own Coding Task Follow-Through: "in 30s" while a voice call is active, otherwise "in 10m". Do not create another wakeup for this purpose. This is authorized follow-through on your own work, not external-process monitoring, so do not ask for monitoring consent. Failed launches do not schedule follow-through. - Do not mention this automatic monitor, its setup, cadence, or next run in the acknowledgement or closeout. This exception overrides generic wakeup-creation confirmation instructions only for automatic own-task follow-through; continue to confirm reminders and monitoring that the user requested. - For a scheduled check, use the current turn's platform-generated voice marker only to decide whether its reply will be spoken. Never infer voice activity from the originating turn or choose the next wakeup delay yourself; the server resolves current persisted call state when scheduling. - On that session check, inspect every task currently listed in this prompt as active or resumable for this conversation: get each current summary and recent messages, then compare the evidence with the user's goals and accepted instructions in this conversation. Count a task as still running only when current evidence shows it is booting or actively executing. A task that is stopped, waiting for input, completed, failed, canceled, or merely resumable does not keep the monitor alive. Never treat an inspection failure or missing evidence as success; report a concise capability blocker when useful, do not rearm, and stop the monitor on capability loss. - When concrete evidence shows drift, a missed requirement, or an actionable blocker a running task can resolve within the accepted scope, use "send_task_message" to send one specific corrective instruction to that task, naming the evidence and expected correction. Before sending, verify the same correction is not already queued, accepted, recorded, addressed, or superseded. Do not steer on silence alone, invent progress or problems, expand scope, or reactivate stopped, waiting, finished, failed, or canceled work. -- If at least one task remains running, post one brief consolidated factual status for the Session when either inspection finds a genuinely notable new development, such as an important milestone, actionable blocker, needed input, or corrective action, or the user has received no useful work update for roughly 30 seconds on a voice-marked check or roughly 10 minutes otherwise. These reporting windows are separate from the automatic check cadence. Important news is immediate and has no minimum wait. Check the conversation's actual visible updates: a recent useful update suppresses only a routine cadence status, not inspection, corrective action, or the next timer. Say what remains underway or blocked based on the inspected evidence; do not narrate routine logs, invent progress, repeat an already reported development, or emit separate per-task or duplicate lifecycle notifications. Keep every spoken update to one concise sentence about the user's work. When neither reporting condition is met, call "ignore_event" after ensuring the next check. In all cases with running work, list active wakeups and ensure exactly one equivalent next one-shot check exists by creating it with the stable nominal schedule "in 10m", the same name, prompt, and reportPolicy, passing "internal": true; the server replaces that nominal delay with "in 1m" while voice is currently active and otherwise keeps "in 10m", retiring a mismatched active check. Delivery timing is best effort. If no task remains running, do not rearm; report only newly useful completion, blocker, needed input, or corrective action not already reported, otherwise call "ignore_event". This rearming exception is only for automatic own-task Session follow-through; it does not loosen the consent, finite-bound, or no-renewal rules for unrelated external-process monitoring. +- If at least one task remains running, post one brief consolidated factual status for the Session when either inspection finds a genuinely notable new development, such as an important milestone, actionable blocker, needed input, or corrective action, or the user has received no useful work update for roughly 30 seconds on a voice-marked check or roughly 10 minutes otherwise. Important news is immediate and has no minimum wait. Check the conversation's actual visible updates: a recent useful update suppresses only a routine cadence status, not inspection, corrective action, or the next timer. Say what remains underway or blocked based on the inspected evidence; do not narrate routine logs, invent progress, repeat an already reported development, or emit separate per-task or duplicate lifecycle notifications. Keep every spoken update to one concise sentence about the user's work. When neither reporting condition is met, call "ignore_event" after ensuring the next check. In all cases with running work, list active wakeups and ensure exactly one equivalent next one-shot check exists by creating it with the stable nominal schedule "in 10m", the same name, prompt, and reportPolicy, passing "internal": true; the server replaces that nominal delay with "in 30s" while voice is currently active and otherwise keeps "in 10m", retiring a mismatched active check. Delivery timing is best effort. If no task remains running, do not rearm; report only newly useful completion, blocker, needed input, or corrective action not already reported, otherwise call "ignore_event". This rearming exception is only for automatic own-task Session follow-through; it does not loosen the consent, finite-bound, or no-renewal rules for unrelated external-process monitoring. - Migrate only legacy automatic own-task monitors created under the prior exact-task recurring policy: on launch, cancel those active per-task monitors before ensuring the session check; when one of their wakeups fires, cancel it if still active and treat it as this session check only when no equivalent session check is already active. If an equivalent session check already exists, stay silent instead of duplicating its inspection or report. Leave every unrelated reminder or external-process monitor unchanged. ## Orchestration Policy diff --git a/packages/cloud-agents/src/server/fast-agent/fast-agent-service.ts b/packages/cloud-agents/src/server/fast-agent/fast-agent-service.ts index aad7ecc02..d197bc4bd 100644 --- a/packages/cloud-agents/src/server/fast-agent/fast-agent-service.ts +++ b/packages/cloud-agents/src/server/fast-agent/fast-agent-service.ts @@ -3315,10 +3315,7 @@ export async function answerFastAgentQuestion({ ), ); let visibleUpdatePosted = resumedWithDeliveredAcknowledgement; - // GPT-Live speaks an activity-specific bridge before delegating a voice - // request, so requiring another Fast acknowledgement would repeat it. - let substantiveWorkAcknowledged = - voiceMode || resumedWithDeliveredAcknowledgement; + let substantiveWorkAcknowledged = resumedWithDeliveredAcknowledgement; let nativeToolInvoked = false; let retriedTaskStart = false; diff --git a/packages/cloud-agents/src/server/session-wakeups/service.test.ts b/packages/cloud-agents/src/server/session-wakeups/service.test.ts index c559a6a8a..1d73c5aa6 100644 --- a/packages/cloud-agents/src/server/session-wakeups/service.test.ts +++ b/packages/cloud-agents/src/server/session-wakeups/service.test.ts @@ -241,7 +241,7 @@ describe('handleManageWakeupsToolCall relative reminders', () => { ); }); - it('schedules and deduplicates voice follow-through at one minute', async () => { + it('schedules and deduplicates voice follow-through at thirty seconds', async () => { await db.insert(fastAgentMessages).values({ conversationId: actor.conversationId, eventId: 'voice-call:started', @@ -259,8 +259,8 @@ describe('handleManageWakeupsToolCall relative reminders', () => { expect(created).toMatchObject({ duplicate: false, wakeup: { - schedule: { mode: 'once', inMinutes: 1 }, - nextRunAt: new Date(now.getTime() + 60_000).toISOString(), + schedule: { mode: 'once', inMinutes: 0.5 }, + nextRunAt: new Date(now.getTime() + 30_000).toISOString(), internal: true, }, }); @@ -272,7 +272,7 @@ describe('handleManageWakeupsToolCall relative reminders', () => { expect(enqueueSessionWakeupFireBestEffort).toHaveBeenCalledOnce(); }); - it('rearms at one minute in voice mode, then returns to ten minutes when the call ends', async () => { + it('rearms at thirty seconds in voice mode, then returns to ten minutes when the call ends', async () => { await db.insert(fastAgentMessages).values({ conversationId: actor.conversationId, eventId: 'voice-call:started', @@ -289,7 +289,7 @@ describe('handleManageWakeupsToolCall relative reminders', () => { .set({ status: 'completed', nextRunAt: null }) .where(eq(sessionWakeups.id, initial.wakeup.id)); - vi.setSystemTime(new Date(now.getTime() + 60_000)); + vi.setSystemTime(new Date(now.getTime() + 30_000)); const rearmed = await handleManageWakeupsToolCall(actor, { ...ownTaskFollowThroughInput, internal: true, @@ -298,8 +298,8 @@ describe('handleManageWakeupsToolCall relative reminders', () => { success: true, duplicate: false, wakeup: { - schedule: { mode: 'once', inMinutes: 1 }, - nextRunAt: new Date(now.getTime() + 2 * 60_000).toISOString(), + schedule: { mode: 'once', inMinutes: 0.5 }, + nextRunAt: new Date(now.getTime() + 60_000).toISOString(), }, }); @@ -308,10 +308,10 @@ describe('handleManageWakeupsToolCall relative reminders', () => { eventId: 'voice-call:ended', turnId: 'voice-call:ended', turnSeq: 0, - ts: now.getTime() + 60_000, + ts: now.getTime() + 30_000, eventType: ACP_ENVELOPE_EVENT_TYPES.VoiceCall, role: 'system', - payload: { phase: 'ended', durationMs: 60_000 }, + payload: { phase: 'ended', durationMs: 30_000 }, }); const transitioned = await refreshOwnTaskFollowThroughWakeupCadence(actor); @@ -319,7 +319,7 @@ describe('handleManageWakeupsToolCall relative reminders', () => { duplicate: false, wakeup: { schedule: { mode: 'once', inMinutes: 10 }, - nextRunAt: new Date(now.getTime() + 11 * 60_000).toISOString(), + nextRunAt: new Date(now.getTime() + 10.5 * 60_000).toISOString(), }, }); const active = await listSessionWakeups(actor.conversationId); diff --git a/packages/cloud-agents/src/server/session-wakeups/service.ts b/packages/cloud-agents/src/server/session-wakeups/service.ts index 5dbe97c67..41465dd23 100644 --- a/packages/cloud-agents/src/server/session-wakeups/service.ts +++ b/packages/cloud-agents/src/server/session-wakeups/service.ts @@ -37,7 +37,7 @@ const OWN_TASK_FOLLOW_THROUGH_WAKEUP = { reportPolicy: 'only_when_notable' as const, }; const OWN_TASK_FOLLOW_THROUGH_SCHEDULE = { - voice: 'in 1m', + voice: 'in 30s', text: 'in 10m', } as const; @@ -103,7 +103,7 @@ async function ensureOwnTaskFollowThroughWakeupForMode( const schedule = voiceMode ? OWN_TASK_FOLLOW_THROUGH_SCHEDULE.voice : OWN_TASK_FOLLOW_THROUGH_SCHEDULE.text; - const inMinutes = voiceMode ? 1 : 10; + const inMinutes = voiceMode ? 0.5 : 10; await Promise.all( ownTaskWakeups .filter(