From 62b35d349b2d2c59c2a7a1b9145cd76ec77b76a6 Mon Sep 17 00:00:00 2001 From: Ada-1017 Date: Wed, 14 Oct 2026 00:56:58 -0400 Subject: [PATCH 1/5] fix(trace): a turn dropped after its compile ends with inference:aborted beginAgentTurn emits inference:started before the compile. When the compile returns to an agent that was replaced or removed, or a framework that is stopping or was stopped and started again, the turn is dropped before it sends anything, and returned with no terminal event, against onTrace's contract that the terminal events fire once per turn. A module bracketing a turn between inference:started and a terminal event (the Discord module's typing loop, connectome-host's activity module) kept the dropped turn open. The drop now emits inference:aborted, with the reason the console line gives and the turn's duration from its start. The test of a compile from before a stop and restart records the agent's inference:* events and checks the dropped turn gets exactly one terminal event. --- changelog.d/turn-drop-terminal-event.fixed.md | 1 + src/framework.ts | 20 ++++++++++++++----- test/tool-result-guard.test.ts | 12 +++++++++++ 3 files changed, 28 insertions(+), 5 deletions(-) create mode 100644 changelog.d/turn-drop-terminal-event.fixed.md diff --git a/changelog.d/turn-drop-terminal-event.fixed.md b/changelog.d/turn-drop-terminal-event.fixed.md new file mode 100644 index 000000000..b5faafffd --- /dev/null +++ b/changelog.d/turn-drop-terminal-event.fixed.md @@ -0,0 +1 @@ +- A turn dropped after its compile, because its agent was replaced or removed, or the framework was stopping or had been stopped and started again, now ends with `inference:aborted`, whose `reason` says which. The turn had emitted `inference:started` and then no terminal event, so a module that brackets a turn between them, such as a typing indicator, kept that turn open. diff --git a/src/framework.ts b/src/framework.ts index 9877d24c3..fe9a0f1fd 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -11341,6 +11341,7 @@ export class AgentFramework { const startLogical = this.turnRouting.get(agent.name)?.logical; this.touchEphemeralRun(agent.name, true); + const turnStartedAt = Date.now(); this.emitTrace({ type: 'inference:started', agentName: agent.name, @@ -11354,7 +11355,7 @@ export class AgentFramework { if (!continuingTurn) { this.eventGate?.onInferenceStarted(agent.name); } - this.lastInferenceAt.set(agent.name, { ...this.lastInferenceAt.get(agent.name), startedAt: Date.now() }); + this.lastInferenceAt.set(agent.name, { ...this.lastInferenceAt.get(agent.name), startedAt: turnStartedAt }); // Typing indicator, started at TURN START. It says "attending", which is // true from the moment the turn is accepted — but it used to start inside @@ -11459,10 +11460,10 @@ export class AgentFramework { // Membrane's yielding stream starts its request when it is first // iterated, so cancelling it here, before driveStream, sends nothing. if (this.agents.get(agent.name) !== agent || this.providerAdmissionClosed || lifecycle !== this.lifecycleGeneration) { - console.error(`[turn] ${agent.name}: a turn compiled while ${ - this.agents.get(agent.name) !== agent ? 'the agent was replaced or removed' - : this.providerAdmissionClosed ? 'the framework was stopping' - : 'the framework was stopped and started again'} is dropped before it sends anything`); + const why = this.agents.get(agent.name) !== agent ? 'the agent was replaced or removed' + : this.providerAdmissionClosed ? 'the framework was stopping' + : 'the framework was stopped and started again'; + console.error(`[turn] ${agent.name}: a turn compiled while ${why} is dropped before it sends anything`); stream.cancel(); agent.cancelStream(); if (this.agents.get(agent.name) === agent) { @@ -11474,6 +11475,15 @@ export class AgentFramework { } this.channelRegistry?.stopTyping(); this.eventGate?.onInferenceEnded(agent.name); + // The turn emitted inference:started before its compile, and every + // turn that does gets one terminal event, which listeners such as a + // typing loop rely on. Nothing was sent, so it ends as aborted. + this.emitTrace({ + type: 'inference:aborted', + agentName: agent.name, + reason: `a turn compiled while ${why} is dropped before it sends anything`, + durationMs: Date.now() - turnStartedAt, + }); return false; } // A conversation agent takes no provider gate, so its own compression diff --git a/test/tool-result-guard.test.ts b/test/tool-result-guard.test.ts index ba1a56d75..cf8a42585 100644 --- a/test/tool-result-guard.test.ts +++ b/test/tool-result-guard.test.ts @@ -3224,6 +3224,13 @@ test('a framework compile from before stop cannot drive after restart', async () return result; }; let starting: Promise | undefined; + // The dropped turn's inference:* events, as a module's trace listener sees them. + const turnEvents: Array<{ type: string; reason?: string }> = []; + const unsubscribe = framework.onTrace((event) => { + if (event.type.startsWith('inference:') && (event as { agentName?: string }).agentName === 'assistant') { + turnEvents.push({ type: event.type, ...('reason' in event ? { reason: String(event.reason) } : {}) }); + } + }); try { const lines: string[] = []; const error = console.error; @@ -3238,6 +3245,11 @@ test('a framework compile from before stop cannot drive after restart', async () } finally { console.error = error; } assert.equal(driven, 0); assert.ok(lines.some((line) => /stopped and started again/.test(line) && /dropped before it sends anything/.test(line)), lines.join('\n')); + // The turn emitted inference:started before its compile, so it gets one + // terminal event: a listener's bracket (a typing loop) closes. + assert.deepEqual(turnEvents.map((event) => event.type), ['inference:started', 'inference:aborted']); + assert.match(turnEvents[1]?.reason ?? '', /stopped and started again.*dropped before it sends anything/); + unsubscribe(); // A turn begun after the restart is admitted and drives its stream. await (framework as unknown as { startAgentStream(a: typeof agent): Promise }).startAgentStream(agent); await until(() => driven === 1, 'the turn begun after the restart drives'); From 112990d4cbd20df2304b0aa693bf5ac604dfbd5a Mon Sep 17 00:00:00 2001 From: Ada-1017 Date: Wed, 14 Oct 2026 01:14:42 -0400 Subject: [PATCH 2/5] fix(trace): a turn dropped before its compile ends with inference:aborted too The check ahead of the compile (an agent replaced or removed while the modules' gatherContext and the MCPL beforeInference hooks awaited, or a framework that began stopping) also returned after inference:started with no terminal event, and without a console line. It now logs and emits inference:aborted as the drop after the compile does, with its reason. Two tests hold a turn in gatherContext, then stop the framework or remove the agent, and check its one terminal event and its reason. The fragment now covers both drops. --- changelog.d/turn-drop-terminal-event.fixed.md | 2 +- src/framework.ts | 10 +++++ test/tool-result-guard.test.ts | 44 +++++++++++++++++++ 3 files changed, 55 insertions(+), 1 deletion(-) diff --git a/changelog.d/turn-drop-terminal-event.fixed.md b/changelog.d/turn-drop-terminal-event.fixed.md index b5faafffd..c6f998756 100644 --- a/changelog.d/turn-drop-terminal-event.fixed.md +++ b/changelog.d/turn-drop-terminal-event.fixed.md @@ -1 +1 @@ -- A turn dropped after its compile, because its agent was replaced or removed, or the framework was stopping or had been stopped and started again, now ends with `inference:aborted`, whose `reason` says which. The turn had emitted `inference:started` and then no terminal event, so a module that brackets a turn between them, such as a typing indicator, kept that turn open. +- A turn the framework drops before it sends anything, because its agent was replaced or removed, or the framework was stopping or had been stopped and started again while the turn was assembled or compiled, now ends with `inference:aborted`, whose `reason` says which. The turn had emitted `inference:started` and then no terminal event, so a module that brackets a turn between them, such as a typing indicator, kept that turn open. diff --git a/src/framework.ts b/src/framework.ts index fe9a0f1fd..28d363a64 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -11430,8 +11430,18 @@ export class AgentFramework { // and stop() may close provider admission. Do not create a provider // stream after its generation lost ownership, or once admission closed. if (this.agents.get(agent.name) !== agent || this.providerAdmissionClosed) { + const why = this.agents.get(agent.name) !== agent ? 'the agent was replaced or removed' : 'the framework was stopping'; + console.error(`[turn] ${agent.name}: a turn assembled while ${why} is dropped before it sends anything`); this.channelRegistry?.stopTyping(); this.eventGate?.onInferenceEnded(agent.name); + // As at the drop after the compile below: the turn emitted + // inference:started, so it gets its one terminal event. + this.emitTrace({ + type: 'inference:aborted', + agentName: agent.name, + reason: `a turn assembled while ${why} is dropped before it sends anything`, + durationMs: Date.now() - turnStartedAt, + }); return false; } diff --git a/test/tool-result-guard.test.ts b/test/tool-result-guard.test.ts index cf8a42585..cd47f0bbc 100644 --- a/test/tool-result-guard.test.ts +++ b/test/tool-result-guard.test.ts @@ -3256,6 +3256,50 @@ test('a framework compile from before stop cannot drive after restart', async () } finally { compile.open(); await starting; await framework.stop(); store.close(); } }); +for (const how of ['stopping', 'removed'] as const) { + test(`a turn dropped before its compile (the framework ${how === 'stopping' ? 'stopping' : 'losing its agent'}) ends with inference:aborted`, async () => { + const dir = mkdtempSync(join(tmpdir(), 'guard-drop-before-compile-')); dirs.push(dir); + const store = JsStore.openOrCreate({ path: join(dir, 'store') }); + const membrane = new ScriptMembrane([[answer()]]); + const framework = await AgentFramework.create({ store, membrane: membrane.asMembrane(), + agents: [{ name: 'assistant', model: 'test', systemPrompt: 'system' }], + modules: [], syncIntervalMs: 0, maintenanceIntervalMs: 0 }); + const agent = framework.getAgent('assistant')!; + // The turn waits in a module's gatherContext, before the check that comes ahead of the compile. + const held = gate(); + const entered = gate(); + const internal = framework as unknown as { moduleRegistry: { gatherContext: (n: string) => Promise }; agents: Map }; + internal.moduleRegistry.gatherContext = async () => { entered.open(); await held.opened; return []; }; + const turnEvents: Array<{ type: string; reason?: string }> = []; + framework.onTrace((event) => { + if (event.type.startsWith('inference:') && (event as { agentName?: string }).agentName === 'assistant') { + turnEvents.push({ type: event.type, ...('reason' in event ? { reason: String(event.reason) } : {}) }); + } + }); + const lines: string[] = []; + const error = console.error; + console.error = (...args: unknown[]) => { lines.push(args.map(String).join(' ')); }; + let starting: Promise | undefined; + try { + starting = (framework as unknown as { startAgentStream(a: typeof agent): Promise }).startAgentStream(agent); + await entered.opened; + if (how === 'stopping') await framework.stop(); + else internal.agents.delete('assistant'); + held.open(); + await starting; + } finally { console.error = error; held.open(); await starting; } + try { + const why = how === 'stopping' ? 'the framework was stopping' : 'the agent was replaced or removed'; + assert.deepEqual(turnEvents.map((event) => event.type), ['inference:started', 'inference:aborted']); + assert.equal(turnEvents[1]?.reason, `a turn assembled while ${why} is dropped before it sends anything`); + assert.ok(lines.some((line) => line.includes(`a turn assembled while ${why} is dropped before it sends anything`)), lines.join('\n')); + } finally { + if (how === 'removed') internal.agents.set('assistant', agent); + await framework.stop(); store.close(); + } + }); +} + // Drain async storage/annotation continuations without advancing deadline time. async function flushAsync() { for (let i = 0; i < 8; i++) await new Promise((resolve) => setImmediate(resolve)); From d695fb82227eba06bba119371dd9ad187db4cb37 Mon Sep 17 00:00:00 2001 From: Ada-1017 Date: Wed, 14 Oct 2026 08:20:29 -0400 Subject: [PATCH 3/5] fix(trace): a dropped turn's terminal event leaves a newer turn of its name alone The review of the drop-terminal change found that a turn dropped after a stop and start, or after its agent object was replaced, can reach its drop while a newer turn of the same name is running: the new lifecycle's, or the replacement's. inference:aborted and lastInference carry only the agent's name, so the dropped turn's event would end that newer turn's bracket: the Discord module's typing loop, keyed by name, would stop while it runs. Both drops now end their turn through abortDroppedTurn, which does nothing when another turn holds the name's turn token, and otherwise emits inference:aborted and records lastInference.endedAt, as driveStream records it, so the health snapshot no longer shows a dropped turn as running. The reason is the logged line's text without its prefix, built once. - inference:aborted's reason and durationMs get doc comments: reason now has three sources (an operator's abort, 'quiesce_abandoned', a dropped turn's sentence), and durationMs runs from this start of the turn. - The web UI in ui/ returns an agent to idle on the four terminal events onTrace's contract names, not only inference:completed and failed. - The drop tests cover both checks (before the compile and after it) and four ways each: the framework stopping, the agent removed, the agent object replaced, and a newer turn holding the name's token. The first three check the exact reason, durationMs against the time held, and lastInference.endedAt; the last checks that nothing is emitted or recorded. - The changelog fragment says what is recorded, the newer-turn rule, and the UI. --- changelog.d/turn-drop-terminal-event.fixed.md | 2 +- src/framework.ts | 43 ++++--- src/types/trace.ts | 8 ++ test/tool-result-guard.test.ts | 119 ++++++++++++------ ui/src/App.vue | 10 +- 5 files changed, 121 insertions(+), 61 deletions(-) diff --git a/changelog.d/turn-drop-terminal-event.fixed.md b/changelog.d/turn-drop-terminal-event.fixed.md index c6f998756..d40a6c904 100644 --- a/changelog.d/turn-drop-terminal-event.fixed.md +++ b/changelog.d/turn-drop-terminal-event.fixed.md @@ -1 +1 @@ -- A turn the framework drops before it sends anything, because its agent was replaced or removed, or the framework was stopping or had been stopped and started again while the turn was assembled or compiled, now ends with `inference:aborted`, whose `reason` says which. The turn had emitted `inference:started` and then no terminal event, so a module that brackets a turn between them, such as a typing indicator, kept that turn open. +- A turn the framework drops before it sends anything, because its agent was replaced or removed, or the framework was stopping or had been stopped and started again while the turn was assembled or compiled, now ends with `inference:aborted`, whose `reason` says which, and records `lastInference.endedAt`. The turn had emitted `inference:started` and then no terminal event, so a module that brackets a turn between them, such as a typing indicator, kept that turn open, and the health snapshot showed it still running. When a newer turn of the agent's name already holds its turn token, the dropped turn emits and records nothing, since both carry only the name and would end that newer turn. The web UI in `ui/` returns an agent to idle on each of the four terminal events (`inference:turn_ended`, `inference:aborted`, `inference:failed`, `inference:exhausted`). diff --git a/src/framework.ts b/src/framework.ts index 28d363a64..bbaf0a23d 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -11132,6 +11132,24 @@ export class AgentFramework { } } + /** + * End a turn that beginAgentTurn drops before it sends anything. The turn + * emitted inference:started, and listeners that bracket a turn, such as a + * typing loop or an activity indicator, end it on a terminal event, so it + * gets inference:aborted with `reason`, and lastInference records its end + * as driveStream's would. Nothing of the kind when another turn of this + * agent's name holds the turn token (one begun after a stop and start, or + * by a replacement agent object): the event and the record carry only the + * name, so this frame's would end that newer turn instead. + */ + private abortDroppedTurn(agentName: string, turnToken: number, reason: string, startedAt: number): void { + const owner = this.activeTurnTokens.get(agentName); + if (owner !== undefined && owner !== turnToken) return; + const now = Date.now(); + this.lastInferenceAt.set(agentName, { ...this.lastInferenceAt.get(agentName), endedAt: now }); + this.emitTrace({ type: 'inference:aborted', agentName, reason, durationMs: now - startedAt }); + } + /** * Body of startAgentStream, split out so the caller's try/finally owns the * turn token unconditionally. Returns true iff the token was handed off to @@ -11431,17 +11449,11 @@ export class AgentFramework { // stream after its generation lost ownership, or once admission closed. if (this.agents.get(agent.name) !== agent || this.providerAdmissionClosed) { const why = this.agents.get(agent.name) !== agent ? 'the agent was replaced or removed' : 'the framework was stopping'; - console.error(`[turn] ${agent.name}: a turn assembled while ${why} is dropped before it sends anything`); + const reason = `a turn assembled while ${why} is dropped before it sends anything`; + console.error(`[turn] ${agent.name}: ${reason}`); this.channelRegistry?.stopTyping(); this.eventGate?.onInferenceEnded(agent.name); - // As at the drop after the compile below: the turn emitted - // inference:started, so it gets its one terminal event. - this.emitTrace({ - type: 'inference:aborted', - agentName: agent.name, - reason: `a turn assembled while ${why} is dropped before it sends anything`, - durationMs: Date.now() - turnStartedAt, - }); + this.abortDroppedTurn(agent.name, turnToken, reason, turnStartedAt); return false; } @@ -11473,7 +11485,8 @@ export class AgentFramework { const why = this.agents.get(agent.name) !== agent ? 'the agent was replaced or removed' : this.providerAdmissionClosed ? 'the framework was stopping' : 'the framework was stopped and started again'; - console.error(`[turn] ${agent.name}: a turn compiled while ${why} is dropped before it sends anything`); + const reason = `a turn compiled while ${why} is dropped before it sends anything`; + console.error(`[turn] ${agent.name}: ${reason}`); stream.cancel(); agent.cancelStream(); if (this.agents.get(agent.name) === agent) { @@ -11485,15 +11498,7 @@ export class AgentFramework { } this.channelRegistry?.stopTyping(); this.eventGate?.onInferenceEnded(agent.name); - // The turn emitted inference:started before its compile, and every - // turn that does gets one terminal event, which listeners such as a - // typing loop rely on. Nothing was sent, so it ends as aborted. - this.emitTrace({ - type: 'inference:aborted', - agentName: agent.name, - reason: `a turn compiled while ${why} is dropped before it sends anything`, - durationMs: Date.now() - turnStartedAt, - }); + this.abortDroppedTurn(agent.name, turnToken, reason, turnStartedAt); return false; } // A conversation agent takes no provider gate, so its own compression diff --git a/src/types/trace.ts b/src/types/trace.ts index c96a9a579..4e94a25c3 100644 --- a/src/types/trace.ts +++ b/src/types/trace.ts @@ -60,7 +60,15 @@ export type TraceEvent = | (TraceEventBase & { type: 'inference:aborted'; agentName: string; + /** How long this start of the turn ran; after a context-budget restart, from the restart. */ durationMs: number; + /** + * Why the turn ended unfinished: the reason an operator gave + * `abortInference`, `'quiesce_abandoned'` for a turn an operator quiesce + * abandoned, or, for a turn dropped before it sent anything (its agent + * replaced or removed, or the framework stopping or restarted), a + * sentence saying which. + */ reason?: string; }) | (TraceEventBase & { diff --git a/test/tool-result-guard.test.ts b/test/tool-result-guard.test.ts index cd47f0bbc..9555fa11b 100644 --- a/test/tool-result-guard.test.ts +++ b/test/tool-result-guard.test.ts @@ -3256,48 +3256,87 @@ test('a framework compile from before stop cannot drive after restart', async () } finally { compile.open(); await starting; await framework.stop(); store.close(); } }); -for (const how of ['stopping', 'removed'] as const) { - test(`a turn dropped before its compile (the framework ${how === 'stopping' ? 'stopping' : 'losing its agent'}) ends with inference:aborted`, async () => { - const dir = mkdtempSync(join(tmpdir(), 'guard-drop-before-compile-')); dirs.push(dir); - const store = JsStore.openOrCreate({ path: join(dir, 'store') }); - const membrane = new ScriptMembrane([[answer()]]); - const framework = await AgentFramework.create({ store, membrane: membrane.asMembrane(), - agents: [{ name: 'assistant', model: 'test', systemPrompt: 'system' }], - modules: [], syncIntervalMs: 0, maintenanceIntervalMs: 0 }); - const agent = framework.getAgent('assistant')!; - // The turn waits in a module's gatherContext, before the check that comes ahead of the compile. - const held = gate(); - const entered = gate(); - const internal = framework as unknown as { moduleRegistry: { gatherContext: (n: string) => Promise }; agents: Map }; - internal.moduleRegistry.gatherContext = async () => { entered.open(); await held.opened; return []; }; - const turnEvents: Array<{ type: string; reason?: string }> = []; - framework.onTrace((event) => { - if (event.type.startsWith('inference:') && (event as { agentName?: string }).agentName === 'assistant') { - turnEvents.push({ type: event.type, ...('reason' in event ? { reason: String(event.reason) } : {}) }); +// A turn dropped before it sends anything, at either check: before its +// compile (it waits in a module's gatherContext) or after it (it waits in its +// compile). Each way it can lose its place: the framework stops, the agent is +// removed, the agent object is replaced under its name, or the replacement's +// own turn already holds the name's turn token. +for (const phase of ['before', 'after'] as const) { + for (const how of ['stopping', 'removed', 'replaced', 'superseded'] as const) { + const title = { stopping: 'the framework stopping', removed: 'its agent removed', replaced: 'its agent replaced', + superseded: "a newer turn of its agent's name running" }[how]; + test(`a turn dropped ${phase} its compile, with ${title}, ${how === 'superseded' ? "leaves the newer turn's bracket alone" : 'ends with inference:aborted'}`, async () => { + const dir = mkdtempSync(join(tmpdir(), `guard-drop-${phase}-compile-`)); dirs.push(dir); + const store = JsStore.openOrCreate({ path: join(dir, 'store') }); + const membrane = new ScriptMembrane([[answer()]]); + const framework = await AgentFramework.create({ store, membrane: membrane.asMembrane(), + agents: [{ name: 'assistant', model: 'test', systemPrompt: 'system' }], + modules: [], syncIntervalMs: 0, maintenanceIntervalMs: 0 }); + const agent = framework.getAgent('assistant')!; + const internal = framework as unknown as { + moduleRegistry: { gatherContext: (n: string) => Promise }; + agents: Map; + activeTurnTokens: Map; + lastInferenceAt: Map; + }; + const held = gate(); + const entered = gate(); + if (phase === 'before') { + internal.moduleRegistry.gatherContext = async () => { entered.open(); await held.opened; return []; }; + } else { + const build = agent.buildActivationRequest.bind(agent); + agent.buildActivationRequest = async (...args) => { entered.open(); await held.opened; return build(...args); }; + } + const turnEvents: Array<{ type: string; reason?: string; durationMs?: number }> = []; + framework.onTrace((event) => { + if (event.type.startsWith('inference:') && (event as { agentName?: string }).agentName === 'assistant') { + turnEvents.push({ type: event.type, + ...('reason' in event ? { reason: String(event.reason) } : {}), + ...('durationMs' in event ? { durationMs: Number(event.durationMs) } : {}) }); + } + }); + const lines: string[] = []; + const error = console.error; + console.error = (...args: unknown[]) => { lines.push(args.map(String).join(' ')); }; + const heldMs = 25; + let starting: Promise | undefined; + try { + starting = (framework as unknown as { startAgentStream(a: typeof agent): Promise }).startAgentStream(agent); + await entered.opened; + if (how === 'stopping') await framework.stop(); + else if (how === 'removed') internal.agents.delete('assistant'); + else { + internal.agents.set('assistant', Object.create(agent)); + if (how === 'superseded') internal.activeTurnTokens.set('assistant', -1); + } + await new Promise((resolve) => setTimeout(resolve, heldMs)); + held.open(); + await starting; + } finally { console.error = error; held.open(); await starting; } + try { + const why = how === 'stopping' ? 'the framework was stopping' : 'the agent was replaced or removed'; + const reason = `a turn ${phase === 'before' ? 'assembled' : 'compiled'} while ${why} is dropped before it sends anything`; + assert.ok(lines.some((line) => line.includes(reason)), lines.join('\n')); + const last = internal.lastInferenceAt.get('assistant'); + if (how === 'superseded') { + // The terminal event and lastInference carry only the name, so they + // would end the newer turn: the dropped frame emits and records nothing. + assert.deepEqual(turnEvents.map((event) => event.type), ['inference:started']); + assert.equal(last?.endedAt, undefined); + } else { + assert.deepEqual(turnEvents.map((event) => event.type), ['inference:started', 'inference:aborted']); + assert.equal(turnEvents[1]?.reason, reason); + const durationMs = turnEvents[1]?.durationMs ?? NaN; + assert.ok(Number.isFinite(durationMs) && durationMs >= heldMs - 5 && durationMs < 60_000, `durationMs ${durationMs}`); + assert.ok(last?.endedAt !== undefined && last.endedAt >= (last.startedAt ?? Infinity), JSON.stringify(last)); + } + } finally { + if (how === 'superseded') internal.activeTurnTokens.delete('assistant'); + if (how !== 'stopping') internal.agents.set('assistant', agent); + await framework.stop(); store.close(); } }); - const lines: string[] = []; - const error = console.error; - console.error = (...args: unknown[]) => { lines.push(args.map(String).join(' ')); }; - let starting: Promise | undefined; - try { - starting = (framework as unknown as { startAgentStream(a: typeof agent): Promise }).startAgentStream(agent); - await entered.opened; - if (how === 'stopping') await framework.stop(); - else internal.agents.delete('assistant'); - held.open(); - await starting; - } finally { console.error = error; held.open(); await starting; } - try { - const why = how === 'stopping' ? 'the framework was stopping' : 'the agent was replaced or removed'; - assert.deepEqual(turnEvents.map((event) => event.type), ['inference:started', 'inference:aborted']); - assert.equal(turnEvents[1]?.reason, `a turn assembled while ${why} is dropped before it sends anything`); - assert.ok(lines.some((line) => line.includes(`a turn assembled while ${why} is dropped before it sends anything`)), lines.join('\n')); - } finally { - if (how === 'removed') internal.agents.set('assistant', agent); - await framework.stop(); store.close(); - } - }); + } } // Drain async storage/annotation continuations without advancing deadline time. diff --git a/ui/src/App.vue b/ui/src/App.vue index 631987d3a..fcf861d93 100644 --- a/ui/src/App.vue +++ b/ui/src/App.vue @@ -85,7 +85,15 @@ function handleEvent(data: { event: string; data: unknown }) { if (data.event === 'inference:started') { const { agentName } = data.data as { agentName: string }; agentsStore.updateAgentStatus(agentName, 'inferring'); - } else if (data.event === 'inference:completed' || data.event === 'inference:failed') { + } else if ( + data.event === 'inference:completed' || + // The turn's terminal events (onTrace's contract): a turn dropped + // before it sends anything ends with inference:aborted alone. + data.event === 'inference:turn_ended' || + data.event === 'inference:aborted' || + data.event === 'inference:failed' || + data.event === 'inference:exhausted' + ) { const { agentName } = data.data as { agentName: string }; agentsStore.updateAgentStatus(agentName, 'idle'); } From 3f4cdb64ef6ad1de4778a6fa01faf90767a1d31d Mon Sep 17 00:00:00 2001 From: Ada-1017 Date: Wed, 14 Oct 2026 08:34:16 -0400 Subject: [PATCH 4/5] fix(discord): the typing keepalive stops when a turn completes normally A turn's normal end is inference:completed: driveStream emits it once, when the turn's stream completes, after agent.reset(). Tool-call cycles stay inside the yielding stream (provideToolResults, inference:stream_resumed), so it does not fire between tool rounds. inference:turn_ended fires only when a tool result ends the turn (shouldEndTurn). onTrace's documentation named four terminal events (turn_ended, aborted, failed, exhausted) and left the normal end out, and the Discord module's TURN_TERMINAL_TRACES followed it. With autoTyping on, its default, the keepalive therefore went on refreshing every TYPING_REFRESH_MS after an ordinary reply, until the agent's next inference:started or TYPING_MAX_MS. - TURN_TERMINAL_TRACES includes inference:completed. - onTrace's doc names inference:completed as the normal end, and says what a bracket meets besides: a context-budget restart emits inference:started again with no terminal event for the stream it replaced, and a failing turn can emit inference:failed and then inference:exhausted. - The web UI's comment in ui/src/App.vue groups inference:completed with the other terminal events (no behaviour change: it already idled on it). - test/discord-typing-keepalive.test.ts: for each of the five terminal events, a turn's keepalive shows typing at inference:started, refreshes once while the turn runs (mocked setInterval), and refreshes no more after the event. With inference:completed removed from the set, exactly the inference:completed case fails. - Fragments: a new fixed fragment for the Discord module and the doc; the drop-terminal fragment's UI sentence now says which events the UI added. --- ...iscord-typing-stops-at-completion.fixed.md | 1 + changelog.d/turn-drop-terminal-event.fixed.md | 2 +- src/modules/discord/index.ts | 6 +- src/types/module.ts | 12 ++- test/discord-typing-keepalive.test.ts | 85 +++++++++++++++++++ ui/src/App.vue | 4 +- 6 files changed, 101 insertions(+), 9 deletions(-) create mode 100644 changelog.d/discord-typing-stops-at-completion.fixed.md create mode 100644 test/discord-typing-keepalive.test.ts diff --git a/changelog.d/discord-typing-stops-at-completion.fixed.md b/changelog.d/discord-typing-stops-at-completion.fixed.md new file mode 100644 index 000000000..c277d52b4 --- /dev/null +++ b/changelog.d/discord-typing-stops-at-completion.fixed.md @@ -0,0 +1 @@ +- The Discord module's typing keepalive now stops when a turn completes normally, at `inference:completed`. It stopped only at `inference:turn_ended`, `inference:aborted`, `inference:failed` and `inference:exhausted`, so with `autoTyping` on, its default, the indicator went on refreshing after an ordinary reply until the agent's next `inference:started` or its five-minute cap. `onTrace`'s documentation now names `inference:completed` as a turn's normal end, beside the other four. diff --git a/changelog.d/turn-drop-terminal-event.fixed.md b/changelog.d/turn-drop-terminal-event.fixed.md index d40a6c904..700f73445 100644 --- a/changelog.d/turn-drop-terminal-event.fixed.md +++ b/changelog.d/turn-drop-terminal-event.fixed.md @@ -1 +1 @@ -- A turn the framework drops before it sends anything, because its agent was replaced or removed, or the framework was stopping or had been stopped and started again while the turn was assembled or compiled, now ends with `inference:aborted`, whose `reason` says which, and records `lastInference.endedAt`. The turn had emitted `inference:started` and then no terminal event, so a module that brackets a turn between them, such as a typing indicator, kept that turn open, and the health snapshot showed it still running. When a newer turn of the agent's name already holds its turn token, the dropped turn emits and records nothing, since both carry only the name and would end that newer turn. The web UI in `ui/` returns an agent to idle on each of the four terminal events (`inference:turn_ended`, `inference:aborted`, `inference:failed`, `inference:exhausted`). +- A turn the framework drops before it sends anything, because its agent was replaced or removed, or the framework was stopping or had been stopped and started again while the turn was assembled or compiled, now ends with `inference:aborted`, whose `reason` says which, and records `lastInference.endedAt`. The turn had emitted `inference:started` and then no terminal event, so a module that brackets a turn between them, such as a typing indicator, kept that turn open, and the health snapshot showed it still running. When a newer turn of the agent's name already holds its turn token, the dropped turn emits and records nothing, since both carry only the name and would end that newer turn. The web UI in `ui/` now also returns an agent to idle on `inference:turn_ended`, `inference:aborted` and `inference:exhausted`, where it had looked only at `inference:completed` and `inference:failed`. diff --git a/src/modules/discord/index.ts b/src/modules/discord/index.ts index 665057fd7..585f9b85e 100644 --- a/src/modules/discord/index.ts +++ b/src/modules/discord/index.ts @@ -67,9 +67,11 @@ const TYPING_REFRESH_MS = 8_000; // stuck on forever. const TYPING_MAX_MS = 5 * 60_000; -// Trace event types that terminate an agent's turn — any of these stops the -// typing keepalive. They fire once per turn, after all tool-call cycles. +// Trace event types that end an agent's turn (onTrace's contract) — any of +// these stops the typing keepalive. A turn's tool-call cycles stay inside its +// stream, so they come after them. inference:completed is the normal end. const TURN_TERMINAL_TRACES = new Set([ + 'inference:completed', 'inference:turn_ended', 'inference:aborted', 'inference:failed', diff --git a/src/types/module.ts b/src/types/module.ts index afb2638da..a6fda9ed2 100644 --- a/src/types/module.ts +++ b/src/types/module.ts @@ -321,10 +321,14 @@ export interface ModuleContext { * * Modules use this to react to agent lifecycle without polling — e.g. an * adapter that wants to show a "typing" indicator can bracket it between - * `inference:started` and the terminal events (`inference:turn_ended`, - * `inference:aborted`, `inference:failed`, `inference:exhausted`), all of - * which carry `agentName`. The terminal events fire once per turn, after any - * tool-call cycles, so a single start/stop bracket spans the whole turn. + * `inference:started` and the terminal events: `inference:completed` (the + * normal end), `inference:turn_ended` (a tool result ended the turn), + * `inference:aborted`, `inference:failed` and `inference:exhausted`, all of + * which carry `agentName`. A turn's tool-call cycles stay inside its + * stream, so one bracket spans them. A context-budget restart emits + * `inference:started` again with no terminal event for the stream it + * replaced, and a failing turn can emit `inference:failed` and then + * `inference:exhausted`, so a bracket should allow a repeated start or end. * * Returns an unsubscribe function. Call it in the module's `stop()` to avoid * leaking listeners across teardown/recreate (e.g. session switch). diff --git a/test/discord-typing-keepalive.test.ts b/test/discord-typing-keepalive.test.ts new file mode 100644 index 000000000..d5f383e72 --- /dev/null +++ b/test/discord-typing-keepalive.test.ts @@ -0,0 +1,85 @@ +import { test, mock } from 'node:test'; +import assert from 'node:assert/strict'; +import { DiscordModule } from '../src/modules/discord/index.js'; +import type { ConversationContext, DiscordClientInterface } from '../src/modules/discord/types.js'; +import type { ModuleContext, TraceEvent, TraceEventListener } from '../src/types/index.js'; + +// Discord's typing indicator expires after about 10 s, so the module refreshes +// it every 8 s (TYPING_REFRESH_MS) from a turn's inference:started until an +// event that ends the turn. +const REFRESH_MS = 8_000; +const agentName = 'assistant'; + +const turnEnds: TraceEvent[] = [ + // The normal end: the turn's stream completed. + { type: 'inference:completed', agentName, durationMs: 5, timestamp: 0 }, + { type: 'inference:turn_ended', agentName, timestamp: 0 }, + { type: 'inference:aborted', agentName, durationMs: 5, timestamp: 0 }, + { type: 'inference:failed', agentName, error: 'provider error', timestamp: 0 }, + { type: 'inference:exhausted', agentName, error: 'provider error', timestamp: 0 }, +]; + +async function startModule(): Promise<{ + module: DiscordModule; + emit: TraceEventListener; + typings: () => number; +}> { + let typings = 0; + const client = { + connect: async () => {}, + disconnect: async () => {}, + isConnected: () => true, + getBotUserId: () => undefined, + onMessage: () => {}, + onMessageEdit: () => {}, + onMessageDelete: () => {}, + sendTyping: async () => { + typings += 1; + }, + } as unknown as DiscordClientInterface; + let listener: TraceEventListener | undefined; + const ctx = { + isRestart: false, + getState: () => undefined, + setState: () => {}, + registerSpeechHandler: () => {}, + onTrace: (subscriber: TraceEventListener) => { + listener = subscriber; + return () => { + listener = undefined; + }; + }, + } as unknown as ModuleContext; + const module = new DiscordModule(client, { token: 'test-token' }); + await module.start(ctx); + assert.ok(listener, 'autoTyping, on by default, subscribes to the trace stream'); + // Where the agent's reply goes, as an incoming Discord message would set it. + (module as unknown as { replyContexts: Map }).replyContexts.set('default', { + channelId: 'channel-1', + userId: 'user-1', + } as ConversationContext); + return { module, emit: (event) => listener?.(event), typings: () => typings }; +} + +for (const end of turnEnds) { + test(`the typing keepalive a turn started stops at ${end.type}`, async () => { + mock.timers.enable({ apis: ['setInterval'] }); + try { + const { module, emit, typings } = await startModule(); + try { + emit({ type: 'inference:started', agentName, timestamp: 0 }); + assert.equal(typings(), 1, 'typing shows as the turn starts'); + mock.timers.tick(REFRESH_MS); + assert.equal(typings(), 2, 'and is refreshed while the turn runs'); + + emit(end); + mock.timers.tick(REFRESH_MS * 3); + assert.equal(typings(), 2, `no refresh after ${end.type}`); + } finally { + await module.stop(); + } + } finally { + mock.timers.reset(); + } + }); +} diff --git a/ui/src/App.vue b/ui/src/App.vue index fcf861d93..19981cdb2 100644 --- a/ui/src/App.vue +++ b/ui/src/App.vue @@ -86,9 +86,9 @@ function handleEvent(data: { event: string; data: unknown }) { const { agentName } = data.data as { agentName: string }; agentsStore.updateAgentStatus(agentName, 'inferring'); } else if ( - data.event === 'inference:completed' || - // The turn's terminal events (onTrace's contract): a turn dropped + // The events that end a turn (onTrace's contract). A turn dropped // before it sends anything ends with inference:aborted alone. + data.event === 'inference:completed' || data.event === 'inference:turn_ended' || data.event === 'inference:aborted' || data.event === 'inference:failed' || From 9d81e5d925eced85e299ba62fe140aa6592822d8 Mon Sep 17 00:00:00 2001 From: Ada-1017 Date: Wed, 14 Oct 2026 10:05:49 -0400 Subject: [PATCH 5/5] fix(turns): a dropped turn ends only its own lastInference record Greptile's P2 on the PR: abortDroppedTurn wrote {...existing, endedAt} whenever no other turn held the name's token. When an ephemeral run is disposed while its turn waits (runEphemeralToCompletion deletes the name's record and token), the drop then gave the disposed name an orphan { endedAt } record that nothing removes. The drop now ends the record only when it is still this turn's, the one beginAgentTurn stamped with this turn's startedAt; the terminal event still goes out, so a listener's bracket closes. The dropped-turn tests gain a 'disposed' case at both checks (agent, record and token gone, as the disposal does): inference:aborted, and no record for the name. With the guard taken out, both disposed cases fail and the other eight pass. --- src/framework.ts | 14 +++++++++----- test/tool-result-guard.test.ts | 20 ++++++++++++++++---- 2 files changed, 25 insertions(+), 9 deletions(-) diff --git a/src/framework.ts b/src/framework.ts index bbaf0a23d..0faaa2511 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -11137,16 +11137,20 @@ export class AgentFramework { * emitted inference:started, and listeners that bracket a turn, such as a * typing loop or an activity indicator, end it on a terminal event, so it * gets inference:aborted with `reason`, and lastInference records its end - * as driveStream's would. Nothing of the kind when another turn of this - * agent's name holds the turn token (one begun after a stop and start, or - * by a replacement agent object): the event and the record carry only the - * name, so this frame's would end that newer turn instead. + * as driveStream's would, when the record is still this turn's. Nothing of + * the kind when another turn of this agent's name holds the turn token (one + * begun after a stop and start, or by a replacement agent object): the + * event and the record carry only the name, so this frame's would end that + * newer turn instead. */ private abortDroppedTurn(agentName: string, turnToken: number, reason: string, startedAt: number): void { const owner = this.activeTurnTokens.get(agentName); if (owner !== undefined && owner !== turnToken) return; const now = Date.now(); - this.lastInferenceAt.set(agentName, { ...this.lastInferenceAt.get(agentName), endedAt: now }); + // Only this turn's own record ends: a name whose run was disposed meanwhile + // (runEphemeralToCompletion deletes its record) gets none back. + const last = this.lastInferenceAt.get(agentName); + if (last?.startedAt === startedAt) this.lastInferenceAt.set(agentName, { ...last, endedAt: now }); this.emitTrace({ type: 'inference:aborted', agentName, reason, durationMs: now - startedAt }); } diff --git a/test/tool-result-guard.test.ts b/test/tool-result-guard.test.ts index 9555fa11b..c80179e33 100644 --- a/test/tool-result-guard.test.ts +++ b/test/tool-result-guard.test.ts @@ -3262,8 +3262,9 @@ test('a framework compile from before stop cannot drive after restart', async () // removed, the agent object is replaced under its name, or the replacement's // own turn already holds the name's turn token. for (const phase of ['before', 'after'] as const) { - for (const how of ['stopping', 'removed', 'replaced', 'superseded'] as const) { - const title = { stopping: 'the framework stopping', removed: 'its agent removed', replaced: 'its agent replaced', + for (const how of ['stopping', 'removed', 'disposed', 'replaced', 'superseded'] as const) { + const title = { stopping: 'the framework stopping', removed: 'its agent removed', + disposed: 'its run disposed with its record', replaced: 'its agent replaced', superseded: "a newer turn of its agent's name running" }[how]; test(`a turn dropped ${phase} its compile, with ${title}, ${how === 'superseded' ? "leaves the newer turn's bracket alone" : 'ends with inference:aborted'}`, async () => { const dir = mkdtempSync(join(tmpdir(), `guard-drop-${phase}-compile-`)); dirs.push(dir); @@ -3305,7 +3306,12 @@ for (const phase of ['before', 'after'] as const) { await entered.opened; if (how === 'stopping') await framework.stop(); else if (how === 'removed') internal.agents.delete('assistant'); - else { + else if (how === 'disposed') { + // As runEphemeralToCompletion's disposal does: the agent, its record and its token go. + internal.agents.delete('assistant'); + internal.lastInferenceAt.delete('assistant'); + internal.activeTurnTokens.delete('assistant'); + } else { internal.agents.set('assistant', Object.create(agent)); if (how === 'superseded') internal.activeTurnTokens.set('assistant', -1); } @@ -3328,7 +3334,13 @@ for (const phase of ['before', 'after'] as const) { assert.equal(turnEvents[1]?.reason, reason); const durationMs = turnEvents[1]?.durationMs ?? NaN; assert.ok(Number.isFinite(durationMs) && durationMs >= heldMs - 5 && durationMs < 60_000, `durationMs ${durationMs}`); - assert.ok(last?.endedAt !== undefined && last.endedAt >= (last.startedAt ?? Infinity), JSON.stringify(last)); + if (how === 'disposed') { + // The terminal event still closes a listener's bracket, but the + // disposed name gets no record back. + assert.equal(internal.lastInferenceAt.has('assistant'), false, JSON.stringify(last)); + } else { + assert.ok(last?.endedAt !== undefined && last.endedAt >= (last.startedAt ?? Infinity), JSON.stringify(last)); + } } } finally { if (how === 'superseded') internal.activeTurnTokens.delete('assistant');