From 3d83ad0cb229c6efbaac37510d89347d16898e77 Mon Sep 17 00:00:00 2001 From: Ada-1017 Date: Sat, 10 Oct 2026 17:52:54 -0400 Subject: [PATCH 01/34] fix(deferral): hold window-only messages to the turn's end; injected messages are read Context-manager #155 (thinking binding): a provider that binds signed thinking to the prefix it was minted under refuses a reply whose stored prefix holds a message its request didn't. Context-manager #160 stamps each reply from the store as it stands when the reply is stored, and the host's side of that contract (acceptRound's doc) is that what's stored after a compile, up to a reply, is what the round's request carried. Three agent-framework paths broke it. 1. Window-only deferred messages. At a tool boundary every message deferred for the agent was stored, and only the injectable ones were injected: one carrying tool blocks (the membrane's tool cycle) or named as the agent (a prefill) was stored between the turn's rounds without being injected. Sources: the built-in Discord module's backfill stores the bot's own messages under the agent's name; any module's addMessage as the agent or with tool blocks; the puppet pair's defensive queueing. Oscar-1005 showed the direction (room-203 #67671): the reply stored after such a message is stamped with it, later compiles render it, the stamp matches, and the thinking goes out bound to a request without it. Fix: drainDeferredFor takes an optional stopAt; the tool boundary drains the agent's queue only up to its first message that isInjectableMidTurn rejects. That message and everything queued behind it stay queued for the turn-end flush (driveStream's finally), where no reply follows them in the stream. A cut rather than a skip: the queue's order is kept everywhere else, the Discord backfill depends on chronology, and a pair must stay adjacent. Cost: messages queued behind a held one wait for the turn's end. A budget restart doesn't drain at its turn start (continuingTurn), so they wait through it too. 2. Re-deferral split a pair. addMessage's deferral skipped any write carrying tool_result. A turn-end flush can run after a successor turn has taken the agent (the 2026-07-31 interleave) and re-adds drained entries through addMessage, so a queued pair's tool_use re-deferred while its tool_result was stored alone. (1) sends every mid-turn pair through that flush. Now only a fresh tool_result write keeps the bypass; one coming back from the queue (deferredWriteId) re-defers like anything else. resume() clears quiesced before its flush and boot recovery runs with no turn alive, so neither changes. This also covers a flush while another agent's tool cycle is pending (the default path's global midToolCycle). The post-surgery flush (the surgery hold's release) was the one re-add from the queue that passed no deferredWriteId, so its re-adds counted as fresh, and boot recovery couldn't dedupe its writes after a crash; it now passes the id like the others (Petra-1012 found it reading the PR text). 3. RFC-006 watermark. The consumed watermark moved only at a compile (and boot, branch switch, a new fork). A message injected at a tool boundary is stored above the turn compile's watermark, so isUnreadStoredMessage called it unread and a coalesced replacement for its subject could remove it (push-coalescer acceptPlain/acceptDeferred/retract -> removePlacement -> cm.removeMessage), mid-turn or before the next compile: a message the agent heard and answered, and, under #160, either a stripped reply or a refused one depending on whether the removal came after or before the reply was stored (Oscar-1005, room-475 #67755). Fix: after provideToolResults hands injected messages to the stream, agent.markContextConsumed(). Store head, not the last injected sequence: conservative for other slots' messages stored since the compile (they read as read, so a replacement appends). Only when something was injected; endTurn, restart and non-streaming boundaries inject nothing and leave the watermark alone. Tests: test/deferred-window-only.test.ts, 7. Red at 02adf27: 6 fail; the control (a message stored at an endTurn boundary stays unread) passes before and after. operator-surgery, +1: a write deferred behind a surgery lands with its durable id (fails with the post-surgery flush passing none). --- .../mid-turn-window-only-held.fixed.md | 18 ++ src/agent.ts | 6 +- src/framework.ts | 87 ++++-- test/deferred-window-only.test.ts | 251 ++++++++++++++++++ test/operator-surgery.test.ts | 29 ++ 5 files changed, 360 insertions(+), 31 deletions(-) create mode 100644 changelog.d/mid-turn-window-only-held.fixed.md create mode 100644 test/deferred-window-only.test.ts diff --git a/changelog.d/mid-turn-window-only-held.fixed.md b/changelog.d/mid-turn-window-only-held.fixed.md new file mode 100644 index 000000000..50cbaf296 --- /dev/null +++ b/changelog.d/mid-turn-window-only-held.fixed.md @@ -0,0 +1,18 @@ +- A message deferred during a turn that can't join the live stream at a tool + boundary — one carrying tool blocks, or named as the agent itself — now + waits for the turn's end, after its last round, instead of being stored at + the boundary between rounds the stream never showed it. Messages queued + behind it wait with it, so the queue keeps its order; messages queued + before it are still heard at the boundary. Stored between rounds, it sat + before the turn's later replies in every later compile although their + requests never held it: a prompt-cache divergence, and on a provider that + binds signed thinking to the prefix it was minted under (context-manager + #155), a reply whose thinking is refused. +- A message injected into the live stream at a tool boundary now counts as + read for RFC-006 coalescing: the consumed watermark moves when messages + are injected, not only at a compile. Before, an injected message sat above + the watermark of the turn's compile, so a coalesced replacement for its + subject could remove a message the agent had heard and answered. +- A deferred write that goes back into the queue because a turn is alive now + does so whatever its content. A `tool_result` used to land regardless, + apart from the `tool_use` it answers, which went back into the queue. diff --git a/src/agent.ts b/src/agent.ts index 412707a33..4b3ad9f2c 100644 --- a/src/agent.ts +++ b/src/agent.ts @@ -555,7 +555,8 @@ export class Agent { /** * RFC-006 consumed watermark: the newest stored message at the last compile - * of a model request, per branch. Everything above it has never been in a + * of a model request, or at the last tool boundary that injected messages + * into a live one, per branch. Everything above it has never been in a * request and may still be replaced or withdrawn in place by its sender. */ private consumedWatermark: { branch: string; sequence: number } | null = null; @@ -564,7 +565,8 @@ export class Agent { return this.consumedWatermark; } - /** Mark everything currently stored as consumed (boot, branch switch). */ + /** Mark everything currently stored as consumed (boot, branch switch, and a + * tool boundary that injected messages into the live stream). */ markContextConsumed(): void { // Tolerates partial context-manager doubles (tests build Agents over // stubs): without a watermark nothing is ever treated as unread. diff --git a/src/framework.ts b/src/framework.ts index 349bdd693..63b84efd8 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -480,6 +480,14 @@ function withDeferredWriteId(metadata: MessageMetadata | undefined, id: string): return { ...(metadata ?? {}), deferredWriteId: id } as MessageMetadata; } +/** Whether a deferred message can join a live stream at a tool boundary: + * no tool blocks (the membrane's tool cycle) and not named as the agent + * itself (it would render as an assistant turn, a prefill). */ +function isInjectableMidTurn(msg: { participant: string; content: ContentBlock[] }, agentName: string): boolean { + return msg.participant !== agentName + && !msg.content.some((b) => b.type === 'tool_use' || b.type === 'tool_result'); +} + function isTurnContinuation(reason: string): boolean { return reason === 'context_budget_restart' || reason === 'tool_results_ready' || reason === 'tool_result_guard_retry'; @@ -5925,8 +5933,10 @@ export class AgentFramework { if (this.activeTurnTokens.has(name)) continue; for (const msg of this.drainDeferredFor(name)) { try { - this.addMessage(msg.participant, msg.content, msg.metadata, - msg.forAgent ? { forAgent: msg.forAgent } : undefined); + this.addMessage(msg.participant, msg.content, msg.metadata, { + deferredWriteId: msg.id, + ...(msg.forAgent ? { forAgent: msg.forAgent } : {}), + }); } catch (error) { console.error(`[operator] post-surgery deferred flush failed for ${name}: ${error instanceof Error ? error.message : String(error)}`); } @@ -6694,28 +6704,29 @@ export class AgentFramework { // target's live stream (deaf agent + window/live divergence). Left // queued, they flush at the target's own next boundary — with // injection — or in driveStream's finally when its turn ends. + // + // Only what can be injected is drained here. Tool blocks would + // corrupt the tool-cycle structure the membrane enforces, and a + // message named as the agent itself would render as an ASSISTANT + // turn on the wire (an unintended prefill the model would + // continue). Stored here without being injected, such a message + // would sit in the window before the turn's later replies although + // their requests never held it: the divergence above, and a reply + // whose signed thinking the provider then refuses, since it binds + // thinking to the prefix it was minted under (context-manager + // #155). So the drain stops at the first such message, which waits + // with everything queued behind it — the queue keeps its order — + // for the turn-end flush, where no reply follows it in this stream. const midTurnInjections: Array<{ participant: string; content: ContentBlock[]; metadata?: MessageMetadata }> = []; { - const deferred = this.drainDeferredFor(agent.name); - if (deferred.length > 0) { - for (const msg of deferred) { - agent.getContextManager().addMessage(msg.participant, msg.content, withDeferredWriteId(msg.metadata, msg.id)); - // Injection guards: tool blocks would corrupt the tool-cycle - // structure the membrane enforces, and a message named as the - // agent itself would render as an ASSISTANT turn on the wire - // (an unintended prefill the model would continue). Such - // messages stay window-only — visible next turn. - const hasToolBlocks = msg.content.some( - (b) => b.type === 'tool_use' || b.type === 'tool_result' - ); - if (!hasToolBlocks && msg.participant !== agent.name) { - midTurnInjections.push({ - participant: msg.participant, - content: msg.content, - ...(msg.metadata ? { metadata: msg.metadata } : {}), - }); - } - } + const deferred = this.drainDeferredFor(agent.name, (msg) => !isInjectableMidTurn(msg, agent.name)); + for (const msg of deferred) { + agent.getContextManager().addMessage(msg.participant, msg.content, withDeferredWriteId(msg.metadata, msg.id)); + midTurnInjections.push({ + participant: msg.participant, + content: msg.content, + ...(msg.metadata ? { metadata: msg.metadata } : {}), + }); } } @@ -6918,6 +6929,13 @@ export class AgentFramework { agent.toolResultGuard.submissionResults(membraneResults), midTurnInjections.length > 0 ? { injectedMessages: midTurnInjections } : undefined, ); + // RFC-006 §3.3: consumption is determined at assembly, and this + // is assembly too — the injected messages are in the live + // request. Without this they would sit above the watermark of + // the turn's compile and read as unread, so a coalesced + // replacement could remove a message the agent heard and + // answered (and that its later replies were minted after). + if (midTurnInjections.length > 0) agent.markContextConsumed(); agent.setStreaming(currentState.stream); this.emitTrace({ type: 'inference:stream_resumed', @@ -12549,7 +12567,13 @@ export class AgentFramework { // against a FROZEN context — module events and api message.send must not // append mid-surgery. Deferred here, flushed by resume(). tool_result // still lands (a draining turn's continuation depends on it). - const hasToolResult = content.some(b => b.type === 'tool_result'); + // + // A write coming back from the deferred queue (deferredWriteId) is + // re-deferred whatever its content: its tool_use went back into the + // queue just before it, in the same drain, and a tool_result stored + // without it would split the pair. (A turn-end flush can run after a + // successor turn has taken the agent: the 2026-07-31 interleave.) + const freshToolResult = opts?.deferredWriteId === undefined && content.some(b => b.type === 'tool_result'); // Explicitly-targeted deliveries scope the tool-cycle check to the // target (pendingAssistantBlocks is keyed by agent); the default path // keeps the historical GLOBAL check byte-for-byte (conservative: any @@ -12558,7 +12582,7 @@ export class AgentFramework { ? this.pendingAssistantBlocks.has(agent.name) : this.pendingAssistantBlocks.size > 0; if ( - !hasToolResult && + !freshToolResult && opts?.bypassDeferralFor !== agent.name && (this.quiesced || midToolCycle || @@ -12592,9 +12616,12 @@ export class AgentFramework { /** * Splice out the deferred messages addressed to `agentName` (explicitly * via forAgent, or implicitly when it is the primary), leaving other - * targets' messages queued for their own boundaries. + * targets' messages queued for their own boundaries. With `stopAt`, the + * drain ends at the first of them (in queue order) that it matches: that + * message and every later one stay queued, so what is drained is always a + * prefix of the agent's queue. */ - private drainDeferredFor(agentName: string): Array<{ + private drainDeferredFor(agentName: string, stopAt?: (msg: DeferredWrite) => boolean): Array<{ /** Durable identity — every write of this entry must carry it * (`withDeferredWriteId` / `opts.deferredWriteId`). */ id: string; @@ -12610,14 +12637,16 @@ export class AgentFramework { const target = msg.forAgent ?? this.primaryAgentName; (target === agentName ? mine : rest).push(msg); } - this.deferredMessages = rest; mine.sort(bySeq); + const stop = stopAt ? mine.findIndex(stopAt) : -1; + const drained = stop === -1 ? mine : mine.slice(0, stop); + this.deferredMessages = stop === -1 ? rest : [...rest, ...mine.slice(stop)]; // The drained messages are about to be written by the caller; they stay // in the DURABLE queue (as un-acked) until the caller's // ackDeferredWrites() has synced the chronicle. Removing them here would // acknowledge before the write is durable. - this.handOffDeferredWrites(mine); - return mine; + this.handOffDeferredWrites(drained); + return drained; } private editMessage(id: MessageId, content: ContentBlock[]): void { diff --git a/test/deferred-window-only.test.ts b/test/deferred-window-only.test.ts new file mode 100644 index 000000000..6bfa3302d --- /dev/null +++ b/test/deferred-window-only.test.ts @@ -0,0 +1,251 @@ +/** + * What the window stores mid-turn is what the live stream carried. + * + * A message deferred during a turn is stored at the agent's next tool + * boundary and injected into the resumed stream (hear-while-acting), so the + * next compile's prefix is the request the turn's later replies were minted + * under. Two kinds of message can't be injected: one carrying tool blocks + * (it would break the tool cycle membrane enforces) and one named as the + * agent itself (a prefill). Those wait for the turn's end, and so does + * everything queued behind them, so the queue's order holds. A provider that + * binds signed thinking to its prefix (context-manager #155) refuses a reply + * whose stored prefix holds a message its request didn't. + * + * A message the stream carried has been read: RFC-006's consumed watermark + * covers it, so a coalesced replacement can no longer remove it. + */ + +import { describe, it, beforeEach, afterEach } from 'node:test'; +import assert from 'node:assert/strict'; +import { tmpdir } from 'node:os'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { join } from 'node:path'; +import type { + Module, + ModuleContext, + ProcessState, + ProcessEvent, + EventResponse, + ToolDefinition, + ToolCall, + ToolResult, +} from '../src/index.js'; +import { AgentFramework } from '../src/index.js'; +import { MockMembrane, createMockResponse } from './helpers/mock-membrane.js'; +import type { ContentBlock } from '@animalabs/membrane'; + +interface Interjection { + participant: string; + content: ContentBlock[]; +} + +/** Tools that deliver messages while they run, the way a module or a channel + * would while the agent is busy: `move` continues the turn, `halt` ends it. */ +class InterjectingModule implements Module { + readonly name = 'robot'; + framework: AgentFramework | null = null; + /** Delivered, in order, by the next tool call before it returns. */ + interjections: Interjection[] = []; + + async start(_ctx: ModuleContext): Promise {} + async stop(): Promise {} + + getTools(): ToolDefinition[] { + return [ + { name: 'move', description: 'Move', inputSchema: { type: 'object', properties: {} } }, + { name: 'halt', description: 'End the turn', inputSchema: { type: 'object', properties: {} } }, + ]; + } + + async handleToolCall(call: ToolCall): Promise { + const pending = this.interjections; + this.interjections = []; + for (const interjection of pending) { + this.framework!.pushEvent({ type: 'interjection', source: 'test', interjection } as unknown as ProcessEvent); + } + // Let the run loop handle them while this round's tools are pending, so + // they land in the deferred queue before the tool-result boundary. + if (pending.length > 0) await new Promise((r) => setTimeout(r, 30)); + return call.name.endsWith('halt') + ? { success: true, data: { halted: true }, endTurn: true } + : { success: true, data: { ok: true } }; + } + + async onProcess(event: ProcessEvent, _state: ProcessState): Promise { + if (event.type === 'external-message') { + return { + addMessages: [{ participant: 'Antra', content: [{ type: 'text', text: 'go' }] }], + requestInference: true, + }; + } + if ((event as { type: string }).type === 'interjection') { + const { participant, content } = (event as unknown as { interjection: Interjection }).interjection; + // No wake: this suite is about where the message lands in this turn. + return { addMessages: [{ participant, content }] }; + } + return {}; + } +} + +const text = (t: string): ContentBlock[] => [{ type: 'text', text: t } as ContentBlock]; + +/** One line per stored message: participant, then its text or block types. */ +function windowOf(framework: AgentFramework): string[] { + return framework.getAgent('assistant')!.getContextManager().getAllMessages().map((m) => { + const parts = m.content.map((b) => (b.type === 'text' ? (b as { text: string }).text : `<${b.type}>`)); + return `${m.participant}: ${parts.join(' ')}`; + }); +} + +describe('mid-turn deliveries keep the window what the stream carried', () => { + let tempDir: string; + let membrane: MockMembrane; + let module: InterjectingModule; + + beforeEach(() => { + tempDir = mkdtempSync(join(tmpdir(), 'dwo-test-')); + membrane = new MockMembrane(); + module = new InterjectingModule(); + }); + + afterEach(() => { + rmSync(tempDir, { recursive: true, force: true }); + }); + + async function createFramework(): Promise { + const framework = await AgentFramework.create({ + storePath: join(tempDir, 'test.chronicle'), + membrane: membrane.asMembrane(), + agents: [{ name: 'assistant', model: 'test-model', systemPrompt: 'You are a robot pilot.' }], + modules: [module], + }); + module.framework = framework; + return framework; + } + + /** A turn of two rounds: `tool` with the interjections, then a reply. */ + async function runTurn(tool: 'move' | 'halt', interjections: Interjection[]): Promise { + membrane.pushResponse(createMockResponse([ + { type: 'text', text: 'Moving.' }, + { type: 'tool_use', id: 'c1', name: `robot--${tool}`, input: {} }, + ] as ContentBlock[], 'tool_use')); + membrane.pushResponse(createMockResponse(text('Done.'))); + const framework = await createFramework(); + module.interjections = interjections; + framework.pushEvent({ type: 'external-message', source: 'test', content: 'go' } as unknown as ProcessEvent); + await framework.runUntilIdle(); + return framework; + } + + function injected(): string[] { + const options = membrane.lastStream!.receivedToolResultOptions[0]; + return (options?.injectedMessages ?? []).map((m) => + `${m.participant}: ${(m.content as Array<{ text?: string }>).map((b) => b.text).join(' ')}`); + } + + it('a message named as the agent waits for the turn to end, after its last round', async () => { + const framework = await runTurn('move', [{ participant: 'assistant', content: text('my own words') }]); + assert.deepEqual(injected(), []); + assert.deepEqual(windowOf(framework), [ + 'Antra: go', + 'assistant: Moving. ', + 'user: ', + 'assistant: Done.', + 'assistant: my own words', + ]); + await framework.stop(); + }); + + it('a message carrying tool blocks waits too', async () => { + const framework = await runTurn('move', [{ + participant: 'Antra', + content: [{ type: 'tool_use', id: 'x1', name: 'robot--move', input: {} } as ContentBlock], + }]); + assert.deepEqual(injected(), []); + assert.deepEqual(windowOf(framework).slice(2), ['user: ', 'assistant: Done.', 'Antra: ']); + await framework.stop(); + }); + + it('what is queued behind a held message waits with it, in order', async () => { + const framework = await runTurn('move', [ + { participant: 'assistant', content: text('my own words') }, + { participant: 'Antra', content: text('look left') }, + ]); + assert.deepEqual(injected(), []); + assert.deepEqual(windowOf(framework).slice(2), [ + 'user: ', + 'assistant: Done.', + 'assistant: my own words', + 'Antra: look left', + ]); + await framework.stop(); + }); + + it('what is queued before a held message is still heard at the boundary', async () => { + const framework = await runTurn('move', [ + { participant: 'Antra', content: text('look left') }, + { participant: 'assistant', content: text('my own words') }, + ]); + assert.deepEqual(injected(), ['Antra: look left']); + assert.deepEqual(windowOf(framework).slice(2), [ + 'user: ', + 'Antra: look left', + 'assistant: Done.', + 'assistant: my own words', + ]); + await framework.stop(); + }); + + it('a message the stream carried is read: the consumed watermark covers it', async () => { + const framework = await runTurn('move', [{ participant: 'Antra', content: text('look left') }]); + assert.deepEqual(injected(), ['Antra: look left']); + const agent = framework.getAgent('assistant')!; + const heard = agent.getContextManager().getAllMessages() + .find((m) => m.content.some((b) => (b as { text?: string }).text === 'look left'))!; + const isUnread = (framework as unknown as { + isUnreadStoredMessage: (a: typeof agent, id: string) => boolean; + }).isUnreadStoredMessage.bind(framework); + assert.equal(isUnread(agent, heard.id), false, 'a coalesced replacement must not remove what the agent heard'); + await framework.stop(); + }); + + it('a message stored at a boundary that ends the turn was never carried, and stays unread', async () => { + const framework = await runTurn('halt', [{ participant: 'Antra', content: text('too late') }]); + const agent = framework.getAgent('assistant')!; + const late = agent.getContextManager().getAllMessages() + .find((m) => m.content.some((b) => (b as { text?: string }).text === 'too late'))!; + const isUnread = (framework as unknown as { + isUnreadStoredMessage: (a: typeof agent, id: string) => boolean; + }).isUnreadStoredMessage.bind(framework); + assert.equal(isUnread(agent, late.id), true); + await framework.stop(); + }); + + it('a write coming back from the deferred queue re-defers while a turn is alive, tool_result or not', async () => { + const framework = await createFramework(); + const internals = framework as unknown as { + activeTurnTokens: Map; + deferredMessages: Array<{ id: string }>; + addMessage: (p: string, c: ContentBlock[], m?: unknown, o?: { deferredWriteId?: string }) => string; + }; + internals.activeTurnTokens.set('assistant', 1); + try { + // A queued pair's tool_result must follow its tool_use back into the + // queue, or the pair splits (the turn-end flush can run after a + // successor turn has started). + const requeued = internals.addMessage('user', [ + { type: 'tool_result', toolUseId: 'p1', content: 'ok' } as ContentBlock, + ], undefined, { deferredWriteId: 'w1' }); + assert.equal(requeued, ''); + assert.ok(internals.deferredMessages.some((m) => m.id === 'w1')); + // A fresh tool_result write keeps landing as before. + const fresh = internals.addMessage('user', [ + { type: 'tool_result', toolUseId: 'p2', content: 'ok' } as ContentBlock, + ]); + assert.notEqual(fresh, ''); + } finally { + internals.activeTurnTokens.delete('assistant'); + } + await framework.stop(); + }); +}); diff --git a/test/operator-surgery.test.ts b/test/operator-surgery.test.ts index aeb162b59..2be1ce269 100644 --- a/test/operator-surgery.test.ts +++ b/test/operator-surgery.test.ts @@ -228,6 +228,35 @@ describe('live operator surgery', () => { } }); + it('a write deferred behind the surgery lands with its durable id, as every flush from the queue does', async () => { + const c = cm() as unknown as { switchBranch: (name: string) => Promise }; + const originalSwitch = c.switchBranch; + let releaseSwitch!: () => void; + const gate = new Promise((resolve) => { releaseSwitch = resolve; }); + c.switchBranch = async (name: string) => { await gate; return originalSwitch.call(c, name); }; + const internals = framework as unknown as { + surgeryHold: unknown; + deferredMessages: Array<{ id: string }>; + addMessage(participant: string, content: unknown[], metadata?: Record): string; + }; + try { + const rollback = framework.rollbackToMessage('scout', { messageId: ids[2] }); + await new Promise((resolve) => setImmediate(resolve)); + assert.ok(internals.surgeryHold, 'store hold is up while the switch is awaited'); + assert.equal(internals.addMessage('user', [{ type: 'text', text: 'arrived mid-surgery' }]), '', 'deferred'); + const queuedId = internals.deferredMessages.at(-1)!.id; + releaseSwitch(); + await rollback; + const landed = cm().getAllMessages().find((m) => (m.content[0] as { text?: string }).text === 'arrived mid-surgery'); + assert.ok(landed, 'flushed when the hold released'); + // The id makes a queued pair's tool_result re-defer with its tool_use + // rather than land alone, and lets boot recovery skip a replay. + assert.equal((landed!.metadata as { deferredWriteId?: string } | undefined)?.deferredWriteId, queuedId); + } finally { + c.switchBranch = originalSwitch; + } + }); + // --- strategy-initialization failure ----------------------------------------- // switchBranch()/fork() move the chronicle branch BEFORE awaiting strategy // initialization, so a rejection there leaves the store on the new branch. From 903beb1e9afe4aa19c6f3893bf1b41d8abc76a7f Mon Sep 17 00:00:00 2001 From: Ada-1017 Date: Sat, 10 Oct 2026 19:19:54 -0400 Subject: [PATCH 02/34] fix(deferral): a turn's end delivers despite another agent's tool cycle; stop() stores what is still deferred Greptile's P1 on #286 (discussion_r4239537209): the cut leaves held messages depending on the turn-end flush, and that flush was gated on EVERY agent's pending tool cycle (pendingAssistantBlocks.size === 0, from the single-agent era, efc7e7a). If agent A's turn ended while agent B waited on tools, A's flush was skipped; B's boundary and turn end drain only B's entries, so A's messages stayed queued until A's next turn, and stop() never drained the queue, so a memory-only queue lost them. - driveStream's turn-end flush is now gated on this agent's own cycle (!pendingAssistantBlocks.has(agent.name)), matching addMessage's scoped check for targeted writes. Each drained entry still goes through addMessage: a targeted write lands; a primary-bound (default-path) write keeps the historical global rule and re-defers under its id and seq while any agent's cycle is pending, for the target's next turn start or stop(). - stop() now drains every agent's remaining entries and stores them (deferredWriteId stamped), then acks, after every turn is over and before the final sync and close. A quiesced framework re-defers them into the persisted queue, as before. Tests (deferred-window-only, +2): a message held for the turn's end lands then while another agent's cycle is pending (simulated through pendingAssistantBlocks); a message still deferred at stop() is in the store when it's reopened. Both fail at 3d83ad0's source. Neighbours (deferred-*, present-while-acting, operator-surgery, host-quiesce, mcpl-coalescing, puppet-*, offline-recovery-branch): 168/168. --- .../mid-turn-window-only-held.fixed.md | 8 +++ src/framework.ts | 32 +++++++++++- test/deferred-window-only.test.ts | 52 +++++++++++++++++++ 3 files changed, 90 insertions(+), 2 deletions(-) diff --git a/changelog.d/mid-turn-window-only-held.fixed.md b/changelog.d/mid-turn-window-only-held.fixed.md index 50cbaf296..36641ad8e 100644 --- a/changelog.d/mid-turn-window-only-held.fixed.md +++ b/changelog.d/mid-turn-window-only-held.fixed.md @@ -16,3 +16,11 @@ - A deferred write that goes back into the queue because a turn is alive now does so whatever its content. A `tool_result` used to land regardless, apart from the `tool_use` it answers, which went back into the queue. +- A turn's end now delivers the agent's deferred messages even while another + agent's tool cycle is pending; only the agent's own cycle holds them. A + write still deferred then (a primary-bound write waits out any agent's + cycle) lands at the agent's next turn start, and `stop()` now stores + whatever is still deferred before the store closes. Before, a message + deferred for an agent whose turn ended during another agent's tool cycle + stayed queued until that agent's next turn, and a memory-only queue lost it + at `stop()`. diff --git a/src/framework.ts b/src/framework.ts index 63b84efd8..9f6a45760 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -2043,6 +2043,29 @@ export class AgentFramework { } } + // Every turn is over, so nothing else will deliver what is still + // deferred: a message held for a turn's end whose flush found the write + // still deferred (a primary-bound write waits out any agent's tool + // cycle), or one that arrived for an idle agent during another's cycle. + // Land each at its target now, while the store is open; a memory-only + // queue would otherwise lose it. One addMessage still defers (quiesced) + // stays in the persisted queue for boot recovery. + if (this.deferredMessages.length > 0) { + for (const name of this.agents.keys()) { + for (const msg of this.drainDeferredFor(name)) { + try { + this.addMessage(msg.participant, msg.content, msg.metadata, { + deferredWriteId: msg.id, + ...(msg.forAgent ? { forAgent: msg.forAgent } : {}), + }); + } catch (error) { + console.error(`[stop] deferred write for ${name} could not be stored: ${error instanceof Error ? error.message : String(error)}`); + } + } + } + this.ackDeferredWrites(); + } + // Final sync before closing try { this.store.sync(); @@ -10953,8 +10976,13 @@ export class AgentFramework { // Flush any deferred messages (e.g. if stream failed while tools were // pending). Only THIS agent's messages: other targets' entries wait // for their own boundaries (re-adding via addMessage re-defers if the - // target has meanwhile started a turn). - if (frameReachedTerminal && this.deferredMessages.length > 0 && this.pendingAssistantBlocks.size === 0) { + // target has meanwhile started a turn). Gated on this agent's own tool + // cycle, not every agent's: messages held for this turn's end depend on + // this flush, and another agent's pending tools say nothing about this + // window. A write addMessage still defers (a primary-bound write waits + // out any agent's cycle) goes back into the queue under its id, for the + // target's next turn start or stop(). + if (frameReachedTerminal && this.deferredMessages.length > 0 && !this.pendingAssistantBlocks.has(agent.name)) { const deferred = this.drainDeferredFor(agent.name); for (const msg of deferred) { this.addMessage(msg.participant, msg.content, msg.metadata, { diff --git a/test/deferred-window-only.test.ts b/test/deferred-window-only.test.ts index 6bfa3302d..246293027 100644 --- a/test/deferred-window-only.test.ts +++ b/test/deferred-window-only.test.ts @@ -46,6 +46,8 @@ class InterjectingModule implements Module { framework: AgentFramework | null = null; /** Delivered, in order, by the next tool call before it returns. */ interjections: Interjection[] = []; + /** Run inside the next tool call, after the interjections are delivered. */ + duringTool: (() => void) | null = null; async start(_ctx: ModuleContext): Promise {} async stop(): Promise {} @@ -66,6 +68,9 @@ class InterjectingModule implements Module { // Let the run loop handle them while this round's tools are pending, so // they land in the deferred queue before the tool-result boundary. if (pending.length > 0) await new Promise((r) => setTimeout(r, 30)); + const during = this.duringTool; + this.duringTool = null; + during?.(); return call.name.endsWith('halt') ? { success: true, data: { halted: true }, endTurn: true } : { success: true, data: { ok: true } }; @@ -221,6 +226,53 @@ describe('mid-turn deliveries keep the window what the stream carried', () => { await framework.stop(); }); + /** The framework's own queue and its record of pending tool cycles. */ + function internalsOf(framework: AgentFramework) { + return framework as unknown as { + pendingAssistantBlocks: Map; + deferredMessages: Array<{ id: string }>; + addMessage(participant: string, content: ContentBlock[], metadata?: undefined, opts?: { forAgent?: string }): string; + }; + } + + it('a message held for the turn\'s end lands then, while another agent\'s tool cycle is pending', async () => { + membrane.pushResponse(createMockResponse([ + { type: 'text', text: 'Moving.' }, + { type: 'tool_use', id: 'c1', name: 'robot--move', input: {} }, + ] as ContentBlock[], 'tool_use')); + membrane.pushResponse(createMockResponse(text('Done.'))); + const framework = await createFramework(); + const internals = internalsOf(framework); + module.duringTool = () => { + // Another agent is mid tool cycle when this turn ends, and a write + // named as this agent (held: it can't join the live stream) waits. + internals.pendingAssistantBlocks.set('builder', [{ type: 'tool_use', id: 'b1', name: 'robot--move', input: {} } as ContentBlock]); + assert.equal(internals.addMessage('assistant', text('my own words'), undefined, { forAgent: 'assistant' }), '', 'deferred'); + }; + framework.pushEvent({ type: 'external-message', source: 'test', content: 'go' } as unknown as ProcessEvent); + await framework.runUntilIdle(); + assert.ok(internals.pendingAssistantBlocks.has('builder'), 'the other cycle was still pending at the flush'); + assert.deepEqual(windowOf(framework).slice(-2), ['assistant: Done.', 'assistant: my own words']); + internals.pendingAssistantBlocks.delete('builder'); + await framework.stop(); + }); + + it('a message still deferred when the framework stops lands before the store closes', async () => { + const framework = await createFramework(); + const internals = internalsOf(framework); + // A primary-bound write waits out any agent's tool cycle, even with this + // agent idle; that cycle's stream then ends without delivering it. + internals.pendingAssistantBlocks.set('builder', [{ type: 'tool_use', id: 'b1', name: 'robot--move', input: {} } as ContentBlock]); + assert.equal(internals.addMessage('Antra', text('late word')), '', 'deferred'); + internals.pendingAssistantBlocks.delete('builder'); + assert.equal(internals.deferredMessages.length, 1); + await framework.stop(); + + const reopened = await createFramework(); + assert.ok(windowOf(reopened).includes('Antra: late word'), 'stored before the store closed'); + await reopened.stop(); + }); + it('a write coming back from the deferred queue re-defers while a turn is alive, tool_result or not', async () => { const framework = await createFramework(); const internals = framework as unknown as { From 0c5c0b0d0d02bbf52cf408d823d2f63b701223ce Mon Sep 17 00:00:00 2001 From: Olive-1011 Date: Sun, 11 Oct 2026 16:08:00 -0400 Subject: [PATCH 03/34] fix(deferral): one rule for every write; stop() admits no turn or stream; a failed store never drops a drained entry; the watermark only advances A review of this pull request's first two commits found that the deferral rules still disagreed with each other, that stop()'s drain could meet a live turn, that a failed store could drop a drained entry, and that a compile could roll the consumed watermark back. This commit answers each. One rule for every write: - addMessage's default path checked every agent's tool cycle (pendingAssistantBlocks.size > 0, from efc7e7af, when the agents shared one store), while a targeted write checked only its target's. Now every write waits while its target's own turn, stream or tool cycle is alive, or the host is quiesced. So the primary's writes no longer wait behind a fork's, subagent's or the subconscious's tools, which protected nothing in the primary's window, and one drained queue is judged by one rule: the turn-end flush stores the drained prefix in order, or it goes back whole when a successor turn has taken the agent. Before, a later targeted entry could be stored ahead of an earlier primary-bound one. - Every flush gates on its own agent's cycle: the post-surgery flush (per reserved agent), the withdrawn-turn flush and the puppet's finally, as the turn-end flush already did. The puppet's comment now says what is true. The post-surgery flush also acks after its re-adds, as the other flushes do; before, its entries stayed un-acked until some later ack. - Residents that share the un-namespaced slot with overlapping turns put each other's rounds and writes inside the other's live turn whatever this check does. That limit is stated in the pull request, not fixed here. stop()'s premise made true: once stop() begins, no turn starts, for any agent (startAgentStream checks providerAdmissionClosed first, so a fork's or subagent's retry frame and a wake the loop already dequeued are refused, not only gate-owning residents), and no stream registers (beginAgentTurn refuses beside the identity check, cancelling the stream; the caller's finally releases the token). Before, a turn mid-compile at the cancel pass registered afterwards and outlived stop(), and the drain re-deferred its agent's entries into memory, where they were lost at exit. An ephemeral run refused either way, or one admitted while stopping (runEphemeralToCompletion), ends at once with "the framework is stopping" rather than at its startup watchdog. So "every turn is over" holds at stop()'s drain, which stores every registered target's entries; a quiesced host still re-defers into the persisted queue. A failed store never drops a drained entry: writeDrained, through which every site writes a drained entry (stop's drain, the post-surgery, withdrawn-turn, turn-end and puppet flushes and resume/boot recovery via readdDrained; the turn-start flush and the tool boundary, which store straight to the context manager), logs a write that throws, moves the entry from the un-acked set back into the queue under its id and seq (persisting a persisted queue), and lets the rest continue. At a tool boundary such an entry isn't injected. Before, it stayed un-acked and the next ack dropped it from the persisted queue; at a turn start or from a memory-only queue it was lost at once. A write that always throws is retried, and logged, at each later flush of its target; it is never dropped. drainDeferredFor now returns the queue's own entries, which carry seq. The watermark only advances: compileContext and compileWithInjections write their watermark back through applyCompileWatermark, which counts marks: if one was made while the compile ran (a tool boundary's, a branch switch's), the compile's older watermark applies only where it is later on that mark's branch. Before, a preview compiling during a live turn rolled a boundary's mark back. The puppet's defensive branch persists the queue after queueing its pair, as addMessage's deferral does, so a crash before the next persist can't lose the pair from a persisted queue. The boundary's mark is the store's head, a scalar, as a compile's is, so it also covers what was stored since the compile without being carried: another slot's messages, and a write that bypassed deferral into this slot. The comment there now names that over-cover, says that under anima-research/context-manager#160 such a write already gives the replies stored after it stamps their requests didn't carry, whatever the mark says, and points at deferring those writers as the fix. Changelog: stop() stores what is deferred for a registered target, a quiesced host keeps its persisted queue, and an entry for an unregistered target is dropped as before; a re-deferral's cause is the target's own turn, stream or tool cycle, or a quiesced host; the cost to an addressed message queued behind a held one is named; context-manager references are fully qualified; the one rule, the stop premise, a failed store and the watermark are described. Tests (test/deferred-window-only.test.ts, 16, was 9): - the interjecting tool returns once its interjections are queued (bounded) instead of after 30 ms, and the hold tests record what the boundary's drain found queued, so the cut is exercised rather than reached by a late message; - the held-for-turn-end test records, with a spy at the flush itself, that the other agent's cycle was pending there; - a write waits on its target's own turn and cycle, not another agent's, primary-bound or targeted; - the puppet's flush lands what was deferred behind it while another agent's cycle is pending; - stop() while a turn is mid-compile under the real event loop, with an entry deferred for that agent: no stream is left, the refused turn produced nothing, and after reopening the entry is stored; - a drained write whose store throws at stop() stays in the persisted queue and is stored at the next boot; - at a tool boundary, a write whose store throws stays queued and isn't injected, the next is stored and injected, and the first lands at the turn's end; - a message stored straight into the context during a round, with nothing injected at the boundary, stays unread; - a compile's watermark doesn't roll back a mark made while it ran, and a later compile still advances it; - a held message waits out a successor turn's tool cycle (the stream generation, token and cycle the successor's when the first turn's teardown flushes) and lands once it is over; - the re-deferral test also re-defers plain text coming back from the queue. - test/host-quiesce.test.ts's poison-write test pinned the old drop ("the queue is drained even though one write failed"); resume's flush writes drained entries too, so it now pins that the write that failed stays queued (deferredWrites 1), while the others land and the reopen completes. Each of these fixes, undone alone, fails test/deferred-window-only.test.ts: the default path waiting on every agent's cycle; the puppet's flush waiting on every agent's cycle; a stream registering while stopping; a failed store only logged; the boundary's store unguarded; a compile's watermark always written back; every continuing boundary marking the watermark. --- .../mid-turn-window-only-held.fixed.md | 54 +++- src/agent.ts | 25 +- src/framework.ts | 229 ++++++++------ test/deferred-window-only.test.ts | 298 ++++++++++++++++-- test/host-quiesce.test.ts | 6 +- 5 files changed, 472 insertions(+), 140 deletions(-) diff --git a/changelog.d/mid-turn-window-only-held.fixed.md b/changelog.d/mid-turn-window-only-held.fixed.md index 36641ad8e..0b741f96b 100644 --- a/changelog.d/mid-turn-window-only-held.fixed.md +++ b/changelog.d/mid-turn-window-only-held.fixed.md @@ -3,24 +3,46 @@ waits for the turn's end, after its last round, instead of being stored at the boundary between rounds the stream never showed it. Messages queued behind it wait with it, so the queue keeps its order; messages queued - before it are still heard at the boundary. Stored between rounds, it sat - before the turn's later replies in every later compile although their - requests never held it: a prompt-cache divergence, and on a provider that - binds signed thinking to the prefix it was minted under (context-manager - #155), a reply whose thinking is refused. + before it are still heard at the boundary. The cost: a message addressed to + the agent but queued behind a held one isn't heard mid-turn, and waits for + the turn's end. Stored between rounds, a held message sat before the turn's + later replies in every later compile although their requests never held + it: a prompt-cache divergence, and on a provider that binds signed thinking + to the prefix it was minted under (anima-research/context-manager#155, + whose binding anima-research/context-manager#160 walks), a reply whose + thinking is refused. - A message injected into the live stream at a tool boundary now counts as read for RFC-006 coalescing: the consumed watermark moves when messages are injected, not only at a compile. Before, an injected message sat above the watermark of the turn's compile, so a coalesced replacement for its - subject could remove a message the agent had heard and answered. -- A deferred write that goes back into the queue because a turn is alive now - does so whatever its content. A `tool_result` used to land regardless, + subject could remove a message the agent had heard and answered. Within a + branch the watermark only advances: a compile that started before such a + mark doesn't roll it back when it finishes. +- A deferred write that goes back into the queue does so whatever its + content. That happens while its target's own turn, stream or tool cycle is + alive, or the host is quiesced. A `tool_result` used to land regardless, apart from the `tool_use` it answers, which went back into the queue. -- A turn's end now delivers the agent's deferred messages even while another - agent's tool cycle is pending; only the agent's own cycle holds them. A - write still deferred then (a primary-bound write waits out any agent's - cycle) lands at the agent's next turn start, and `stop()` now stores - whatever is still deferred before the store closes. Before, a message - deferred for an agent whose turn ended during another agent's tool cycle - stayed queued until that agent's next turn, and a memory-only queue lost it - at `stop()`. +- Every write now waits on its own target's state: the target's turn, stream + and tool cycle, or a quiesced host. Before, a write for the primary agent + waited out every agent's tool cycle, while one addressed to an agent waited + only on that agent's. So a turn's end could store a later addressed message + ahead of an earlier one for the primary, and a fork's, subagent's or the + subconscious's tools held the primary's writes, which protected nothing in + the primary's window. Each flush, at a turn's end, after surgery, after a + withdrawn turn and after a puppet call, now delivers the agent's deferred + messages while another agent's tool cycle is pending; only the agent's own + cycle holds them. +- `stop()` now stores what is still deferred for each registered agent + before the store closes; before, a memory-only queue lost it. Once `stop()` + begins, no turn starts and no stream registers, for any agent, so nothing + is mid-turn when it does: a turn that was mid-compile when the streams + were cancelled is refused rather than left running, and an ephemeral run + refused this way ends at once rather than at its startup watchdog. A + quiesced host keeps what is deferred in its persisted queue for boot + recovery. An entry for a target no longer registered (a disposed fork) is + dropped from a memory-only queue at exit, as before. +- A deferred write whose store throws goes back into the queue under its + id, and the others continue. Before, it stayed in the un-acked set and the + next acknowledgement dropped it from the persisted queue (at a turn start + or a flush it was lost even from memory). At a tool boundary, such a write + isn't injected either. diff --git a/src/agent.ts b/src/agent.ts index 4b3ad9f2c..57a4d1a24 100644 --- a/src/agent.ts +++ b/src/agent.ts @@ -560,6 +560,9 @@ export class Agent { * request and may still be replaced or withdrawn in place by its sender. */ private consumedWatermark: { branch: string; sequence: number } | null = null; + /** Counts marks: a compile writes its watermark back only if no mark was + * made while it ran (a tool boundary's, a branch switch's). */ + private watermarkMarks = 0; getConsumedWatermark(): { branch: string; sequence: number } | null { return this.consumedWatermark; @@ -571,6 +574,22 @@ export class Agent { // Tolerates partial context-manager doubles (tests build Agents over // stubs): without a watermark nothing is ever treated as unread. this.consumedWatermark = this.pendingWatermark(); + this.watermarkMarks++; + } + + /** + * A finished compile's watermark, taken before it awaited. Within a branch + * the watermark only advances: if a mark was made while the compile ran, + * the compile's (older) one is applied only where it is later on that + * mark's branch, so a boundary's mark is never rolled back by a compile + * that started before it (a preview during a live turn). + */ + private applyCompileWatermark(watermark: { branch: string; sequence: number }, marksBefore: number): void { + const current = this.consumedWatermark; + if (this.watermarkMarks !== marksBefore && current + && (current.branch !== watermark.branch || current.sequence >= watermark.sequence)) return; + this.consumedWatermark = watermark; + this.watermarkMarks++; } /** The watermark a successful compile will establish (taken before, applied after). */ @@ -597,8 +616,9 @@ export class Agent { // RFC-006 §3.3: consumption is determined at assembly — a compile that // fails assembled nothing, so the watermark moves only on success. const watermark = this.pendingWatermark(); + const marksBefore = this.watermarkMarks; const result = await this.contextManager.compile(this.compileBudget(budget)); - if (watermark) this.consumedWatermark = watermark; + if (watermark) this.applyCompileWatermark(watermark, marksBefore); if (!budget) this.settleRuntimeSettingsTransition(); return result; } @@ -614,10 +634,11 @@ export class Agent { opts?: { kvUnifiedImmutablePrefixHash?: string }, ): Promise { const watermark = this.pendingWatermark(); + const marksBefore = this.watermarkMarks; const result = await this.contextManager.compile( this.compileBudget(budget), injections, opts as never, ); - if (watermark) this.consumedWatermark = watermark; + if (watermark) this.applyCompileWatermark(watermark, marksBefore); if (!budget) this.settleRuntimeSettingsTransition(); return result; } diff --git a/src/framework.ts b/src/framework.ts index 9f6a45760..14176841e 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -2043,25 +2043,21 @@ export class AgentFramework { } } - // Every turn is over, so nothing else will deliver what is still - // deferred: a message held for a turn's end whose flush found the write - // still deferred (a primary-bound write waits out any agent's tool - // cycle), or one that arrived for an idle agent during another's cycle. - // Land each at its target now, while the store is open; a memory-only - // queue would otherwise lose it. One addMessage still defers (quiesced) - // stays in the persisted queue for boot recovery. + // Every turn is over: from the start of stop() no turn starts and no + // stream registers (providerAdmissionClosed, checked for every agent), + // and every stream registered before was cancelled and awaited above. So + // nothing else will deliver what is still deferred: a message held for + // a turn's end whose flush found the target busy again, or one that + // arrived for an idle agent and was deferred behind its own state. Land + // each at its registered target now, while the store is open; a + // memory-only queue would otherwise lose it. A quiesced host defers it + // again, into the persisted queue, for boot recovery; a write that throws + // stays queued (writeDrained). An entry for a target no longer registered + // (a disposed fork) is left: dropped with a memory-only queue at exit, and + // logged and dropped from a persisted one at boot. if (this.deferredMessages.length > 0) { for (const name of this.agents.keys()) { - for (const msg of this.drainDeferredFor(name)) { - try { - this.addMessage(msg.participant, msg.content, msg.metadata, { - deferredWriteId: msg.id, - ...(msg.forAgent ? { forAgent: msg.forAgent } : {}), - }); - } catch (error) { - console.error(`[stop] deferred write for ${name} could not be stored: ${error instanceof Error ? error.message : String(error)}`); - } - } + for (const msg of this.drainDeferredFor(name)) this.readdDrained(msg, 'stop'); } this.ackDeferredWrites(); } @@ -3378,19 +3374,7 @@ export class AgentFramework { this.handOffDeferredWrites(flush); let stored = 0; for (const msg of flush) { - try { - this.addMessage(msg.participant, msg.content, msg.metadata, { - deferredWriteId: msg.id, - ...(msg.forAgent ? { forAgent: msg.forAgent } : {}), - }); - stored++; - } catch (err) { - console.error( - `[host-mode] ${label}: failed to store a deferred context write ` + - `(participant=${msg.participant}, forAgent=${msg.forAgent ?? 'primary'}):`, - err, - ); - } + if (this.readdDrained(msg, `host-mode ${label}`)) stored++; } // Ack = sync the chronicle FIRST, then rewrite the durable queue without // the batch. Never the other way round. @@ -4198,6 +4182,15 @@ export class AgentFramework { } } + /** A turn refused because stop() has begun: an ephemeral run waiting on it + * ends now, rather than when its startup watchdog fires. */ + private endEphemeralRunRefused(agent: Agent): void { + const run = this.ephemeralRuns.get(agent.name); + if (run && this.agents.get(agent.name) === agent) { + run.settle.reject(new Error(`Ephemeral agent "${agent.name}" refused: the framework is stopping`)); + } + } + private createDeferred(): Deferred { let resolve!: (value: T) => void; let reject!: (reason?: unknown) => void; @@ -4284,6 +4277,11 @@ export class AgentFramework { `refusing new ephemeral run for ${agent.name} — resume() first`, ); } + // Once stop() has begun no turn starts (startAgentStream), so a run + // admitted now could only wait out its startup watchdog. + if (this.providerAdmissionClosed) { + throw new Error(`framework is stopping: refusing new ephemeral run for ${agent.name}`); + } // Only a fresh object returned by createEphemeralAgent may enter this path. // Never overwrite a resident/conversation owner or a concurrent run: their // cleanup is name-keyed and could cancel/deregister the legitimate owner. @@ -5951,20 +5949,12 @@ export class AgentFramework { // Writers that deferred behind the reservation land now, on the branch // the operator chose (or the restored source) — the same end-of-turn // flush driveStream/puppet perform, under the same guard. - if (this.deferredMessages.length > 0 && this.pendingAssistantBlocks.size === 0) { + if (this.deferredMessages.length > 0) { for (const name of reserved.keys()) { - if (this.activeTurnTokens.has(name)) continue; - for (const msg of this.drainDeferredFor(name)) { - try { - this.addMessage(msg.participant, msg.content, msg.metadata, { - deferredWriteId: msg.id, - ...(msg.forAgent ? { forAgent: msg.forAgent } : {}), - }); - } catch (error) { - console.error(`[operator] post-surgery deferred flush failed for ${name}: ${error instanceof Error ? error.message : String(error)}`); - } - } + if (this.activeTurnTokens.has(name) || this.pendingAssistantBlocks.has(name)) continue; + for (const msg of this.drainDeferredFor(name)) this.readdDrained(msg, 'post-surgery flush'); } + this.ackDeferredWrites(); } }; } @@ -6744,7 +6734,12 @@ export class AgentFramework { { const deferred = this.drainDeferredFor(agent.name, (msg) => !isInjectableMidTurn(msg, agent.name)); for (const msg of deferred) { - agent.getContextManager().addMessage(msg.participant, msg.content, withDeferredWriteId(msg.metadata, msg.id)); + // One that can't be stored stays queued and isn't injected; the + // rest are, so the boundary still hands the round's results on. + const stored = this.writeDrained(msg, `tool boundary for ${agent.name}`, () => { + agent.getContextManager().addMessage(msg.participant, msg.content, withDeferredWriteId(msg.metadata, msg.id)); + }); + if (!stored) continue; midTurnInjections.push({ participant: msg.participant, content: msg.content, @@ -6958,6 +6953,14 @@ export class AgentFramework { // the turn's compile and read as unread, so a coalesced // replacement could remove a message the agent heard and // answered (and that its later replies were minted after). + // The mark is the store's head, a scalar, as a compile's is: it + // also covers what was stored since the compile without being + // carried, such as another slot's messages, and a write that + // bypassed deferral into this slot (a fresh tool_result, a + // routed channel message). That over-cover can't be left out. + // Under context-manager#160 such a write already gives the + // replies stored after it stamps their requests didn't carry, + // whatever the mark says; deferring those writers is the fix. if (midTurnInjections.length > 0) agent.markContextConsumed(); agent.setStreaming(currentState.stream); this.emitTrace({ @@ -9022,6 +9025,15 @@ export class AgentFramework { attempt = 0, providerGateAlreadyHeld = false, ): Promise { + // Once stop() has begun, no turn starts, for any agent: a fork's or a + // subagent's retry after its delay, or a wake the loop dequeued before it + // ended, would otherwise begin a turn stop() never cancels, and the + // deferred drain at stop's end would meet it live. + if (this.providerAdmissionClosed) { + if (providerGateAlreadyHeld) this.releasePrimaryProviderGate(agent.name); + this.endEphemeralRunRefused(agent); + return; + } const ownsProviderGate = !this.ephemeralRuns.has(agent.name) && !this.conversationAgentHomes.has(agent.name); if (ownsProviderGate && !providerGateAlreadyHeld) { @@ -9147,21 +9159,15 @@ export class AgentFramework { { const deferred = this.drainDeferredFor(agent.name); for (const msg of deferred) { - // Per-message try/catch (mirrors announceLocusIfChanged): one - // poison message — or a transient store-write failure — must not - // abort the turn or drop the messages behind it in the queue. - try { + // One poison message, or a transient store-write failure, must not + // abort the turn or drop the messages behind it in the queue; the + // one that failed stays queued (writeDrained). + this.writeDrained(msg, `turn-start flush for ${agent.name}`, () => { const id = agent.getContextManager().addMessage( msg.participant, msg.content, withDeferredWriteId(msg.metadata, msg.id), ); this.emitTrace({ type: 'message:added', messageId: id, source: 'deferred-flush:turn-start' }); - } catch (err) { - console.error( - `[deferred-flush] turn-start flush failed to store a message for ${agent.name} ` + - `(participant=${msg.participant}):`, - err, - ); - } + }); } this.ackDeferredWrites(); } @@ -9201,13 +9207,8 @@ export class AgentFramework { this.activeTurnTriggers.delete(agent.name); } if (!this.activeTurnTokens.has(agent.name) - && this.deferredMessages.length > 0 && this.pendingAssistantBlocks.size === 0) { - for (const msg of this.drainDeferredFor(agent.name)) { - this.addMessage(msg.participant, msg.content, msg.metadata, { - deferredWriteId: msg.id, - ...(msg.forAgent ? { forAgent: msg.forAgent } : {}), - }); - } + && this.deferredMessages.length > 0 && !this.pendingAssistantBlocks.has(agent.name)) { + for (const msg of this.drainDeferredFor(agent.name)) this.readdDrained(msg, 'withdrawn-turn flush'); this.ackDeferredWrites(); } return false; @@ -9413,11 +9414,16 @@ export class AgentFramework { } = await agent.startStreamWithInjections(tools, injections, undefined, compressionTools, { silentHeartbeat: trigger?.silentHeartbeat, }); - if (this.agents.get(agent.name) !== agent) { + // A turn that was mid-compile when stop() cancelled every stream would + // register after that pass and outlive stop(): refuse it here, as a + // replaced agent is. The caller's finally releases the turn token. + const stopping = this.providerAdmissionClosed; + if (this.agents.get(agent.name) !== agent || stopping) { stream.cancel(); agent.cancelStream(); this.channelRegistry?.stopTyping(); this.eventGate?.onInferenceEnded(agent.name); + if (stopping) this.endEphemeralRunRefused(agent); return false; } @@ -10979,17 +10985,11 @@ export class AgentFramework { // target has meanwhile started a turn). Gated on this agent's own tool // cycle, not every agent's: messages held for this turn's end depend on // this flush, and another agent's pending tools say nothing about this - // window. A write addMessage still defers (a primary-bound write waits - // out any agent's cycle) goes back into the queue under its id, for the - // target's next turn start or stop(). + // window. addMessage judges each entry by the same rule, so the drained + // prefix is stored in order, or goes back into the queue whole under + // its ids when a successor turn has already taken the agent. if (frameReachedTerminal && this.deferredMessages.length > 0 && !this.pendingAssistantBlocks.has(agent.name)) { - const deferred = this.drainDeferredFor(agent.name); - for (const msg of deferred) { - this.addMessage(msg.participant, msg.content, msg.metadata, { - deferredWriteId: msg.id, - ...(msg.forAgent ? { forAgent: msg.forAgent } : {}), - }); - } + for (const msg of this.drainDeferredFor(agent.name)) this.readdDrained(msg, 'turn-end flush'); this.ackDeferredWrites(); } } @@ -11144,6 +11144,9 @@ export class AgentFramework { { id: randomUUID(), seq: ++this.deferredSeq, participant: agentName, content: toolUse, forAgent: agentName }, { id: randomUUID(), seq: ++this.deferredSeq, participant: 'user', content: blocks, forAgent: agentName }, ); + // As addMessage's own deferral does: a crash before the next persist + // must not lose the pair from a persisted queue. + if (this.quiesced || this.deferredWritesPersisted) this.persistDeferredWrites(); } this.emitTrace({ @@ -11164,19 +11167,14 @@ export class AgentFramework { if (this.activeTurnTokens.get(agentName) === turnToken) this.activeTurnTokens.delete(agentName); // Messages deferred while we held the token land now, after the pair // — the same end-of-turn flush driveStream performs, under the same - // tool-cycle guard. (No wake is requested: the agent sees the pair, - // and anything that arrived meanwhile, on its next turn.) Keyed on - // "no turn alive", not on token ownership: if a turn did replace us - // and has already ended, its flush ran before our pair was queued — - // nobody else will flush it, so we do. + // guard: this agent's own tool cycle. (No wake is requested: the agent + // sees the pair, and anything that arrived meanwhile, on its next + // turn.) Keyed on "no turn alive", not on token ownership: if a turn + // did replace us and has already ended, its flush ran before our pair + // was queued — nobody else will flush it, so we do. if (!this.activeTurnTokens.has(agentName) - && this.deferredMessages.length > 0 && this.pendingAssistantBlocks.size === 0) { - for (const msg of this.drainDeferredFor(agentName)) { - this.addMessage(msg.participant, msg.content, msg.metadata, { - deferredWriteId: msg.id, - ...(msg.forAgent ? { forAgent: msg.forAgent } : {}), - }); - } + && this.deferredMessages.length > 0 && !this.pendingAssistantBlocks.has(agentName)) { + for (const msg of this.drainDeferredFor(agentName)) this.readdDrained(msg, 'puppet flush'); this.ackDeferredWrites(); } } @@ -12602,13 +12600,17 @@ export class AgentFramework { // without it would split the pair. (A turn-end flush can run after a // successor turn has taken the agent: the 2026-07-31 interleave.) const freshToolResult = opts?.deferredWriteId === undefined && content.some(b => b.type === 'tool_result'); - // Explicitly-targeted deliveries scope the tool-cycle check to the - // target (pendingAssistantBlocks is keyed by agent); the default path - // keeps the historical GLOBAL check byte-for-byte (conservative: any - // agent's pending cycle defers primary-bound messages). - const midToolCycle = opts?.forAgent - ? this.pendingAssistantBlocks.has(agent.name) - : this.pendingAssistantBlocks.size > 0; + // One rule for every write, primary-bound or targeted: it waits while ITS + // TARGET's own tool cycle is pending (pendingAssistantBlocks is keyed by + // agent). Another agent's cycle says nothing about this window. The + // default path used to check every agent's cycle (efc7e7af, from when + // all agents shared one message store), which held the primary's writes + // behind a fork's, subagent's or the subconscious's tools, and judged a + // drained queue by two rules, so a later targeted entry could land ahead + // of an earlier primary-bound one. Residents that share the un-namespaced + // slot with overlapping turns put each other's writes inside the other's + // live turn whatever this check does: the limit #286 states. + const midToolCycle = this.pendingAssistantBlocks.has(agent.name); if ( !freshToolResult && opts?.bypassDeferralFor !== agent.name && @@ -12649,15 +12651,7 @@ export class AgentFramework { * message and every later one stay queued, so what is drained is always a * prefix of the agent's queue. */ - private drainDeferredFor(agentName: string, stopAt?: (msg: DeferredWrite) => boolean): Array<{ - /** Durable identity — every write of this entry must carry it - * (`withDeferredWriteId` / `opts.deferredWriteId`). */ - id: string; - participant: string; - content: ContentBlock[]; - metadata?: MessageMetadata; - forAgent?: string; - }> { + private drainDeferredFor(agentName: string, stopAt?: (msg: DeferredWrite) => boolean): DeferredWrite[] { if (this.deferredMessages.length === 0) return []; const mine: typeof this.deferredMessages = []; const rest: typeof this.deferredMessages = []; @@ -12677,6 +12671,41 @@ export class AgentFramework { return drained; } + /** + * Write one drained entry. If the write throws, the entry goes back into + * the queue under its id and seq, and the caller goes on with the rest: + * left in the un-acked set, the next ack would drop it from the durable + * queue, and a memory-only queue would lose it at once. Every site that + * writes a drained entry writes through this. Returns whether it was + * written (stored, or deferred again by addMessage). + */ + private writeDrained(msg: DeferredWrite, where: string, write: () => void): boolean { + try { + write(); + return true; + } catch (error) { + console.error( + `[deferral] ${where}: a deferred write for ${msg.forAgent ?? this.primaryAgentName ?? 'the primary'} ` + + `(participant=${msg.participant}) could not be stored; it stays queued: ` + + `${error instanceof Error ? error.message : String(error)}`, + ); + this.unackedDeferredWrites = this.unackedDeferredWrites.filter((m) => m.id !== msg.id); + if (!this.deferredMessages.some((m) => m.id === msg.id)) this.deferredMessages.push(msg); + if (this.quiesced || this.deferredWritesPersisted) this.persistDeferredWrites(); + return false; + } + } + + /** Re-add one drained entry through addMessage, which stores it or defers it again. */ + private readdDrained(msg: DeferredWrite, where: string): boolean { + return this.writeDrained(msg, where, () => { + this.addMessage(msg.participant, msg.content, msg.metadata, { + deferredWriteId: msg.id, + ...(msg.forAgent ? { forAgent: msg.forAgent } : {}), + }); + }); + } + private editMessage(id: MessageId, content: ContentBlock[]): void { const agent = this.primaryAgentName ? this.agents.get(this.primaryAgentName) diff --git a/test/deferred-window-only.test.ts b/test/deferred-window-only.test.ts index 246293027..e6d93a526 100644 --- a/test/deferred-window-only.test.ts +++ b/test/deferred-window-only.test.ts @@ -65,9 +65,17 @@ class InterjectingModule implements Module { for (const interjection of pending) { this.framework!.pushEvent({ type: 'interjection', source: 'test', interjection } as unknown as ProcessEvent); } - // Let the run loop handle them while this round's tools are pending, so - // they land in the deferred queue before the tool-result boundary. - if (pending.length > 0) await new Promise((r) => setTimeout(r, 30)); + // Let the run loop handle them while this round's tools are pending, and + // return only once each is in the deferred queue, so the tool-result + // boundary meets them there (bounded: a run that never queues them fails). + if (pending.length > 0) { + const queue = () => (this.framework as unknown as { deferredMessages: unknown[] }).deferredMessages.length; + const end = Date.now() + 5000; + while (queue() < pending.length) { + if (Date.now() > end) throw new Error(`only ${queue()} of ${pending.length} interjection(s) were queued`); + await new Promise((r) => setTimeout(r, 5)); + } + } const during = this.duringTool; this.duringTool = null; during?.(); @@ -129,6 +137,7 @@ describe('mid-turn deliveries keep the window what the stream carried', () => { } /** A turn of two rounds: `tool` with the interjections, then a reply. */ + let drains: Array<{ atBoundary: boolean; queued: string[] }> = []; async function runTurn(tool: 'move' | 'halt', interjections: Interjection[]): Promise { membrane.pushResponse(createMockResponse([ { type: 'text', text: 'Moving.' }, @@ -136,12 +145,37 @@ describe('mid-turn deliveries keep the window what the stream carried', () => { ] as ContentBlock[], 'tool_use')); membrane.pushResponse(createMockResponse(text('Done.'))); const framework = await createFramework(); + drains = recordDrains(framework); module.interjections = interjections; framework.pushEvent({ type: 'external-message', source: 'test', content: 'go' } as unknown as ProcessEvent); await framework.runUntilIdle(); return framework; } + /** What the tool boundary's drain found queued. */ + const atBoundary = () => drains.find((d) => d.atBoundary)?.queued; + + /** What each drain for the agent found queued, in order, and whether it + * stopped at a held message (the tool boundary's drain does). */ + function recordDrains(framework: AgentFramework): Array<{ atBoundary: boolean; queued: string[] }> { + const drains: Array<{ atBoundary: boolean; queued: string[] }> = []; + const internals = framework as unknown as { + deferredMessages: Array<{ participant: string; content: ContentBlock[] }>; + drainDeferredFor: (name: string, stopAt?: unknown) => unknown[]; + }; + const drain = internals.drainDeferredFor.bind(framework); + internals.drainDeferredFor = (name: string, stopAt?: unknown) => { + if (name === 'assistant' && internals.deferredMessages.length > 0) { + drains.push({ + atBoundary: stopAt !== undefined, + queued: internals.deferredMessages.map((m) => `${m.participant}: ${m.content.map((b) => (b.type === 'text' ? (b as { text: string }).text : `<${b.type}>`)).join(' ')}`), + }); + } + return drain(name, stopAt); + }; + return drains; + } + function injected(): string[] { const options = membrane.lastStream!.receivedToolResultOptions[0]; return (options?.injectedMessages ?? []).map((m) => @@ -150,6 +184,7 @@ describe('mid-turn deliveries keep the window what the stream carried', () => { it('a message named as the agent waits for the turn to end, after its last round', async () => { const framework = await runTurn('move', [{ participant: 'assistant', content: text('my own words') }]); + assert.deepEqual(atBoundary(), ['assistant: my own words'], 'it was queued when the boundary drained'); assert.deepEqual(injected(), []); assert.deepEqual(windowOf(framework), [ 'Antra: go', @@ -176,6 +211,7 @@ describe('mid-turn deliveries keep the window what the stream carried', () => { { participant: 'assistant', content: text('my own words') }, { participant: 'Antra', content: text('look left') }, ]); + assert.deepEqual(atBoundary(), ['assistant: my own words', 'Antra: look left'], 'both were queued when the boundary drained'); assert.deepEqual(injected(), []); assert.deepEqual(windowOf(framework).slice(2), [ 'user: ', @@ -191,6 +227,7 @@ describe('mid-turn deliveries keep the window what the stream carried', () => { { participant: 'Antra', content: text('look left') }, { participant: 'assistant', content: text('my own words') }, ]); + assert.deepEqual(atBoundary(), ['Antra: look left', 'assistant: my own words'], 'both were queued when the boundary drained'); assert.deepEqual(injected(), ['Antra: look left']); assert.deepEqual(windowOf(framework).slice(2), [ 'user: ', @@ -226,14 +263,19 @@ describe('mid-turn deliveries keep the window what the stream carried', () => { await framework.stop(); }); - /** The framework's own queue and its record of pending tool cycles. */ + /** The framework's own queue and its record of turns and tool cycles. */ function internalsOf(framework: AgentFramework) { return framework as unknown as { pendingAssistantBlocks: Map; - deferredMessages: Array<{ id: string }>; - addMessage(participant: string, content: ContentBlock[], metadata?: undefined, opts?: { forAgent?: string }): string; + activeTurnTokens: Map; + activeStreams: Map; + deferredMessages: Array<{ id: string; participant: string }>; + deferredWritesPersisted: boolean; + drainDeferredFor: (name: string, stopAt?: unknown) => unknown[]; + addMessage(participant: string, content: ContentBlock[], metadata?: undefined, opts?: { forAgent?: string; deferredWriteId?: string }): string; }; } + const otherCycle = (): ContentBlock[] => [{ type: 'tool_use', id: 'b1', name: 'robot--move', input: {} } as ContentBlock]; it('a message held for the turn\'s end lands then, while another agent\'s tool cycle is pending', async () => { membrane.pushResponse(createMockResponse([ @@ -243,43 +285,256 @@ describe('mid-turn deliveries keep the window what the stream carried', () => { membrane.pushResponse(createMockResponse(text('Done.'))); const framework = await createFramework(); const internals = internalsOf(framework); + // Whether the other cycle was pending at the turn-end flush itself. + const pendingAtFlush: boolean[] = []; + const drain = internals.drainDeferredFor.bind(framework); + internals.drainDeferredFor = (name: string, stopAt?: unknown) => { + if (name === 'assistant' && stopAt === undefined && internals.deferredMessages.length > 0) { + pendingAtFlush.push(internals.pendingAssistantBlocks.has('builder')); + } + return drain(name, stopAt); + }; module.duringTool = () => { // Another agent is mid tool cycle when this turn ends, and a write // named as this agent (held: it can't join the live stream) waits. - internals.pendingAssistantBlocks.set('builder', [{ type: 'tool_use', id: 'b1', name: 'robot--move', input: {} } as ContentBlock]); + internals.pendingAssistantBlocks.set('builder', otherCycle()); assert.equal(internals.addMessage('assistant', text('my own words'), undefined, { forAgent: 'assistant' }), '', 'deferred'); }; framework.pushEvent({ type: 'external-message', source: 'test', content: 'go' } as unknown as ProcessEvent); await framework.runUntilIdle(); - assert.ok(internals.pendingAssistantBlocks.has('builder'), 'the other cycle was still pending at the flush'); + assert.deepEqual(pendingAtFlush, [true], 'the other cycle was pending at the turn-end flush'); assert.deepEqual(windowOf(framework).slice(-2), ['assistant: Done.', 'assistant: my own words']); internals.pendingAssistantBlocks.delete('builder'); await framework.stop(); }); - it('a message still deferred when the framework stops lands before the store closes', async () => { + it("a write waits on its target's own turn and tool cycle, not another agent's, primary-bound or targeted", async () => { const framework = await createFramework(); const internals = internalsOf(framework); - // A primary-bound write waits out any agent's tool cycle, even with this - // agent idle; that cycle's stream then ends without delivering it. - internals.pendingAssistantBlocks.set('builder', [{ type: 'tool_use', id: 'b1', name: 'robot--move', input: {} } as ContentBlock]); - assert.equal(internals.addMessage('Antra', text('late word')), '', 'deferred'); - internals.pendingAssistantBlocks.delete('builder'); - assert.equal(internals.deferredMessages.length, 1); + internals.pendingAssistantBlocks.set('builder', otherCycle()); + try { + assert.notEqual(internals.addMessage('Antra', text('primary-bound')), '', 'stored: the primary is idle'); + assert.notEqual(internals.addMessage('Antra', text('targeted'), undefined, { forAgent: 'assistant' }), '', 'stored too'); + internals.pendingAssistantBlocks.set('assistant', otherCycle()); + assert.equal(internals.addMessage('Antra', text('in its own cycle')), '', "deferred: the primary's own cycle is pending"); + } finally { + internals.pendingAssistantBlocks.delete('builder'); + internals.pendingAssistantBlocks.delete('assistant'); + } await framework.stop(); + }); + + it("the puppet's flush lands what was deferred behind it while another agent's tool cycle is pending", async () => { + const framework = await createFramework(); + const internals = internalsOf(framework); + module.duringTool = () => { + internals.pendingAssistantBlocks.set('builder', otherCycle()); + assert.equal(internals.addMessage('Antra', text('while puppeted')), '', 'deferred behind the puppet'); + }; + try { + await framework.puppetToolCall('assistant', 'robot--move', {}); + assert.equal(internals.deferredMessages.length, 0); + assert.deepEqual(windowOf(framework).slice(-3), ['assistant: ', 'user: ', 'Antra: while puppeted']); + } finally { + internals.pendingAssistantBlocks.delete('builder'); + } + await framework.stop(); + }); + + it('a turn mid-compile when the framework stops never starts a stream, and what was deferred for it is stored', async () => { + membrane.pushResponse(createMockResponse(text('too late'))); + const framework = await createFramework(); + const internals = internalsOf(framework); + const agent = framework.getAgent('assistant')!; + let compiling!: () => void; + const inCompile = new Promise((resolve) => { compiling = resolve; }); + let release!: () => void; + const released = new Promise((resolve) => { release = resolve; }); + const start = agent.startStreamWithInjections.bind(agent); + (agent as unknown as { startStreamWithInjections: typeof start }).startStreamWithInjections = async (...args) => { + compiling(); + await released; + return start(...args); + }; + // The event loop starts the turn, as in a host (stop() awaits the loop). + framework.start(); + framework.pushEvent({ type: 'external-message', source: 'test', content: 'go' } as unknown as ProcessEvent); + await inCompile; + assert.equal(internals.addMessage('Antra', text('late word')), '', 'deferred: the turn is alive'); + const stopped = framework.stop(); + release(); + await stopped; + assert.equal(internals.activeStreams.size, 0, 'no stream outlived stop()'); const reopened = await createFramework(); assert.ok(windowOf(reopened).includes('Antra: late word'), 'stored before the store closed'); + assert.ok(!windowOf(reopened).includes('assistant: too late'), 'the refused turn produced nothing'); await reopened.stop(); }); - it('a write coming back from the deferred queue re-defers while a turn is alive, tool_result or not', async () => { + it('a drained write whose store throws stays in the persisted queue, and is stored on the next boot', async () => { const framework = await createFramework(); - const internals = framework as unknown as { - activeTurnTokens: Map; - deferredMessages: Array<{ id: string }>; - addMessage: (p: string, c: ContentBlock[], m?: unknown, o?: { deferredWriteId?: string }) => string; + const internals = internalsOf(framework); + internals.deferredWritesPersisted = true; + internals.activeTurnTokens.set('assistant', 1); + assert.equal(internals.addMessage('Antra', text('kept')), '', 'deferred'); + internals.activeTurnTokens.delete('assistant'); + const cm = framework.getAgent('assistant')!.getContextManager(); + const add = cm.addMessage.bind(cm); + (cm as unknown as { addMessage: typeof add }).addMessage = (...args) => { + if ((args[1] as Array<{ text?: string }>).some((b) => b.text === 'kept')) throw new Error('disk full'); + return add(...args); }; + const errors: string[] = []; + const error = console.error; + console.error = (...args: unknown[]) => { errors.push(args.map(String).join(' ')); }; + try { await framework.stop(); } finally { console.error = error; } + assert.ok(errors.some((e) => e.includes('could not be stored; it stays queued') && e.includes('disk full')), errors.join('\n')); + assert.equal(internals.deferredMessages.length, 1, 'still queued'); + + const reopened = await createFramework(); + assert.ok(windowOf(reopened).includes('Antra: kept'), 'boot recovery stored it'); + await reopened.stop(); + }); + + it('at a tool boundary, a write whose store throws stays queued and is not injected; the rest are', async () => { + membrane.pushResponse(createMockResponse([ + { type: 'text', text: 'Moving.' }, + { type: 'tool_use', id: 'c1', name: 'robot--move', input: {} }, + ] as ContentBlock[], 'tool_use')); + membrane.pushResponse(createMockResponse(text('Done.'))); + const framework = await createFramework(); + const cm = framework.getAgent('assistant')!.getContextManager(); + const add = cm.addMessage.bind(cm); + let failOnce = true; + (cm as unknown as { addMessage: typeof add }).addMessage = (...args) => { + if (failOnce && (args[1] as Array<{ text?: string }>).some((b) => b.text === 'look left')) { + failOnce = false; + throw new Error('transient'); + } + return add(...args); + }; + module.interjections = [ + { participant: 'Antra', content: text('look left') }, + { participant: 'Antra', content: text('look right') }, + ]; + const error = console.error; + console.error = () => {}; + try { + framework.pushEvent({ type: 'external-message', source: 'test', content: 'go' } as unknown as ProcessEvent); + await framework.runUntilIdle(); + } finally { console.error = error; } + assert.deepEqual(injected(), ['Antra: look right']); + assert.deepEqual(windowOf(framework).slice(2), [ + 'user: ', + 'Antra: look right', + 'assistant: Done.', + 'Antra: look left', + ], 'the one that failed was kept, and stored at the turn end'); + await framework.stop(); + }); + + it('a message stored straight into the context during a round, with nothing injected at the boundary, stays unread', async () => { + membrane.pushResponse(createMockResponse([ + { type: 'text', text: 'Moving.' }, + { type: 'tool_use', id: 'c1', name: 'robot--move', input: {} }, + ] as ContentBlock[], 'tool_use')); + membrane.pushResponse(createMockResponse(text('Done.'))); + const framework = await createFramework(); + const agent = framework.getAgent('assistant')!; + let bypassed = ''; + module.duringTool = () => { + // A writer that bypasses deferral (an older route): the stream never carried it. + bypassed = agent.getContextManager().addMessage('Antra', text('bypassed')); + }; + framework.pushEvent({ type: 'external-message', source: 'test', content: 'go' } as unknown as ProcessEvent); + await framework.runUntilIdle(); + assert.deepEqual(injected(), []); + const isUnread = (framework as unknown as { + isUnreadStoredMessage: (a: typeof agent, id: string) => boolean; + }).isUnreadStoredMessage.bind(framework); + assert.equal(isUnread(agent, bypassed), true, 'a boundary that injected nothing marks nothing read'); + await framework.stop(); + }); + + it("a compile's watermark never rolls back a mark made while it ran", async () => { + const framework = await createFramework(); + const agent = framework.getAgent('assistant')!; + const cm = agent.getContextManager(); + cm.addMessage('Antra', text('before')); + const compile = cm.compile.bind(cm); + let release!: () => void; + const released = new Promise((resolve) => { release = resolve; }); + (cm as unknown as { compile: typeof compile }).compile = async (...args) => { + await released; + return compile(...args); + }; + const compiling = agent.compileContext(); + cm.addMessage('Antra', text('heard at a boundary')); + agent.markContextConsumed(); + const marked = agent.getConsumedWatermark(); + release(); + await compiling; + assert.deepEqual(agent.getConsumedWatermark(), marked, "the boundary's mark stands"); + (cm as unknown as { compile: typeof compile }).compile = compile; + await agent.compileContext(); + assert.ok(agent.getConsumedWatermark()!.sequence >= marked!.sequence, 'a later compile still advances it'); + await framework.stop(); + }); + + it("a held message waits out a successor turn's tool cycle, and lands once it is over", async () => { + membrane.pushResponse(createMockResponse([ + { type: 'text', text: 'Moving.' }, + { type: 'tool_use', id: 'c1', name: 'robot--move', input: {} }, + ] as ContentBlock[], 'tool_use')); + membrane.pushResponse(createMockResponse(text('Done.'))); + const framework = await createFramework(); + const internals = internalsOf(framework) as ReturnType & { releasePrimaryProviderGate: (name: string) => void }; + const agent = framework.getAgent('assistant')!; + module.duringTool = () => { + assert.equal(internals.addMessage('assistant', text('my own words'), undefined, { forAgent: 'assistant' }), '', 'held'); + }; + // As the turn's stream completes, a successor turn takes the agent and is + // mid tool cycle (the 2026-07-31 interleave): the stream generation, the + // turn token and the cycle are the successor's when the first turn's + // teardown flushes. + let completed = false; + const off = framework.onTrace((event) => { + if ((event as { type: string }).type !== 'inference:completed') return; + off(); + completed = true; + (agent as unknown as { _streamId: number })._streamId++; + internals.activeTurnTokens.set('assistant', 1_000_000); + internals.pendingAssistantBlocks.set('assistant', otherCycle()); + }); + // The first turn's teardown releases its provider gate, then flushes, + // synchronously: once the release is seen, the flush has run. + let tornDown!: () => void; + const teardown = new Promise((resolve) => { tornDown = resolve; }); + const release = internals.releasePrimaryProviderGate.bind(framework); + internals.releasePrimaryProviderGate = (name: string) => { + release(name); + if (completed && name === 'assistant') tornDown(); + }; + framework.start(); + framework.pushEvent({ type: 'external-message', source: 'test', content: 'go' } as unknown as ProcessEvent); + await teardown; + await new Promise((resolve) => setImmediate(resolve)); + assert.ok(!windowOf(framework).includes('assistant: my own words'), "not stored inside the successor's cycle"); + assert.ok(internals.deferredMessages.some((m) => m.participant === 'assistant'), 'still queued'); + // The successor's cycle and turn end; what is still queued lands (here at stop's drain). + internals.pendingAssistantBlocks.delete('assistant'); + internals.activeTurnTokens.delete('assistant'); + internals.activeStreams.delete('assistant'); + await framework.stop(); + const reopened = await createFramework(); + assert.equal(windowOf(reopened).at(-1), 'assistant: my own words'); + await reopened.stop(); + }); + + it('a write coming back from the deferred queue re-defers while a turn is alive, tool_result or not', async () => { + const framework = await createFramework(); + const internals = internalsOf(framework); internals.activeTurnTokens.set('assistant', 1); try { // A queued pair's tool_result must follow its tool_use back into the @@ -290,6 +545,9 @@ describe('mid-turn deliveries keep the window what the stream carried', () => { ], undefined, { deferredWriteId: 'w1' }); assert.equal(requeued, ''); assert.ok(internals.deferredMessages.some((m) => m.id === 'w1')); + // So does plain text. + assert.equal(internals.addMessage('Antra', text('plain'), undefined, { deferredWriteId: 'w2' }), ''); + assert.ok(internals.deferredMessages.some((m) => m.id === 'w2')); // A fresh tool_result write keeps landing as before. const fresh = internals.addMessage('user', [ { type: 'tool_result', toolUseId: 'p2', content: 'ok' } as ContentBlock, diff --git a/test/host-quiesce.test.ts b/test/host-quiesce.test.ts index cae60190e..c0a08f611 100644 --- a/test/host-quiesce.test.ts +++ b/test/host-quiesce.test.ts @@ -683,7 +683,7 @@ test('a batch carrying a budget restart re-parks its sibling wakes instead of co }); }); -test('resume flushes deferred writes per message: one poison write neither drops the rest nor skips the reopen', async () => { +test('resume flushes deferred writes per message: one poison write neither drops the rest nor skips the reopen, and stays queued', async () => { const membrane = new MockMembrane(); await withFramework(membrane, async (framework) => { const traces: TraceEvent[] = []; @@ -709,7 +709,9 @@ test('resume flushes deferred writes per message: one poison write neither drops const status = await framework.resume(); assert.equal(status.quiesced, false); - assert.equal(status.deferredWrites, 0, 'the queue is drained even though one write failed'); + // A drained write whose store throws is never dropped: it goes back into + // the queue under its id, for the next flush. + assert.equal(status.deferredWrites, 1, 'only the write that failed is still queued'); assert.deepEqual( cm.getAllMessages().slice(before).map((m) => (m.content[0] as { text: string }).text), ['first', 'third'], From 1e4f6b71a0b251f7b5e277965ade4646bb412860 Mon Sep 17 00:00:00 2001 From: Duncan-2410 Date: Tue, 13 Oct 2026 09:11:43 -0400 Subject: [PATCH 04/34] docs(deferral): say where isInjectableMidTurn's exact name check comes from The prefill guard compares a deferred message's participant with the agent's name exactly. That matches the renderer's own test: membrane gives a message the assistant role only when its participant equals the request's assistantParticipant, which Agent sets to its name, so a differently spelled name renders as another speaker rather than as a prefill. A comment at the check now says so. Co-Authored-By: Claude Opus 5.5 --- src/framework.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/framework.ts b/src/framework.ts index 083c11e1e..21b4a03be 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -485,6 +485,8 @@ function withDeferredWriteId(metadata: MessageMetadata | undefined, id: string): * no tool blocks (the membrane's tool cycle) and not named as the agent * itself (it would render as an assistant turn, a prefill). */ function isInjectableMidTurn(msg: { participant: string; content: ContentBlock[] }, agentName: string): boolean { + // Exact, as membrane's role test is: only a participant equal to the + // request's assistantParticipant (Agent's name) renders as the assistant. return msg.participant !== agentName && !msg.content.some((b) => b.type === 'tool_use' || b.type === 'tool_result'); } From 1e95eb21380698df6d484b0fac674ea3281577e3 Mon Sep 17 00:00:00 2001 From: Duncan-2410 Date: Tue, 13 Oct 2026 10:06:45 -0400 Subject: [PATCH 05/34] fix(deferral): a failed write cuts the drain; stop() stores past the deferral unless quiesced An internal review of the last round, after main was merged in, found two places where its rules didn't hold. A failed write didn't cut the drain. writeDrained put the entry that threw back into the queue and every caller went on to the next one: stop, host-mode resume and boot recovery, the post-surgery, turn-end, withdrawn-turn and puppet flushes, and the tool boundary. So the failed entry landed after entries queued behind it, and a tool_use that threw left its tool_result to be stored alone. The drain now cuts there, through one helper every site uses (writeDrainedInOrder): the entry that threw and every later entry for that agent go back into the queue under their ids and seq, and the tool boundary injects only what was stored before the cut. Host-mode resume and boot recovery cut per target, so one agent's stuck entry doesn't hold another's. An entry that keeps throwing holds its agent's later deferred writes, and the line each flush logs names it (id, participant, block types). stop()'s drain could meet a live turn token. It drained while a frame that stop() doesn't wait for (a puppet call, or a turn started outside the event loop) still held its target's token, so addMessage deferred each entry again, into memory, where a memory-only queue lost it at exit. Once stop() has begun no stream can register, so no reply can follow a stored entry: the drain now writes past the deferral (bypassDeferralFor) unless the host is quiesced, whose entries are re-deferred into the persisted queue as before. Text: the host-mode flush's doc no longer says a rejected write is acked and dropped; the watermark docs no longer claim a branch switch marks the watermark (no branch switch calls markContextConsumed); the fragment says what a failed write now does, and that a memory-only queue still loses what is queued when the process exits. Tests: - at a tool boundary, a write that throws cuts the drain: neither it nor what follows is injected, and both land at the turn end in order (this replaces the test that pinned the old reorder); - a queued tool_use whose store throws at stop() keeps its tool_result from being stored alone: both stay queued, in order, and boot recovery stores them as a pair; - stop() stores what is deferred for an agent whose turn token is still held by a frame it doesn't wait for; - a turn waiting to retry when the framework stops gives up rather than compile again, and what was deferred for it is stored; - an ephemeral run whose turn stop() refuses ends at once with "the framework is stopping", and one asked for once stop() has begun is refused; - the post-surgery flush lands what waited behind the hold while another agent's tool cycle is pending, and acknowledges it; - host-quiesce's poison-write test now pins that the write behind the poison one stays queued with it, while the one before it lands. Co-Authored-By: Claude Opus 5.5 --- .../mid-turn-window-only-held.fixed.md | 23 ++- src/agent.ts | 7 +- src/framework.ts | 158 ++++++++++++------ test/deferred-window-only.test.ts | 91 +++++++++- test/ephemeral-settle.test.ts | 43 +++++ test/host-quiesce.test.ts | 13 +- test/operator-surgery.test.ts | 31 ++++ 7 files changed, 289 insertions(+), 77 deletions(-) diff --git a/changelog.d/mid-turn-window-only-held.fixed.md b/changelog.d/mid-turn-window-only-held.fixed.md index 0b741f96b..ff15b2e87 100644 --- a/changelog.d/mid-turn-window-only-held.fixed.md +++ b/changelog.d/mid-turn-window-only-held.fixed.md @@ -34,15 +34,22 @@ cycle holds them. - `stop()` now stores what is still deferred for each registered agent before the store closes; before, a memory-only queue lost it. Once `stop()` - begins, no turn starts and no stream registers, for any agent, so nothing - is mid-turn when it does: a turn that was mid-compile when the streams - were cancelled is refused rather than left running, and an ephemeral run - refused this way ends at once rather than at its startup watchdog. A + begins, no turn starts and no stream registers, for any agent: a turn that + was mid-compile when the streams were cancelled, or one waiting to retry, + is refused rather than left running, and an ephemeral run refused this way + ends at once rather than at its startup watchdog. Since no reply can then + follow a stored entry, `stop()` stores each one even while a frame it + doesn't wait for, such as a puppet call, still holds its target's turn. A quiesced host keeps what is deferred in its persisted queue for boot recovery. An entry for a target no longer registered (a disposed fork) is dropped from a memory-only queue at exit, as before. - A deferred write whose store throws goes back into the queue under its - id, and the others continue. Before, it stayed in the un-acked set and the - next acknowledgement dropped it from the persisted queue (at a turn start - or a flush it was lost even from memory). At a tool boundary, such a write - isn't injected either. + id, and so does every entry queued behind it for that agent, so none lands + out of order and a queued `tool_use` and its `tool_result` stay together. + The log line names the entry, and one that keeps throwing holds the + agent's later deferred writes until it can be stored. Before, it stayed in + the un-acked set and the next acknowledgement dropped it from the + persisted queue (at a turn start or a flush it was lost even from memory). + At a tool boundary, nothing from it on is injected. While the framework + runs it is kept; a memory-only queue still loses it when the process + exits. diff --git a/src/agent.ts b/src/agent.ts index 57a4d1a24..da66b1f47 100644 --- a/src/agent.ts +++ b/src/agent.ts @@ -561,15 +561,16 @@ export class Agent { */ private consumedWatermark: { branch: string; sequence: number } | null = null; /** Counts marks: a compile writes its watermark back only if no mark was - * made while it ran (a tool boundary's, a branch switch's). */ + * made while it ran (a tool boundary's, or another compile's). */ private watermarkMarks = 0; getConsumedWatermark(): { branch: string; sequence: number } | null { return this.consumedWatermark; } - /** Mark everything currently stored as consumed (boot, branch switch, and a - * tool boundary that injected messages into the live stream). */ + /** Mark everything currently stored as consumed (the agent's creation, at + * boot or later, and a tool boundary that injected messages into the live + * stream). */ markContextConsumed(): void { // Tolerates partial context-manager doubles (tests build Agents over // stubs): without a watermark nothing is ever treated as unread. diff --git a/src/framework.ts b/src/framework.ts index 21b4a03be..e05481119 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -481,6 +481,11 @@ function withDeferredWriteId(metadata: MessageMetadata | undefined, id: string): return { ...(metadata ?? {}), deferredWriteId: id } as MessageMetadata; } +/** A deferred entry as a log line names it: its id, participant and block types. */ +function describeDeferredWrite(msg: { id: string; participant: string; content: ContentBlock[] }): string { + return `${msg.id} (${msg.participant}: ${[...new Set(msg.content.map((b) => b.type))].join('+') || 'empty'})`; +} + /** Whether a deferred message can join a live stream at a tool boundary: * no tool blocks (the membrane's tool cycle) and not named as the agent * itself (it would render as an assistant turn, a prefill). */ @@ -2272,21 +2277,25 @@ export class AgentFramework { } } - // Every turn is over: from the start of stop() no turn starts and no - // stream registers (providerAdmissionClosed, checked for every agent), - // and every stream registered before was cancelled and awaited above. So - // nothing else will deliver what is still deferred: a message held for - // a turn's end whose flush found the target busy again, or one that - // arrived for an idle agent and was deferred behind its own state. Land - // each at its registered target now, while the store is open; a - // memory-only queue would otherwise lose it. A quiesced host defers it - // again, into the persisted queue, for boot recovery; a write that throws - // stays queued (writeDrained). An entry for a target no longer registered - // (a disposed fork) is left: dropped with a memory-only queue at exit, and - // logged and dropped from a persisted one at boot. + // No stream is left: from the start of stop() none registers and no turn + // starts (providerAdmissionClosed, checked for every agent), and every + // stream registered before was cancelled and awaited above. So nothing + // else will deliver what is still deferred: a message held for a turn's + // end whose flush found the target busy again, or one that arrived for an + // idle agent and was deferred behind its own state. Land each at its + // registered target now, while the store is open; a memory-only queue + // would otherwise lose it at exit. It goes past the deferral: a frame + // stop() doesn't wait for (a puppet call, or a turn started outside the + // event loop) can still hold its target's turn token, but no reply can + // follow the entry in any stream. A quiesced host defers it again + // instead, into the persisted queue, for boot recovery. A write that + // throws stays queued with what follows it (writeDrainedInOrder), and a + // memory-only queue still loses it at exit. An entry for a target no + // longer registered (a disposed fork) is left: dropped with a memory-only + // queue at exit, and logged and dropped from a persisted one at boot. if (this.deferredMessages.length > 0) { for (const name of this.agents.keys()) { - for (const msg of this.drainDeferredFor(name)) this.readdDrained(msg, 'stop'); + this.readdDrained(this.drainDeferredFor(name), 'stop', this.quiesced ? undefined : { bypassDeferralFor: name }); } this.ackDeferredWrites(); } @@ -3654,8 +3663,9 @@ export class AgentFramework { * acknowledging each durably as it lands (the persisted queue is rewritten * after every message). A crash mid-flush therefore replays exactly the * un-acked remainder at the next boot — no loss, no duplicates. A write - * the store REJECTS is logged and acked too: replaying a poison message - * forever would wedge every later boot on it. + * the store rejects stays queued, logged, with what is queued behind it for + * its target (writeDrainedInOrder), so a target's entries never land out of + * order; a boot that meets it again logs it and goes on. * * Per-target: an agent whose turn is still alive keeps ITS messages * deferred for its own turn boundary; everyone else's flush now. @@ -3678,10 +3688,15 @@ export class AgentFramework { flush.sort(bySeq); this.deferredMessages = keep; this.handOffDeferredWrites(flush); - let stored = 0; + // Each target's entries in queue order, cut at the first that can't be + // stored; other targets' entries go on. + const byTarget = new Map(); for (const msg of flush) { - if (this.readdDrained(msg, `host-mode ${label}`)) stored++; + const target = msg.forAgent ?? this.primaryAgentName; + byTarget.set(target, [...(byTarget.get(target) ?? []), msg]); } + let stored = 0; + for (const entries of byTarget.values()) stored += this.readdDrained(entries, `host-mode ${label}`).length; // Ack = sync the chronicle FIRST, then rewrite the durable queue without // the batch. Never the other way round. this.ackDeferredWrites(); @@ -6270,7 +6285,7 @@ export class AgentFramework { if (this.deferredMessages.length > 0) { for (const name of reserved.keys()) { if (this.activeTurnTokens.has(name) || this.pendingAssistantBlocks.has(name)) continue; - for (const msg of this.drainDeferredFor(name)) this.readdDrained(msg, 'post-surgery flush'); + this.readdDrained(this.drainDeferredFor(name), 'post-surgery flush'); } this.ackDeferredWrites(); } @@ -7089,13 +7104,13 @@ export class AgentFramework { const midTurnInjections: Array<{ participant: string; content: ContentBlock[]; metadata?: MessageMetadata }> = []; { const deferred = this.drainDeferredFor(agent.name, (msg) => !isInjectableMidTurn(msg, agent.name)); - for (const msg of deferred) { - // One that can't be stored stays queued and isn't injected; the - // rest are, so the boundary still hands the round's results on. - const stored = this.writeDrained(msg, `tool boundary for ${agent.name}`, () => { - agent.getContextManager().addMessage(msg.participant, msg.content, withDeferredWriteId(msg.metadata, msg.id)); - }); - if (!stored) continue; + // Stored in queue order, cut at the first that can't be stored: it + // and what follows wait for the turn's end, and only what was + // stored is injected (the round's results go on either way). + const stored = this.writeDrainedInOrder(deferred, `tool boundary for ${agent.name}`, (msg) => { + agent.getContextManager().addMessage(msg.participant, msg.content, withDeferredWriteId(msg.metadata, msg.id)); + }); + for (const msg of stored) { midTurnInjections.push({ participant: msg.participant, content: msg.content, @@ -9603,18 +9618,16 @@ export class AgentFramework { this.deferredMessages.length > 0 ) { { - const deferred = this.drainDeferredFor(agent.name); - for (const msg of deferred) { - // One poison message, or a transient store-write failure, must not - // abort the turn or drop the messages behind it in the queue; the - // one that failed stays queued (writeDrained). - this.writeDrained(msg, `turn-start flush for ${agent.name}`, () => { - const id = agent.getContextManager().addMessage( - msg.participant, msg.content, withDeferredWriteId(msg.metadata, msg.id), - ); - this.emitTrace({ type: 'message:added', messageId: id, source: 'deferred-flush:turn-start' }); - }); - } + // One poison message, or a transient store-write failure, must not + // abort the turn or drop the messages behind it in the queue: the + // drain cuts there, and it and what follows stay queued, in order + // (writeDrainedInOrder). + this.writeDrainedInOrder(this.drainDeferredFor(agent.name), `turn-start flush for ${agent.name}`, (msg) => { + const id = agent.getContextManager().addMessage( + msg.participant, msg.content, withDeferredWriteId(msg.metadata, msg.id), + ); + this.emitTrace({ type: 'message:added', messageId: id, source: 'deferred-flush:turn-start' }); + }); this.ackDeferredWrites(); } } @@ -9654,7 +9667,7 @@ export class AgentFramework { } if (!this.activeTurnTokens.has(agent.name) && this.deferredMessages.length > 0 && !this.pendingAssistantBlocks.has(agent.name)) { - for (const msg of this.drainDeferredFor(agent.name)) this.readdDrained(msg, 'withdrawn-turn flush'); + this.readdDrained(this.drainDeferredFor(agent.name), 'withdrawn-turn flush'); this.ackDeferredWrites(); } return false; @@ -11535,7 +11548,7 @@ export class AgentFramework { // prefix is stored in order, or goes back into the queue whole under // its ids when a successor turn has already taken the agent. if (frameReachedTerminal && this.deferredMessages.length > 0 && !this.pendingAssistantBlocks.has(agent.name)) { - for (const msg of this.drainDeferredFor(agent.name)) this.readdDrained(msg, 'turn-end flush'); + this.readdDrained(this.drainDeferredFor(agent.name), 'turn-end flush'); this.ackDeferredWrites(); } } @@ -11720,7 +11733,7 @@ export class AgentFramework { // was queued — nobody else will flush it, so we do. if (!this.activeTurnTokens.has(agentName) && this.deferredMessages.length > 0 && !this.pendingAssistantBlocks.has(agentName)) { - for (const msg of this.drainDeferredFor(agentName)) this.readdDrained(msg, 'puppet flush'); + this.readdDrained(this.drainDeferredFor(agentName), 'puppet flush'); this.ackDeferredWrites(); } } @@ -13228,12 +13241,12 @@ export class AgentFramework { } /** - * Write one drained entry. If the write throws, the entry goes back into - * the queue under its id and seq, and the caller goes on with the rest: - * left in the un-acked set, the next ack would drop it from the durable - * queue, and a memory-only queue would lose it at once. Every site that - * writes a drained entry writes through this. Returns whether it was - * written (stored, or deferred again by addMessage). + * Write one drained entry. If the write throws, it is logged, naming the + * entry, and goes back into the queue under its id and seq + * (requeueDrained): left in the un-acked set, the next ack would drop it + * from the durable queue. Every site writes a drained list through + * writeDrainedInOrder, which cuts there. Returns whether it was written + * (stored, or deferred again by addMessage). */ private writeDrained(msg: DeferredWrite, where: string, write: () => void): boolean { try { @@ -13241,23 +13254,58 @@ export class AgentFramework { return true; } catch (error) { console.error( - `[deferral] ${where}: a deferred write for ${msg.forAgent ?? this.primaryAgentName ?? 'the primary'} ` + - `(participant=${msg.participant}) could not be stored; it stays queued: ` + - `${error instanceof Error ? error.message : String(error)}`, + `[deferral] ${where}: deferred write ${describeDeferredWrite(msg)} for ` + + `${msg.forAgent ?? this.primaryAgentName ?? 'the primary'} could not be stored; it stays queued, ` + + `holding what is queued behind it for that agent: ${error instanceof Error ? error.message : String(error)}`, ); - this.unackedDeferredWrites = this.unackedDeferredWrites.filter((m) => m.id !== msg.id); - if (!this.deferredMessages.some((m) => m.id === msg.id)) this.deferredMessages.push(msg); - if (this.quiesced || this.deferredWritesPersisted) this.persistDeferredWrites(); + this.requeueDrained([msg]); return false; } } - /** Re-add one drained entry through addMessage, which stores it or defers it again. */ - private readdDrained(msg: DeferredWrite, where: string): boolean { - return this.writeDrained(msg, where, () => { + /** + * Put drained entries back into the queue under their ids and seq: out of + * the un-acked set, and persisted when the queue is. A memory-only queue + * keeps them while the framework runs. + */ + private requeueDrained(entries: DeferredWrite[]): void { + if (entries.length === 0) return; + const ids = new Set(entries.map((m) => m.id)); + this.unackedDeferredWrites = this.unackedDeferredWrites.filter((m) => !ids.has(m.id)); + for (const msg of entries) { + if (!this.deferredMessages.some((m) => m.id === msg.id)) this.deferredMessages.push(msg); + } + if (this.quiesced || this.deferredWritesPersisted) this.persistDeferredWrites(); + } + + /** + * Write one agent's drained entries in queue order, each through + * writeDrained. The drain cuts at the first that can't be stored: it and + * every later entry go back into the queue, so nothing lands ahead of an + * earlier entry and a queued pair stays together. An entry that keeps + * failing holds its agent's later writes, logged at each flush, rather than + * letting them land out of order. Returns what was written, a prefix of + * `entries`. + */ + private writeDrainedInOrder(entries: DeferredWrite[], where: string, write: (msg: DeferredWrite) => void): DeferredWrite[] { + for (let i = 0; i < entries.length; i++) { + const msg = entries[i]!; + if (!this.writeDrained(msg, where, () => write(msg))) { + this.requeueDrained(entries.slice(i + 1)); + return entries.slice(0, i); + } + } + return entries; + } + + /** Re-add one agent's drained entries through addMessage, which stores each + * or defers it again, cut at the first that can't be stored. */ + private readdDrained(entries: DeferredWrite[], where: string, opts?: { bypassDeferralFor?: string }): DeferredWrite[] { + return this.writeDrainedInOrder(entries, where, (msg) => { this.addMessage(msg.participant, msg.content, msg.metadata, { deferredWriteId: msg.id, ...(msg.forAgent ? { forAgent: msg.forAgent } : {}), + ...(opts?.bypassDeferralFor ? { bypassDeferralFor: opts.bypassDeferralFor } : {}), }); }); } diff --git a/test/deferred-window-only.test.ts b/test/deferred-window-only.test.ts index e6d93a526..626f433ab 100644 --- a/test/deferred-window-only.test.ts +++ b/test/deferred-window-only.test.ts @@ -125,12 +125,13 @@ describe('mid-turn deliveries keep the window what the stream carried', () => { rmSync(tempDir, { recursive: true, force: true }); }); - async function createFramework(): Promise { + async function createFramework(extra: Partial[0]> = {}): Promise { const framework = await AgentFramework.create({ storePath: join(tempDir, 'test.chronicle'), membrane: membrane.asMembrane(), agents: [{ name: 'assistant', model: 'test-model', systemPrompt: 'You are a robot pilot.' }], modules: [module], + ...extra, }); module.framework = framework; return framework; @@ -397,7 +398,7 @@ describe('mid-turn deliveries keep the window what the stream carried', () => { await reopened.stop(); }); - it('at a tool boundary, a write whose store throws stays queued and is not injected; the rest are', async () => { + it('at a tool boundary, a write whose store throws cuts the drain: it and what follows wait for the turn end, in order', async () => { membrane.pushResponse(createMockResponse([ { type: 'text', text: 'Moving.' }, { type: 'tool_use', id: 'c1', name: 'robot--move', input: {} }, @@ -424,16 +425,96 @@ describe('mid-turn deliveries keep the window what the stream carried', () => { framework.pushEvent({ type: 'external-message', source: 'test', content: 'go' } as unknown as ProcessEvent); await framework.runUntilIdle(); } finally { console.error = error; } - assert.deepEqual(injected(), ['Antra: look right']); + assert.deepEqual(injected(), [], 'nothing from the failed write on is injected'); assert.deepEqual(windowOf(framework).slice(2), [ 'user: ', - 'Antra: look right', 'assistant: Done.', 'Antra: look left', - ], 'the one that failed was kept, and stored at the turn end'); + 'Antra: look right', + ], 'both were kept, and stored at the turn end in queue order'); await framework.stop(); }); + it("a queued tool_use whose store throws keeps its tool_result from being stored alone", async () => { + const framework = await createFramework(); + const internals = internalsOf(framework); + internals.deferredWritesPersisted = true; + // A pair queued for the agent, as puppetToolCall queues one behind a live turn. + const queue = framework as unknown as { + deferredSeq: number; + deferredMessages: Array<{ id: string; seq: number; participant: string; content: ContentBlock[]; forAgent?: string }>; + }; + queue.deferredMessages.push( + { id: 'pair-use', seq: ++queue.deferredSeq, participant: 'assistant', content: [{ type: 'tool_use', id: 'p1', name: 'robot--move', input: {} } as ContentBlock], forAgent: 'assistant' }, + { id: 'pair-result', seq: ++queue.deferredSeq, participant: 'user', content: [{ type: 'tool_result', toolUseId: 'p1', content: 'moved' } as ContentBlock], forAgent: 'assistant' }, + ); + const cm = framework.getAgent('assistant')!.getContextManager(); + const add = cm.addMessage.bind(cm); + (cm as unknown as { addMessage: typeof add }).addMessage = (...args) => { + if ((args[1] as ContentBlock[]).some((b) => b.type === 'tool_use')) throw new Error('disk full'); + return add(...args); + }; + const errors: string[] = []; + const error = console.error; + console.error = (...args: unknown[]) => { errors.push(args.map(String).join(' ')); }; + try { await framework.stop(); } finally { console.error = error; } + assert.ok(errors.some((e) => e.includes('pair-use (assistant: tool_use)') && e.includes('disk full')), errors.join('\n')); + assert.deepEqual(queue.deferredMessages.map((m) => m.id), ['pair-use', 'pair-result'], 'both stay queued, in order'); + + const reopened = await createFramework(); + const window = windowOf(reopened); + assert.deepEqual(window.slice(-2), ['assistant: ', 'user: '], 'boot recovery stored the pair, in order'); + assert.equal(window.filter((line) => line === 'user: ').length, 1, 'no tool_result was stored alone'); + await reopened.stop(); + }); + + it("stop() stores what is deferred for an agent whose turn frame it doesn't wait for", async () => { + const framework = await createFramework(); + const internals = internalsOf(framework); + // A frame stop() doesn't await, such as a puppet call still running, + // holds the agent's turn token. + internals.activeTurnTokens.set('assistant', 1); + assert.equal(internals.addMessage('Antra', text('still to land')), '', 'deferred: the turn is alive'); + await framework.stop(); + + const reopened = await createFramework(); + assert.ok(windowOf(reopened).includes('Antra: still to land'), 'stored before the store closed'); + await reopened.stop(); + }); + + it('a turn waiting to retry when the framework stops gives up rather than compile again', async () => { + let waiting!: () => void; + const inDelay = new Promise((resolve) => { waiting = resolve; }); + const framework = await createFramework({ + errorPolicy: { + maxRetries: 2, + onInferenceError: (_error: Error, _agent: string, attempt: number) => { + waiting(); + return attempt < 2 ? { retry: true as const, delayMs: 50 } : { retry: false as const }; + }, + }, + }); + const internals = internalsOf(framework); + const agent = framework.getAgent('assistant')!; + let compiles = 0; + (agent as unknown as { startStreamWithInjections: () => Promise }).startStreamWithInjections = async () => { + compiles++; + throw new Error('boom'); + }; + framework.start(); + framework.pushEvent({ type: 'external-message', source: 'test', content: 'go' } as unknown as ProcessEvent); + await inDelay; + assert.equal(internals.addMessage('Antra', text('while it waits')), '', 'deferred: the turn is alive'); + const error = console.error; + console.error = () => {}; + try { await framework.stop(); } finally { console.error = error; } + assert.equal(compiles, 1, 'the retry was refused, not compiled'); + + const reopened = await createFramework(); + assert.ok(windowOf(reopened).includes('Antra: while it waits'), 'what was deferred for it was stored'); + await reopened.stop(); + }); + it('a message stored straight into the context during a round, with nothing injected at the boundary, stays unread', async () => { membrane.pushResponse(createMockResponse([ { type: 'text', text: 'Moving.' }, diff --git a/test/ephemeral-settle.test.ts b/test/ephemeral-settle.test.ts index ce029a56c..d3998aa05 100644 --- a/test/ephemeral-settle.test.ts +++ b/test/ephemeral-settle.test.ts @@ -122,6 +122,49 @@ async function stopAndRemove(framework: AgentFramework, tempDir: string): Promis } describe('runEphemeralToCompletion settle signal', () => { + it('a run whose turn stop() refuses ends at once, and a run asked for once stop() has begun is refused', async () => { + const { tempDir, storePath } = tempStorePath('ephemeral-stopping-'); + const membrane = new MockMembrane(); + membrane.pushResponse(createMockResponse([{ type: 'text', text: 'too late' }])); + const framework = await AgentFramework.create({ + storePath, + membrane: membrane.asMembrane(), + agents: [], + modules: [new EphemeralToolModule()], + syncIntervalMs: 0, + }); + try { + const { agent, contextManager } = await createEphemeral(framework); + const later = await createEphemeral(framework, 'later'); + // Hold the run's compile until stop() has begun. + let compiling!: () => void; + const inCompile = new Promise((resolve) => { compiling = resolve; }); + let release!: () => void; + const released = new Promise((resolve) => { release = resolve; }); + const start = agent.startStreamWithInjections.bind(agent); + (agent as unknown as { startStreamWithInjections: typeof start }).startStreamWithInjections = async (...args) => { + compiling(); + await released; + return start(...args); + }; + const run = runStartedEphemeral(framework, agent, contextManager); + await inCompile; + const stopped = framework.stop(); + release(); + const refusedAt = Date.now(); + await assert.rejects(run, /refused: the framework is stopping/); + assert.ok(Date.now() - refusedAt < 5_000, 'ended at once, not at its startup watchdog'); + await stopped; + await assert.rejects( + framework.runEphemeralToCompletion(later.agent, later.contextManager), + /framework is stopping: refusing new ephemeral run/, + ); + later.cleanup(); + } finally { + rmSync(tempDir, { recursive: true, force: true }); + } + }); + it('resolves plain text completion and deregisters the agent', async () => { const { tempDir, storePath } = tempStorePath('ephemeral-plain-'); const membrane = new MockMembrane(); diff --git a/test/host-quiesce.test.ts b/test/host-quiesce.test.ts index c0a08f611..ac24605d8 100644 --- a/test/host-quiesce.test.ts +++ b/test/host-quiesce.test.ts @@ -683,7 +683,7 @@ test('a batch carrying a budget restart re-parks its sibling wakes instead of co }); }); -test('resume flushes deferred writes per message: one poison write neither drops the rest nor skips the reopen, and stays queued', async () => { +test('resume flushes deferred writes in order: a poison write stays queued with what follows it, and the reopen still completes', async () => { const membrane = new MockMembrane(); await withFramework(membrane, async (framework) => { const traces: TraceEvent[] = []; @@ -709,13 +709,14 @@ test('resume flushes deferred writes per message: one poison write neither drops const status = await framework.resume(); assert.equal(status.quiesced, false); - // A drained write whose store throws is never dropped: it goes back into - // the queue under its id, for the next flush. - assert.equal(status.deferredWrites, 1, 'only the write that failed is still queued'); + // A drained write whose store throws is never dropped, and nothing lands + // ahead of it: it and the write queued behind it go back into the queue + // under their ids, for the next flush. + assert.equal(status.deferredWrites, 2, 'the write that failed and the one behind it are still queued'); assert.deepEqual( cm.getAllMessages().slice(before).map((m) => (m.content[0] as { text: string }).text), - ['first', 'third'], - 'the writes behind the poison one still landed', + ['first'], + 'the write queued before the poison one landed', ); assert.ok(traces.some((t) => t.type === 'host:resume'), 'resume completed its follow-through'); }); diff --git a/test/operator-surgery.test.ts b/test/operator-surgery.test.ts index 2be1ce269..7a4d051a0 100644 --- a/test/operator-surgery.test.ts +++ b/test/operator-surgery.test.ts @@ -257,6 +257,37 @@ describe('live operator surgery', () => { } }); + it("the post-surgery flush lands what waited behind the hold while another agent's tool cycle is pending, and acknowledges it", async () => { + const c = cm() as unknown as { switchBranch: (name: string) => Promise }; + const originalSwitch = c.switchBranch; + let releaseSwitch!: () => void; + const gate = new Promise((resolve) => { releaseSwitch = resolve; }); + c.switchBranch = async (name: string) => { await gate; return originalSwitch.call(c, name); }; + const internals = framework as unknown as { + surgeryHold: unknown; + deferredWritesPersisted: boolean; + unackedDeferredWrites: unknown[]; + pendingAssistantBlocks: Map; + addMessage(participant: string, content: unknown[], metadata?: Record): string; + }; + internals.deferredWritesPersisted = true; + try { + const rollback = framework.rollbackToMessage('scout', { messageId: ids[2] }); + await new Promise((resolve) => setImmediate(resolve)); + assert.ok(internals.surgeryHold, 'store hold is up while the switch is awaited'); + assert.equal(internals.addMessage('user', [{ type: 'text', text: 'waited for the hold' }]), '', 'deferred'); + // Another agent's tool cycle says nothing about scout's window. + internals.pendingAssistantBlocks.set('builder', [{ type: 'tool_use', id: 'b1', name: 'x', input: {} }]); + releaseSwitch(); + await rollback; + assert.ok(remainingTexts().includes('waited for the hold'), 'flushed when the hold released'); + assert.equal(internals.unackedDeferredWrites.length, 0, 'and acknowledged, so the recovery queue no longer holds it'); + } finally { + internals.pendingAssistantBlocks.delete('builder'); + c.switchBranch = originalSwitch; + } + }); + // --- strategy-initialization failure ----------------------------------------- // switchBranch()/fork() move the chronicle branch BEFORE awaiting strategy // initialization, so a rejection there leaves the store on the new branch. From 7e1c588ff3eda192d1df05a22f8258d70ad46c9d Mon Sep 17 00:00:00 2001 From: Duncan-2410 Date: Tue, 13 Oct 2026 10:48:53 -0400 Subject: [PATCH 06/34] fix(deferral): a stuck write's log line can't drop it; say what the hold covers From the internal review of the previous round. Code: writeDrained puts a failed entry back into the queue before it logs, and describeDeferredWrite no longer throws on a malformed entry (a block that isn't an object with a type reads as "malformed"). Before, a null block made the description throw inside the catch, before the requeue: the entry stayed in the un-acked set for the next acknowledgement to drop, writeDrainedInOrder never requeued what followed it, and the flush threw. addMessage refuses a null block, so only a queue persisted by other code could carry one, but the log line must not be able to lose what it reports. Text, as Ada-1017 decided: the cut holds what is queued, and a write that arrives while its agent is idle isn't deferred, so it is stored at once and can land ahead of a held entry; an entry that keeps throwing holds only what is deferred behind it during its agent's turns, or while the host is quiesced. (A gate that held fresh writes too would blind an agent behind a poison entry: every wake's payload held, each turn run without what woke it.) A queued tool_use that can't be stored keeps its tool_result from being stored alone, but a stored tool_use whose tool_result then fails stays stored without it until the result lands; the only queued pair is puppetToolCall's defensive one. stop() neither refuses nor waits for a puppet call, as before: one still running stores its pair after the drain's entries, and on a closed store that write fails. The fragment no longer says no turn starts "for any agent". Tests: - host-quiesce: two residents, one with a stuck (and malformed) write ahead of the other's in the queue. Resume stores the other's writes and holds the stuck one with what is queued behind it for its own agent; a second flush holds it again; each flush's log line names its id and block types. - ephemeral-settle: a run waiting to retry when stop() begins gives up before compiling again. An ephemeral run doesn't hold the primary's provider gate, so startAgentStream's top refusal is the only check its retry meets; the primary's retry also meets the check after the gate, which is why removing the top refusal passed before. The "ended at once" check drops its 5 s wall-clock bound: the rejection's message already tells the refusal from the startup watchdog's. - deferred-window-only: the primary's retry test asserts the retry was granted and re-entered after stop() began, so a retry never granted can't pass for one refused. - operator-surgery: waits for the switchBranch wrapper to be entered rather than one setImmediate tick. Co-Authored-By: Claude Opus 5.5 --- .../mid-turn-window-only-held.fixed.md | 33 +++++---- src/framework.ts | 62 ++++++++++++----- test/deferred-window-only.test.ts | 13 ++++ test/ephemeral-settle.test.ts | 43 +++++++++++- test/host-quiesce.test.ts | 69 +++++++++++++++++++ test/operator-surgery.test.ts | 6 +- 6 files changed, 191 insertions(+), 35 deletions(-) diff --git a/changelog.d/mid-turn-window-only-held.fixed.md b/changelog.d/mid-turn-window-only-held.fixed.md index ff15b2e87..d45b27636 100644 --- a/changelog.d/mid-turn-window-only-held.fixed.md +++ b/changelog.d/mid-turn-window-only-held.fixed.md @@ -34,20 +34,29 @@ cycle holds them. - `stop()` now stores what is still deferred for each registered agent before the store closes; before, a memory-only queue lost it. Once `stop()` - begins, no turn starts and no stream registers, for any agent: a turn that - was mid-compile when the streams were cancelled, or one waiting to retry, - is refused rather than left running, and an ephemeral run refused this way + begins, no agent turn starts and no stream registers: a turn that was + mid-compile when the streams were cancelled, or one waiting to retry, is + refused rather than left running, and an ephemeral run refused this way ends at once rather than at its startup watchdog. Since no reply can then - follow a stored entry, `stop()` stores each one even while a frame it - doesn't wait for, such as a puppet call, still holds its target's turn. A - quiesced host keeps what is deferred in its persisted queue for boot - recovery. An entry for a target no longer registered (a disposed fork) is - dropped from a memory-only queue at exit, as before. + follow a stored entry in a stream, `stop()` stores each one even while a + frame it doesn't wait for still holds its target's turn: a turn started + outside the event loop, or a puppet call. `stop()` neither refuses nor + waits for a puppet call, as before, so one still running stores its pair + when its tool returns, after those entries, and on a store `stop()` has + closed that write fails. A quiesced host keeps what is deferred in its + persisted queue for boot recovery. An entry for a target no longer + registered (a disposed fork) is dropped from a memory-only queue at exit, + as before. - A deferred write whose store throws goes back into the queue under its - id, and so does every entry queued behind it for that agent, so none lands - out of order and a queued `tool_use` and its `tool_result` stay together. - The log line names the entry, and one that keeps throwing holds the - agent's later deferred writes until it can be stored. Before, it stayed in + id, and so does every entry queued behind it for that agent, so none of + them lands ahead of an earlier one, and a queued `tool_use` that can't be + stored keeps its `tool_result` from being stored alone. The log line names + the entry at each flush, and one that keeps throwing holds what is deferred + behind it for that agent until it can be stored. The hold is the queue's: + a write that arrives while the agent is idle isn't deferred, so it is + stored at once and can land ahead of a held entry, and an entry that keeps + throwing holds only what is deferred during the agent's turns (or while the + host is quiesced). Before, it stayed in the un-acked set and the next acknowledgement dropped it from the persisted queue (at a turn start or a flush it was lost even from memory). At a tool boundary, nothing from it on is injected. While the framework diff --git a/src/framework.ts b/src/framework.ts index e05481119..8ebf455df 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -481,9 +481,19 @@ function withDeferredWriteId(metadata: MessageMetadata | undefined, id: string): return { ...(metadata ?? {}), deferredWriteId: id } as MessageMetadata; } -/** A deferred entry as a log line names it: its id, participant and block types. */ +/** A deferred entry as a log line names it: its id, participant and block + * types. It never throws: it describes the entries the store rejected, which + * may be malformed (a block that isn't an object with a type reads as + * `malformed`). */ function describeDeferredWrite(msg: { id: string; participant: string; content: ContentBlock[] }): string { - return `${msg.id} (${msg.participant}: ${[...new Set(msg.content.map((b) => b.type))].join('+') || 'empty'})`; + const content: unknown = msg.content; + const types = Array.isArray(content) + ? [...new Set(content.map((b: unknown) => { + const type = b && typeof b === 'object' ? (b as { type?: unknown }).type : undefined; + return typeof type === 'string' ? type : 'malformed'; + }))].join('+') || 'empty' + : 'malformed'; + return `${msg.id} (${msg.participant}: ${types})`; } /** Whether a deferred message can join a live stream at a tool boundary: @@ -2277,17 +2287,20 @@ export class AgentFramework { } } - // No stream is left: from the start of stop() none registers and no turn - // starts (providerAdmissionClosed, checked for every agent), and every - // stream registered before was cancelled and awaited above. So nothing - // else will deliver what is still deferred: a message held for a turn's - // end whose flush found the target busy again, or one that arrived for an - // idle agent and was deferred behind its own state. Land each at its + // No stream is left: from the start of stop() none registers and no + // agent turn starts (providerAdmissionClosed, checked for every agent), + // and every stream registered before was cancelled and awaited above. + // What is still deferred (a message held for a turn's end whose flush + // found the target busy again, or one deferred behind its target's own + // state) would wait for a flush that may never come. Land each at its // registered target now, while the store is open; a memory-only queue // would otherwise lose it at exit. It goes past the deferral: a frame - // stop() doesn't wait for (a puppet call, or a turn started outside the - // event loop) can still hold its target's turn token, but no reply can - // follow the entry in any stream. A quiesced host defers it again + // stop() doesn't wait for (a turn started outside the event loop, or a + // puppet call) can still hold its target's turn token, but no reply can + // follow the entry in any stream. stop() neither refuses nor waits for a + // puppet call, as before this drain: one still running stores its pair + // when its tool returns, after these entries, and on a store closed + // below that write fails. A quiesced host defers it again // instead, into the persisted queue, for boot recovery. A write that // throws stays queued with what follows it (writeDrainedInOrder), and a // memory-only queue still loses it at exit. An entry for a target no @@ -3664,8 +3677,10 @@ export class AgentFramework { * after every message). A crash mid-flush therefore replays exactly the * un-acked remainder at the next boot — no loss, no duplicates. A write * the store rejects stays queued, logged, with what is queued behind it for - * its target (writeDrainedInOrder), so a target's entries never land out of - * order; a boot that meets it again logs it and goes on. + * its target (writeDrainedInOrder), so the flush never stores a target's + * later entries ahead of it (a write that arrives while its target is idle + * isn't deferred, though, and can land ahead of it); a boot that meets it + * again logs it and goes on. * * Per-target: an agent whose turn is still alive keeps ITS messages * deferred for its own turn boundary; everyone else's flush now. @@ -13253,12 +13268,13 @@ export class AgentFramework { write(); return true; } catch (error) { + // Back into the queue before anything else can go wrong here. + this.requeueDrained([msg]); console.error( `[deferral] ${where}: deferred write ${describeDeferredWrite(msg)} for ` + `${msg.forAgent ?? this.primaryAgentName ?? 'the primary'} could not be stored; it stays queued, ` + `holding what is queued behind it for that agent: ${error instanceof Error ? error.message : String(error)}`, ); - this.requeueDrained([msg]); return false; } } @@ -13281,11 +13297,19 @@ export class AgentFramework { /** * Write one agent's drained entries in queue order, each through * writeDrained. The drain cuts at the first that can't be stored: it and - * every later entry go back into the queue, so nothing lands ahead of an - * earlier entry and a queued pair stays together. An entry that keeps - * failing holds its agent's later writes, logged at each flush, rather than - * letting them land out of order. Returns what was written, a prefix of - * `entries`. + * every later entry go back into the queue, so none of them lands ahead of + * an earlier one, and a queued tool_use that can't be stored keeps its + * tool_result from being stored alone. (A tool_use that is stored, followed + * by a tool_result that can't be, stays stored without it until the + * tool_result lands at a later flush. The only queued pair is + * puppetToolCall's defensive one, which no path reaches today.) An + * entry that keeps failing holds its agent's later queued writes, logged + * at each flush, rather than letting them land out of order. The hold is + * the queue's alone: a write that arrives while its agent is idle isn't + * deferred (addMessage stores it at once), so it can land ahead of a held + * entry, and an entry that keeps failing holds only what is deferred + * behind it while its agent's turn, stream or tool cycle is alive, or the + * host is quiesced. Returns what was written, a prefix of `entries`. */ private writeDrainedInOrder(entries: DeferredWrite[], where: string, write: (msg: DeferredWrite) => void): DeferredWrite[] { for (let i = 0; i < entries.length; i++) { diff --git a/test/deferred-window-only.test.ts b/test/deferred-window-only.test.ts index 626f433ab..e0039d2d3 100644 --- a/test/deferred-window-only.test.ts +++ b/test/deferred-window-only.test.ts @@ -501,6 +501,18 @@ describe('mid-turn deliveries keep the window what the stream carried', () => { compiles++; throw new Error('boom'); }; + // The retry re-enters startAgentStream after its delay; count those + // re-entries, so a retry that was never granted can't pass for one that + // was refused. + const retries: number[] = []; + const frames = framework as unknown as { + startAgentStream(agent: unknown, trigger?: unknown, attempt?: number, held?: boolean): Promise; + }; + const startAgentStream = frames.startAgentStream.bind(framework); + frames.startAgentStream = (a, trigger, attempt = 0, held = false) => { + if (attempt > 0) retries.push(attempt); + return startAgentStream(a, trigger, attempt, held); + }; framework.start(); framework.pushEvent({ type: 'external-message', source: 'test', content: 'go' } as unknown as ProcessEvent); await inDelay; @@ -508,6 +520,7 @@ describe('mid-turn deliveries keep the window what the stream carried', () => { const error = console.error; console.error = () => {}; try { await framework.stop(); } finally { console.error = error; } + assert.deepEqual(retries, [1], 'the retry was granted, and re-entered once stop() had begun'); assert.equal(compiles, 1, 'the retry was refused, not compiled'); const reopened = await createFramework(); diff --git a/test/ephemeral-settle.test.ts b/test/ephemeral-settle.test.ts index d3998aa05..9d20db562 100644 --- a/test/ephemeral-settle.test.ts +++ b/test/ephemeral-settle.test.ts @@ -151,9 +151,9 @@ describe('runEphemeralToCompletion settle signal', () => { await inCompile; const stopped = framework.stop(); release(); - const refusedAt = Date.now(); + // Ended by the refusal, not by its startup watchdog, whose rejection + // names a zombie instead. await assert.rejects(run, /refused: the framework is stopping/); - assert.ok(Date.now() - refusedAt < 5_000, 'ended at once, not at its startup watchdog'); await stopped; await assert.rejects( framework.runEphemeralToCompletion(later.agent, later.contextManager), @@ -165,6 +165,45 @@ describe('runEphemeralToCompletion settle signal', () => { } }); + it('a run waiting to retry when stop() begins gives up before compiling again', async () => { + // An ephemeral run doesn't hold the primary's provider gate, so the + // refusal at the top of startAgentStream is the only check its retry + // meets before it would compile, and start a stream, again. + const { tempDir, storePath } = tempStorePath('ephemeral-retry-stopping-'); + let waiting!: () => void; + const inDelay = new Promise((resolve) => { waiting = resolve; }); + const framework = await AgentFramework.create({ + storePath, + membrane: new MockMembrane().asMembrane(), + agents: [], + modules: [new EphemeralToolModule()], + syncIntervalMs: 0, + errorPolicy: { + maxRetries: 1, + onInferenceError: (_error: Error, _agent: string, attempt: number): ErrorAction => { + waiting(); + return attempt < 1 ? { retry: true, delayMs: 50 } : { retry: false }; + }, + }, + }); + try { + const { agent, contextManager } = await createEphemeral(framework); + let compiles = 0; + (agent as unknown as { startStreamWithInjections: () => Promise }).startStreamWithInjections = async () => { + compiles++; + throw new Error('boom'); + }; + const run = runStartedEphemeral(framework, agent, contextManager); + await inDelay; + const stopped = framework.stop(); + await assert.rejects(run, /refused: the framework is stopping/); + await stopped; + assert.equal(compiles, 1, 'the retry was refused, not compiled'); + } finally { + rmSync(tempDir, { recursive: true, force: true }); + } + }); + it('resolves plain text completion and deregisters the agent', async () => { const { tempDir, storePath } = tempStorePath('ephemeral-plain-'); const membrane = new MockMembrane(); diff --git a/test/host-quiesce.test.ts b/test/host-quiesce.test.ts index ac24605d8..75ff5fbb6 100644 --- a/test/host-quiesce.test.ts +++ b/test/host-quiesce.test.ts @@ -722,6 +722,75 @@ test('resume flushes deferred writes in order: a poison write stays queued with }); }); +test("resume cuts each target at its own stuck write, naming it at every flush, and other targets' writes land", async () => { + const dir = mkdtempSync(join(tmpdir(), 'host-quiesce-targets-')); + const framework = await AgentFramework.create({ + storePath: join(dir, 'store'), + membrane: new MockMembrane().asMembrane(), + agents: [ + { name: 'agent', model: 'test-model', systemPrompt: 'test' }, + { name: 'scout', model: 'test-model', systemPrompt: 'test' }, + ], + modules: [], + }); + const lines: string[] = []; + const error = console.error; + console.error = (...args: unknown[]) => { lines.push(args.map(String).join(' ')); }; + try { + await framework.quiesce({ reason: 'refold' }); + const internals = framework as unknown as { + deferredSeq: number; + deferredMessages: Array<{ id: string; seq: number; participant: string; content: unknown[]; forAgent?: string }>; + addMessage(participant: string, content: unknown[], metadata?: undefined, opts?: { forAgent?: string }): unknown; + }; + // Residents share one message slot, so each one's context manager reads + // what either stored; each stores through its own. + const agentCm = framework.getAgent('agent')!.getContextManager(); + const before = agentCm.getAllMessages().length; + const stored = () => agentCm.getAllMessages().slice(before).map((m) => (m.content[0] as { text: string }).text); + // A malformed entry the store rejects, as only a queue persisted by other + // code could carry (addMessage refuses a null block): the log line still + // names it, and it still goes back into the queue. + internals.deferredMessages.push({ + id: 'stuck-1', seq: ++internals.deferredSeq, participant: 'user', + content: [{ type: 'text', text: 'stuck' }, null], forAgent: 'agent', + }); + internals.addMessage('user', [{ type: 'text', text: 'for scout' }], undefined, { forAgent: 'scout' }); + internals.addMessage('user', [{ type: 'text', text: 'behind the stuck one' }], undefined, { forAgent: 'agent' }); + internals.addMessage('user', [{ type: 'text', text: 'also for scout' }], undefined, { forAgent: 'scout' }); + assert.equal(framework.getHostModeStatus().deferredWrites, 4); + + const realAdd = agentCm.addMessage.bind(agentCm); + (agentCm as unknown as { addMessage: typeof agentCm.addMessage }).addMessage = ((participant, content, metadata, causedBy) => { + if ((content[0] as { text?: string }).text === 'stuck') throw new Error('store write failed'); + return realAdd(participant, content, metadata, causedBy); + }) as typeof agentCm.addMessage; + const named = () => lines.filter((line) => + line.includes('deferred write stuck-1 (user: text+malformed) for agent could not be stored')).length; + + let status = await framework.resume(); + // Queued after agent's stuck write, scout's land; agent's own later + // write is held behind it. + assert.deepEqual(stored(), ['for scout', 'also for scout'], "scout's writes land; agent's stuck write holds what is queued behind it for agent"); + assert.equal(status.deferredWrites, 2); + assert.equal(named(), 1, 'the flush names the stuck write: its id and block types'); + + // A second flush with the store still refusing it: held again, named again. + await framework.quiesce({ reason: 'refold' }); + status = await framework.resume(); + assert.deepEqual(stored(), ['for scout', 'also for scout'], 'still nothing lands ahead of it'); + assert.equal(status.deferredWrites, 2); + assert.equal(named(), 2, 'every flush names it'); + } finally { + try { + await framework.stop(); + } finally { + console.error = error; + rmSync(dir, { recursive: true, force: true }); + } + } +}); + test('context writes deferred while quiesced survive a restart and land at resume', async () => { const dir = mkdtempSync(join(tmpdir(), 'host-quiesce-deferred-')); const storePath = join(dir, 'store'); diff --git a/test/operator-surgery.test.ts b/test/operator-surgery.test.ts index 7a4d051a0..d38379d17 100644 --- a/test/operator-surgery.test.ts +++ b/test/operator-surgery.test.ts @@ -262,7 +262,9 @@ describe('live operator surgery', () => { const originalSwitch = c.switchBranch; let releaseSwitch!: () => void; const gate = new Promise((resolve) => { releaseSwitch = resolve; }); - c.switchBranch = async (name: string) => { await gate; return originalSwitch.call(c, name); }; + let entered!: () => void; + const inSwitch = new Promise((resolve) => { entered = resolve; }); + c.switchBranch = async (name: string) => { entered(); await gate; return originalSwitch.call(c, name); }; const internals = framework as unknown as { surgeryHold: unknown; deferredWritesPersisted: boolean; @@ -273,7 +275,7 @@ describe('live operator surgery', () => { internals.deferredWritesPersisted = true; try { const rollback = framework.rollbackToMessage('scout', { messageId: ids[2] }); - await new Promise((resolve) => setImmediate(resolve)); + await inSwitch; assert.ok(internals.surgeryHold, 'store hold is up while the switch is awaited'); assert.equal(internals.addMessage('user', [{ type: 'text', text: 'waited for the hold' }]), '', 'deferred'); // Another agent's tool cycle says nothing about scout's window. From 2ab4b0f1b4303e4e75b03c8e498ce5c0f9c67d0b Mon Sep 17 00:00:00 2001 From: Duncan-2410 Date: Tue, 13 Oct 2026 11:24:06 -0400 Subject: [PATCH 07/34] test(deferral): a run that stop() should end fails at once if it isn't ended 7e1c588's ephemeral test said a run the refusal didn't end would be ended by its startup watchdog, whose rejection names a zombie. It wouldn't: the turn marks inference started before its compile, so the startup watchdog never fires, and only the idle watchdog would end the run, after fifteen minutes. With the refusal removed, both stop() tests in ephemeral-settle waited out the test runner's timeout, and CI's npm test sets none. (The 5 s bound that 7e1c588 dropped never caught that case either: it was checked only after the rejection, which never came.) The two tests now give their run a 20 s idle watchdog, so a run the refusal doesn't end is rejected as stalled, which the refusal's pattern doesn't match. A refusal that works ends the run within milliseconds of the compile's release, far inside that window. Co-Authored-By: Claude Opus 5.5 --- test/ephemeral-settle.test.ts | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/test/ephemeral-settle.test.ts b/test/ephemeral-settle.test.ts index 9d20db562..21765f833 100644 --- a/test/ephemeral-settle.test.ts +++ b/test/ephemeral-settle.test.ts @@ -110,12 +110,19 @@ async function runStartedEphemeral( framework: AgentFramework, agent: Awaited>['agent'], contextManager: Awaited>['contextManager'], + watchdogs?: Parameters[2], ) { - const promise = framework.runEphemeralToCompletion(agent, contextManager); + const promise = framework.runEphemeralToCompletion(agent, contextManager, watchdogs); framework.start(); return promise; } +// For a run that stop() is to end: its turn has started inference, so if the +// refusal didn't end it, only the idle watchdog would, after fifteen minutes. +// A short one turns that into a "stalled" rejection, which the refusal's +// pattern doesn't match. +const STOPPED_RUN_WATCHDOGS = { idleTimeoutMs: 20_000, idlePollMs: 50 }; + async function stopAndRemove(framework: AgentFramework, tempDir: string): Promise { await framework.stop(); rmSync(tempDir, { recursive: true, force: true }); @@ -147,12 +154,10 @@ describe('runEphemeralToCompletion settle signal', () => { await released; return start(...args); }; - const run = runStartedEphemeral(framework, agent, contextManager); + const run = runStartedEphemeral(framework, agent, contextManager, STOPPED_RUN_WATCHDOGS); await inCompile; const stopped = framework.stop(); release(); - // Ended by the refusal, not by its startup watchdog, whose rejection - // names a zombie instead. await assert.rejects(run, /refused: the framework is stopping/); await stopped; await assert.rejects( @@ -193,7 +198,7 @@ describe('runEphemeralToCompletion settle signal', () => { compiles++; throw new Error('boom'); }; - const run = runStartedEphemeral(framework, agent, contextManager); + const run = runStartedEphemeral(framework, agent, contextManager, STOPPED_RUN_WATCHDOGS); await inDelay; const stopped = framework.stop(); await assert.rejects(run, /refused: the framework is stopping/); From c9d612e98ad6da1a7ff5d489b6be3a4efb3c7773 Mon Sep 17 00:00:00 2001 From: Duncan-2410 Date: Tue, 13 Oct 2026 11:47:16 -0400 Subject: [PATCH 08/34] fix(deferral): flushes store in queue order across agents; recovery drops entries the store can't take From the internal review of the previous round, with Ada-1017 deciding the two design points. Queue order across agents. Residents share one message slot, so the order deferred writes arrived in is the order every resident reads. 1e95eb2 grouped resume's and boot recovery's flush by target to cut per target, so even with nothing failing it stored all of one target's entries before the next target's, where 0c5c0b0 stored them in queue order. stop()'s drain and the post-surgery flush went agent by agent, as main's post-surgery flush did. writeDrainedInOrder now takes any targets' entries in queue (seq) order and cuts each target at its own first entry that can't be stored: that entry and every later one for the same target go back into the queue before anything is logged, while other targets' entries go on. Resume, boot recovery, stop() and the post-surgery flush each hand it every agent's entries at once. writeDrained is folded into it. The log line: a thrown value with no string form (Object.create(null)) made String(error) throw inside the catch, before the entries behind the failed one were requeued, so the next ack dropped them. Everything for the cut target now goes back first, and the reason falls back to "a thrown value that has no text". Malformed recovered entries. An entry with a null block can never be stored, and can reach the queue only from the recovery file: a fresh write with a null block throws in addMessage before it is queued. Held, it blocked its agent's later deferred writes for good, and at a tool boundary isInjectableMidTurn threw on it. The loader already dropped, silently, an entry whose content isn't an array; it now also drops one whose blocks aren't all objects, logs each drop by id and participant, and rewrites the recovery file without them. Every reader of the queue can trust its entries' shape, so describeDeferredWrite is back to its plain form. Text: the fragment, the stop() comment and the docs now say that a fresh tool_result doesn't wait; that while quiesced an idle agent's write is deferred too; that an entry that keeps failing holds what is deferred while its agent's turn, stream or tool cycle is alive, or the host is quiesced; that a stored tool_use whose tool_result can't be stored stays alone until the result lands; that a puppet call whose tool returns before stop()'s drain stores its pair first, and one that returns later stores it after the drained entries; and that a refused ephemeral run ends at once rather than waiting for a watchdog. Tests: - host-quiesce: two residents' interleaved writes land through resume in queue order, and a stuck write, refused with a thrown value that has no text, cuts only its own agent and is named by id at each of two flushes. Boot recovery drops a recovered entry with a null block and one whose content isn't blocks, names each, rewrites the file, and restores and lands the rest. - deferred-window-only: stop() stores two residents' interleaved deferred writes in queue order. The retry test records that stop() had begun when the retry re-entered. - operator-surgery: the post-surgery flush stores two residents' interleaved writes in queue order. - The tests that wait on an event (the post-surgery hold, the ephemeral stop tests, the retry delay) take a 60 s timeout, so a missed event fails rather than hangs. Co-Authored-By: Claude Opus 5.5 --- .../mid-turn-window-only-held.fixed.md | 77 +++--- src/framework.ts | 227 ++++++++++-------- test/deferred-window-only.test.ts | 34 ++- test/ephemeral-settle.test.ts | 4 +- test/host-quiesce.test.ts | 97 ++++++-- test/operator-surgery.test.ts | 39 ++- 6 files changed, 317 insertions(+), 161 deletions(-) diff --git a/changelog.d/mid-turn-window-only-held.fixed.md b/changelog.d/mid-turn-window-only-held.fixed.md index d45b27636..698cb1cb1 100644 --- a/changelog.d/mid-turn-window-only-held.fixed.md +++ b/changelog.d/mid-turn-window-only-held.fixed.md @@ -22,43 +22,56 @@ content. That happens while its target's own turn, stream or tool cycle is alive, or the host is quiesced. A `tool_result` used to land regardless, apart from the `tool_use` it answers, which went back into the queue. -- Every write now waits on its own target's state: the target's turn, stream - and tool cycle, or a quiesced host. Before, a write for the primary agent - waited out every agent's tool cycle, while one addressed to an agent waited - only on that agent's. So a turn's end could store a later addressed message - ahead of an earlier one for the primary, and a fork's, subagent's or the +- A write that waits now waits on its own target's state: the target's + turn, stream and tool cycle, or a quiesced host. (A fresh `tool_result` + still doesn't wait.) Before, a write for the primary agent waited out every + agent's tool cycle, while one addressed to an agent waited only on that + agent's. So a turn's end could store a later addressed message ahead of an + earlier one for the primary, and a fork's, subagent's or the subconscious's tools held the primary's writes, which protected nothing in the primary's window. Each flush, at a turn's end, after surgery, after a withdrawn turn and after a puppet call, now delivers the agent's deferred messages while another agent's tool cycle is pending; only the agent's own - cycle holds them. + cycle holds them. The flush after surgery stores several agents' entries in + the order they arrived, where it went agent by agent: residents share one + message slot, so that reordered their shared transcript. - `stop()` now stores what is still deferred for each registered agent - before the store closes; before, a memory-only queue lost it. Once `stop()` - begins, no agent turn starts and no stream registers: a turn that was - mid-compile when the streams were cancelled, or one waiting to retry, is - refused rather than left running, and an ephemeral run refused this way - ends at once rather than at its startup watchdog. Since no reply can then - follow a stored entry in a stream, `stop()` stores each one even while a - frame it doesn't wait for still holds its target's turn: a turn started - outside the event loop, or a puppet call. `stop()` neither refuses nor - waits for a puppet call, as before, so one still running stores its pair - when its tool returns, after those entries, and on a store `stop()` has - closed that write fails. A quiesced host keeps what is deferred in its - persisted queue for boot recovery. An entry for a target no longer - registered (a disposed fork) is dropped from a memory-only queue at exit, - as before. + before the store closes, in the order it arrived; before, a memory-only + queue lost it. Once `stop()` begins, no agent turn starts and no stream + registers: a turn that was mid-compile when the streams were cancelled, or + one waiting to retry, is refused rather than left running, and an ephemeral + run refused this way ends at once rather than waiting for a watchdog. Since + no reply can then follow a stored entry in a stream, `stop()` stores each + one even while a frame it doesn't wait for still holds its target's turn: + a turn started outside the event loop, or a puppet call. `stop()` neither + refuses nor waits for a puppet call, as before. One still running stores + its pair when its tool returns. If that's before `stop()` stores the + deferred entries, the pair lands first and the puppet's own flush stores + what was deferred behind it; if it's later, the pair lands after them, and + on a store `stop()` has closed that write fails. A quiesced host keeps what + is deferred in its persisted queue for boot recovery. An entry for a target + no longer registered (a disposed fork) is dropped from a memory-only queue + at exit, as before. - A deferred write whose store throws goes back into the queue under its id, and so does every entry queued behind it for that agent, so none of them lands ahead of an earlier one, and a queued `tool_use` that can't be - stored keeps its `tool_result` from being stored alone. The log line names - the entry at each flush, and one that keeps throwing holds what is deferred - behind it for that agent until it can be stored. The hold is the queue's: - a write that arrives while the agent is idle isn't deferred, so it is - stored at once and can land ahead of a held entry, and an entry that keeps - throwing holds only what is deferred during the agent's turns (or while the - host is quiesced). Before, it stayed in - the un-acked set and the next acknowledgement dropped it from the - persisted queue (at a turn start or a flush it was lost even from memory). - At a tool boundary, nothing from it on is injected. While the framework - runs it is kept; a memory-only queue still loses it when the process - exits. + stored keeps the `tool_result` queued behind it from being stored alone. (A + stored `tool_use` whose `tool_result` then can't be stored stays alone + until the result lands at a later flush.) Other agents' entries go on, in + the order they arrived. The log line names the entry at each flush, and one + that keeps throwing holds what is deferred behind it for that agent until + it can be stored. The hold is the queue's: a write that isn't deferred is + stored at once and can land ahead of a held entry. That's a fresh + `tool_result`, and any write while the agent's turn, stream and tool cycle + are idle and the host isn't quiesced. So an entry that keeps throwing holds + only what is deferred while the agent's turn, stream or tool cycle is + alive, or the host is quiesced. Before, it stayed in the un-acked set and + the next acknowledgement dropped it from the persisted queue (at a turn + start or a flush it was lost even from memory). At a tool boundary, + nothing from it on is injected. While the framework runs it is kept; a + memory-only queue still loses it when the process exits. +- Boot recovery drops a recovered deferred write whose content isn't an + array of block objects, which the store could never take, and logs it by + id and participant, rather than holding it in the queue. Before, an entry + whose content wasn't an array was dropped without a line, and one with a + null block was kept, where it could never be stored. diff --git a/src/framework.ts b/src/framework.ts index 8ebf455df..227cf80ce 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -482,18 +482,12 @@ function withDeferredWriteId(metadata: MessageMetadata | undefined, id: string): } /** A deferred entry as a log line names it: its id, participant and block - * types. It never throws: it describes the entries the store rejected, which - * may be malformed (a block that isn't an object with a type reads as - * `malformed`). */ + * types. Every queued entry's blocks are objects: a fresh write with a null + * block throws in addMessage before it could be queued, and a recovered + * entry whose content isn't an array of objects is dropped at load + * (restorePersistedDeferredWrites). */ function describeDeferredWrite(msg: { id: string; participant: string; content: ContentBlock[] }): string { - const content: unknown = msg.content; - const types = Array.isArray(content) - ? [...new Set(content.map((b: unknown) => { - const type = b && typeof b === 'object' ? (b as { type?: unknown }).type : undefined; - return typeof type === 'string' ? type : 'malformed'; - }))].join('+') || 'empty' - : 'malformed'; - return `${msg.id} (${msg.participant}: ${types})`; + return `${msg.id} (${msg.participant}: ${[...new Set(msg.content.map((b) => b.type))].join('+') || 'empty'})`; } /** Whether a deferred message can join a live stream at a tool boundary: @@ -2294,22 +2288,29 @@ export class AgentFramework { // found the target busy again, or one deferred behind its target's own // state) would wait for a flush that may never come. Land each at its // registered target now, while the store is open; a memory-only queue - // would otherwise lose it at exit. It goes past the deferral: a frame - // stop() doesn't wait for (a turn started outside the event loop, or a - // puppet call) can still hold its target's turn token, but no reply can - // follow the entry in any stream. stop() neither refuses nor waits for a - // puppet call, as before this drain: one still running stores its pair - // when its tool returns, after these entries, and on a store closed - // below that write fails. A quiesced host defers it again - // instead, into the persisted queue, for boot recovery. A write that - // throws stays queued with what follows it (writeDrainedInOrder), and a - // memory-only queue still loses it at exit. An entry for a target no - // longer registered (a disposed fork) is left: dropped with a memory-only - // queue at exit, and logged and dropped from a persisted one at boot. + // would otherwise lose it at exit. Every registered agent's entries go in + // queue order (residents share one message slot), each target cut at its + // own first entry that can't be stored (writeDrainedInOrder). They go + // past the deferral: a frame stop() doesn't wait for (a turn started + // outside the event loop, or a puppet call) can still hold its target's + // turn token, but no reply can follow an entry in any stream. + // + // stop() neither refuses nor waits for a puppet call, as before this + // drain. One still running stores its pair when its tool returns. If + // that's before this drain runs, the pair lands first, and the puppet's + // own flush stores what was deferred behind it. If it's later, the pair + // lands after these entries, and on a store closed below that write + // fails. + // + // A quiesced host defers each entry again instead, into the persisted + // queue, for boot recovery. A write that throws stays queued with what + // follows it for its target, and a memory-only queue still loses it at + // exit. An entry for a target no longer registered (a disposed fork) is + // left: dropped with a memory-only queue at exit, and logged and dropped + // from a persisted one at boot. if (this.deferredMessages.length > 0) { - for (const name of this.agents.keys()) { - this.readdDrained(this.drainDeferredFor(name), 'stop', this.quiesced ? undefined : { bypassDeferralFor: name }); - } + const entries = [...this.agents.keys()].flatMap((name) => this.drainDeferredFor(name)); + this.readdDrained(entries, 'stop', { bypassDeferral: !this.quiesced }); this.ackDeferredWrites(); } @@ -3675,12 +3676,13 @@ export class AgentFramework { * Land every deferred write whose target has no turn alive, one at a time, * acknowledging each durably as it lands (the persisted queue is rewritten * after every message). A crash mid-flush therefore replays exactly the - * un-acked remainder at the next boot — no loss, no duplicates. A write - * the store rejects stays queued, logged, with what is queued behind it for - * its target (writeDrainedInOrder), so the flush never stores a target's - * later entries ahead of it (a write that arrives while its target is idle - * isn't deferred, though, and can land ahead of it); a boot that meets it - * again logs it and goes on. + * un-acked remainder at the next boot — no loss, no duplicates. The batch + * is stored in queue order, whatever its targets (writeDrainedInOrder). A + * write the store rejects stays queued, logged, with what is queued behind + * it for its target, so the flush never stores that target's later entries + * ahead of it, and other targets' entries go on. (A write that isn't + * deferred can still land ahead of it, such as one for an idle agent once + * the host has resumed.) A boot that meets it again logs it and goes on. * * Per-target: an agent whose turn is still alive keeps ITS messages * deferred for its own turn boundary; everyone else's flush now. @@ -3703,15 +3705,7 @@ export class AgentFramework { flush.sort(bySeq); this.deferredMessages = keep; this.handOffDeferredWrites(flush); - // Each target's entries in queue order, cut at the first that can't be - // stored; other targets' entries go on. - const byTarget = new Map(); - for (const msg of flush) { - const target = msg.forAgent ?? this.primaryAgentName; - byTarget.set(target, [...(byTarget.get(target) ?? []), msg]); - } - let stored = 0; - for (const entries of byTarget.values()) stored += this.readdDrained(entries, `host-mode ${label}`).length; + const stored = this.readdDrained(flush, `host-mode ${label}`).length; // Ack = sync the chronicle FIRST, then rewrite the durable queue without // the batch. Never the other way round. this.ackDeferredWrites(); @@ -3914,10 +3908,27 @@ export class AgentFramework { if (typeof v === 'number' && Number.isFinite(v)) scanFrom.set(k, Math.max(0, Math.floor(v))); } } + // An entry the store could never take has no place in the queue, whose + // readers trust each entry's shape: one whose participant isn't text, + // or whose content isn't an array of block objects, is dropped here + // with a line naming it, rather than held. + const wellFormed = (m: unknown): m is Omit & { id?: string; seq?: number } => + !!m && typeof m === 'object' && typeof (m as { participant?: unknown }).participant === 'string' + && Array.isArray((m as { content?: unknown }).content) + && (m as { content: unknown[] }).content.every((b) => !!b && typeof b === 'object'); + const malformed = data.filter((m) => !wellFormed(m)); + for (const m of malformed) { + const field = (key: string): string => { + const value = m && typeof m === 'object' ? (m as Record)[key] : undefined; + return typeof value === 'string' ? value : `no ${key}`; + }; + console.error( + `[host-mode] a recovered deferred context write is malformed and was dropped: ` + + `${field('id')} (${field('participant')})`, + ); + } const candidates = data - .filter((m): m is Omit & { id?: string; seq?: number } => - !!m && typeof m === 'object' && typeof (m as { participant?: unknown }).participant === 'string' - && Array.isArray((m as { content?: unknown }).content)) + .filter(wellFormed) .map((m, i) => ({ ...m, id: typeof m.id === 'string' ? m.id : randomUUID(), @@ -3942,8 +3953,9 @@ export class AgentFramework { this.deferredMessages.push(...restored); this.deferredWritesPersisted = true; // Rewrite the durable queue without the already-landed entries so a - // second interrupted boot does not re-scan them. - if (landed.size > 0) this.persistDeferredWrites(); + // second interrupted boot does not re-scan them, and without the + // malformed ones. + if (landed.size > 0 || malformed.length > 0) this.persistDeferredWrites(); return restored.length; } catch (err) { console.error('[host-mode] failed to restore persisted deferred context writes:', err); @@ -6296,12 +6308,12 @@ export class AgentFramework { } // Writers that deferred behind the reservation land now, on the branch // the operator chose (or the restored source) — the same end-of-turn - // flush driveStream/puppet perform, under the same guard. + // flush driveStream/puppet perform, under the same guard — for every + // agent free to take them, in queue order (writeDrainedInOrder). if (this.deferredMessages.length > 0) { - for (const name of reserved.keys()) { - if (this.activeTurnTokens.has(name) || this.pendingAssistantBlocks.has(name)) continue; - this.readdDrained(this.drainDeferredFor(name), 'post-surgery flush'); - } + const free = [...reserved.keys()] + .filter((name) => !this.activeTurnTokens.has(name) && !this.pendingAssistantBlocks.has(name)); + this.readdDrained(free.flatMap((name) => this.drainDeferredFor(name)), 'post-surgery flush'); this.ackDeferredWrites(); } }; @@ -13255,30 +13267,6 @@ export class AgentFramework { return drained; } - /** - * Write one drained entry. If the write throws, it is logged, naming the - * entry, and goes back into the queue under its id and seq - * (requeueDrained): left in the un-acked set, the next ack would drop it - * from the durable queue. Every site writes a drained list through - * writeDrainedInOrder, which cuts there. Returns whether it was written - * (stored, or deferred again by addMessage). - */ - private writeDrained(msg: DeferredWrite, where: string, write: () => void): boolean { - try { - write(); - return true; - } catch (error) { - // Back into the queue before anything else can go wrong here. - this.requeueDrained([msg]); - console.error( - `[deferral] ${where}: deferred write ${describeDeferredWrite(msg)} for ` + - `${msg.forAgent ?? this.primaryAgentName ?? 'the primary'} could not be stored; it stays queued, ` + - `holding what is queued behind it for that agent: ${error instanceof Error ? error.message : String(error)}`, - ); - return false; - } - } - /** * Put drained entries back into the queue under their ids and seq: out of * the un-acked set, and persisted when the queue is. A memory-only queue @@ -13295,41 +13283,80 @@ export class AgentFramework { } /** - * Write one agent's drained entries in queue order, each through - * writeDrained. The drain cuts at the first that can't be stored: it and - * every later entry go back into the queue, so none of them lands ahead of - * an earlier one, and a queued tool_use that can't be stored keeps its - * tool_result from being stored alone. (A tool_use that is stored, followed - * by a tool_result that can't be, stays stored without it until the - * tool_result lands at a later flush. The only queued pair is - * puppetToolCall's defensive one, which no path reaches today.) An - * entry that keeps failing holds its agent's later queued writes, logged - * at each flush, rather than letting them land out of order. The hold is - * the queue's alone: a write that arrives while its agent is idle isn't - * deferred (addMessage stores it at once), so it can land ahead of a held - * entry, and an entry that keeps failing holds only what is deferred - * behind it while its agent's turn, stream or tool cycle is alive, or the - * host is quiesced. Returns what was written, a prefix of `entries`. + * Write drained entries in queue (seq) order, whatever their targets: + * residents share one message slot, so the order entries arrived in is the + * order they're stored. Each target is cut at its own first entry that + * can't be stored: that entry and every later one for the same target go + * back into the queue under their ids and seq (requeueDrained) before + * anything is logged, and other targets' entries go on. Left in the + * un-acked set, the next ack would drop them from the durable queue. + * + * So none of a target's entries lands ahead of an earlier one, and a queued + * tool_use that can't be stored keeps the tool_result queued behind it from + * being stored alone. (A tool_use that is stored, followed by a tool_result + * that can't be, stays stored without it until the tool_result lands at a + * later flush. The only queued pair is puppetToolCall's defensive one, + * which no path reaches today.) An entry that keeps failing holds its + * target's later queued writes, logged by id at each flush, rather than + * letting them land out of order. + * + * The hold is the queue's alone. A write that isn't deferred is stored at + * once and can land ahead of a held entry: a fresh tool_result, whatever + * its agent's state, and any write while its agent's turn, stream and tool + * cycle are all idle and the host isn't quiesced. So an entry that keeps + * failing holds only what is deferred behind it. + * + * Returns what was written (stored, or deferred again by addMessage), in + * queue order: for one target, a prefix of its entries. */ private writeDrainedInOrder(entries: DeferredWrite[], where: string, write: (msg: DeferredWrite) => void): DeferredWrite[] { - for (let i = 0; i < entries.length; i++) { - const msg = entries[i]!; - if (!this.writeDrained(msg, where, () => write(msg))) { - this.requeueDrained(entries.slice(i + 1)); - return entries.slice(0, i); + const ordered = [...entries].sort(bySeq); + const targetOf = (msg: DeferredWrite): string | null => msg.forAgent ?? this.primaryAgentName; + const cut = new Set(); + const written: DeferredWrite[] = []; + for (let i = 0; i < ordered.length; i++) { + const msg = ordered[i]!; + const target = targetOf(msg); + if (cut.has(target)) continue; + try { + write(msg); + written.push(msg); + } catch (error) { + cut.add(target); + this.requeueDrained(ordered.slice(i).filter((m) => targetOf(m) === target)); + this.logStuckDeferredWrite(msg, where, error); } } - return entries; + return written; } - /** Re-add one agent's drained entries through addMessage, which stores each - * or defers it again, cut at the first that can't be stored. */ - private readdDrained(entries: DeferredWrite[], where: string, opts?: { bypassDeferralFor?: string }): DeferredWrite[] { + /** The line naming a drained write that couldn't be stored. The entries are + * already back in the queue, and logging mustn't fail the flush that met + * it, so a thrown value that has no text is named as such. */ + private logStuckDeferredWrite(msg: DeferredWrite, where: string, error: unknown): void { + let reason: string; + try { + reason = error instanceof Error ? error.message : String(error); + } catch { + reason = 'a thrown value that has no text'; + } + console.error( + `[deferral] ${where}: deferred write ${describeDeferredWrite(msg)} for ` + + `${msg.forAgent ?? this.primaryAgentName ?? 'the primary'} could not be stored; it stays queued, ` + + `holding what is queued behind it for that agent: ${reason}`, + ); + } + + /** Re-add drained entries through addMessage, which stores each or defers + * it again, in queue order and cut per target (writeDrainedInOrder). With + * `bypassDeferral`, each is stored past its target's deferral (stop()). */ + private readdDrained(entries: DeferredWrite[], where: string, opts?: { bypassDeferral?: boolean }): DeferredWrite[] { return this.writeDrainedInOrder(entries, where, (msg) => { + const target = msg.forAgent ?? this.primaryAgentName; this.addMessage(msg.participant, msg.content, msg.metadata, { deferredWriteId: msg.id, ...(msg.forAgent ? { forAgent: msg.forAgent } : {}), - ...(opts?.bypassDeferralFor ? { bypassDeferralFor: opts.bypassDeferralFor } : {}), + ...(opts?.bypassDeferral && target ? { bypassDeferralFor: target } : {}), }); }); } diff --git a/test/deferred-window-only.test.ts b/test/deferred-window-only.test.ts index e0039d2d3..fe1b9fc47 100644 --- a/test/deferred-window-only.test.ts +++ b/test/deferred-window-only.test.ts @@ -482,7 +482,32 @@ describe('mid-turn deliveries keep the window what the stream carried', () => { await reopened.stop(); }); - it('a turn waiting to retry when the framework stops gives up rather than compile again', async () => { + it("stop() stores several agents' deferred writes in the order they arrived", async () => { + // Residents share one message slot, so the order writes arrived in is the + // order every resident reads. + const agents = [ + { name: 'assistant', model: 'test-model', systemPrompt: 'You are a robot pilot.' }, + { name: 'scout', model: 'test-model', systemPrompt: 'You scout.' }, + ]; + const framework = await createFramework({ agents }); + const internals = internalsOf(framework); + // Frames stop() doesn't wait for hold both agents' turns. + internals.activeTurnTokens.set('assistant', 1); + internals.activeTurnTokens.set('scout', 2); + assert.equal(internals.addMessage('Antra', text('one')), '', 'deferred'); + assert.equal(internals.addMessage('Antra', text('two'), undefined, { forAgent: 'scout' }), '', 'deferred'); + assert.equal(internals.addMessage('Antra', text('three')), '', 'deferred'); + await framework.stop(); + + const reopened = await createFramework({ agents }); + assert.deepEqual( + windowOf(reopened).filter((line) => /^Antra: (one|two|three)$/.test(line)), + ['Antra: one', 'Antra: two', 'Antra: three'], + ); + await reopened.stop(); + }); + + it('a turn waiting to retry when the framework stops gives up rather than compile again', { timeout: 60_000 }, async () => { let waiting!: () => void; const inDelay = new Promise((resolve) => { waiting = resolve; }); const framework = await createFramework({ @@ -504,13 +529,14 @@ describe('mid-turn deliveries keep the window what the stream carried', () => { // The retry re-enters startAgentStream after its delay; count those // re-entries, so a retry that was never granted can't pass for one that // was refused. - const retries: number[] = []; + const retries: Array<{ attempt: number; stopping: boolean }> = []; const frames = framework as unknown as { + providerAdmissionClosed: boolean; startAgentStream(agent: unknown, trigger?: unknown, attempt?: number, held?: boolean): Promise; }; const startAgentStream = frames.startAgentStream.bind(framework); frames.startAgentStream = (a, trigger, attempt = 0, held = false) => { - if (attempt > 0) retries.push(attempt); + if (attempt > 0) retries.push({ attempt, stopping: frames.providerAdmissionClosed }); return startAgentStream(a, trigger, attempt, held); }; framework.start(); @@ -520,7 +546,7 @@ describe('mid-turn deliveries keep the window what the stream carried', () => { const error = console.error; console.error = () => {}; try { await framework.stop(); } finally { console.error = error; } - assert.deepEqual(retries, [1], 'the retry was granted, and re-entered once stop() had begun'); + assert.deepEqual(retries, [{ attempt: 1, stopping: true }], 'the retry was granted, and re-entered once stop() had begun'); assert.equal(compiles, 1, 'the retry was refused, not compiled'); const reopened = await createFramework(); diff --git a/test/ephemeral-settle.test.ts b/test/ephemeral-settle.test.ts index 21765f833..8ebe8d870 100644 --- a/test/ephemeral-settle.test.ts +++ b/test/ephemeral-settle.test.ts @@ -129,7 +129,7 @@ async function stopAndRemove(framework: AgentFramework, tempDir: string): Promis } describe('runEphemeralToCompletion settle signal', () => { - it('a run whose turn stop() refuses ends at once, and a run asked for once stop() has begun is refused', async () => { + it('a run whose turn stop() refuses ends at once, and a run asked for once stop() has begun is refused', { timeout: 60_000 }, async () => { const { tempDir, storePath } = tempStorePath('ephemeral-stopping-'); const membrane = new MockMembrane(); membrane.pushResponse(createMockResponse([{ type: 'text', text: 'too late' }])); @@ -170,7 +170,7 @@ describe('runEphemeralToCompletion settle signal', () => { } }); - it('a run waiting to retry when stop() begins gives up before compiling again', async () => { + it('a run waiting to retry when stop() begins gives up before compiling again', { timeout: 60_000 }, async () => { // An ephemeral run doesn't hold the primary's provider gate, so the // refusal at the top of startAgentStream is the only check its retry // meets before it would compile, and start a stream, again. diff --git a/test/host-quiesce.test.ts b/test/host-quiesce.test.ts index 75ff5fbb6..fe3b81676 100644 --- a/test/host-quiesce.test.ts +++ b/test/host-quiesce.test.ts @@ -722,7 +722,7 @@ test('resume flushes deferred writes in order: a poison write stays queued with }); }); -test("resume cuts each target at its own stuck write, naming it at every flush, and other targets' writes land", async () => { +test("resume stores several agents' writes in the order they arrived, and cuts each agent at its own stuck write, named at every flush", { timeout: 60_000 }, async () => { const dir = mkdtempSync(join(tmpdir(), 'host-quiesce-targets-')); const framework = await AgentFramework.create({ storePath: join(dir, 'store'), @@ -739,46 +739,48 @@ test("resume cuts each target at its own stuck write, naming it at every flush, try { await framework.quiesce({ reason: 'refold' }); const internals = framework as unknown as { - deferredSeq: number; - deferredMessages: Array<{ id: string; seq: number; participant: string; content: unknown[]; forAgent?: string }>; + deferredMessages: Array<{ id: string; content: Array<{ text?: string }> }>; addMessage(participant: string, content: unknown[], metadata?: undefined, opts?: { forAgent?: string }): unknown; }; + const add = (text: string, forAgent: string) => + internals.addMessage('user', [{ type: 'text', text }], undefined, { forAgent }); // Residents share one message slot, so each one's context manager reads // what either stored; each stores through its own. const agentCm = framework.getAgent('agent')!.getContextManager(); const before = agentCm.getAllMessages().length; const stored = () => agentCm.getAllMessages().slice(before).map((m) => (m.content[0] as { text: string }).text); - // A malformed entry the store rejects, as only a queue persisted by other - // code could carry (addMessage refuses a null block): the log line still - // names it, and it still goes back into the queue. - internals.deferredMessages.push({ - id: 'stuck-1', seq: ++internals.deferredSeq, participant: 'user', - content: [{ type: 'text', text: 'stuck' }, null], forAgent: 'agent', - }); - internals.addMessage('user', [{ type: 'text', text: 'for scout' }], undefined, { forAgent: 'scout' }); - internals.addMessage('user', [{ type: 'text', text: 'behind the stuck one' }], undefined, { forAgent: 'agent' }); - internals.addMessage('user', [{ type: 'text', text: 'also for scout' }], undefined, { forAgent: 'scout' }); - assert.equal(framework.getHostModeStatus().deferredWrites, 4); - + add('one', 'agent'); + add('two', 'scout'); + add('three', 'agent'); + add('stuck', 'agent'); + add('for scout', 'scout'); + add('behind the stuck one', 'agent'); + add('also for scout', 'scout'); + assert.equal(framework.getHostModeStatus().deferredWrites, 7); + const stuckId = internals.deferredMessages.find((m) => m.content[0]?.text === 'stuck')!.id; + + // The store refuses agent's 'stuck' write with a thrown value that has no + // text, so the log line can't print it. const realAdd = agentCm.addMessage.bind(agentCm); (agentCm as unknown as { addMessage: typeof agentCm.addMessage }).addMessage = ((participant, content, metadata, causedBy) => { - if ((content[0] as { text?: string }).text === 'stuck') throw new Error('store write failed'); + if ((content[0] as { text?: string }).text === 'stuck') throw Object.create(null); return realAdd(participant, content, metadata, causedBy); }) as typeof agentCm.addMessage; const named = () => lines.filter((line) => - line.includes('deferred write stuck-1 (user: text+malformed) for agent could not be stored')).length; + line.includes(`deferred write ${stuckId} (user: text) for agent could not be stored`) + && line.includes('a thrown value that has no text')).length; let status = await framework.resume(); - // Queued after agent's stuck write, scout's land; agent's own later - // write is held behind it. - assert.deepEqual(stored(), ['for scout', 'also for scout'], "scout's writes land; agent's stuck write holds what is queued behind it for agent"); - assert.equal(status.deferredWrites, 2); + // In the order they arrived, across both agents; from agent's stuck + // write on, agent's are held, and scout's go on. + assert.deepEqual(stored(), ['one', 'two', 'three', 'for scout', 'also for scout']); + assert.equal(status.deferredWrites, 2, "agent's stuck write and the one behind it are still queued"); assert.equal(named(), 1, 'the flush names the stuck write: its id and block types'); // A second flush with the store still refusing it: held again, named again. await framework.quiesce({ reason: 'refold' }); status = await framework.resume(); - assert.deepEqual(stored(), ['for scout', 'also for scout'], 'still nothing lands ahead of it'); + assert.deepEqual(stored(), ['one', 'two', 'three', 'for scout', 'also for scout'], 'still nothing lands ahead of it'); assert.equal(status.deferredWrites, 2); assert.equal(named(), 2, 'every flush names it'); } finally { @@ -791,6 +793,57 @@ test("resume cuts each target at its own stuck write, naming it at every flush, } }); +test('boot recovery drops a recovered deferred write the store could never take, naming it, and restores the rest', { timeout: 60_000 }, async () => { + const dir = mkdtempSync(join(tmpdir(), 'host-quiesce-malformed-')); + const storePath = join(dir, 'store'); + const config = () => ({ + storePath, + membrane: new MockMembrane().asMembrane(), + agents: [{ name: 'agent', model: 'test-model', systemPrompt: 'test' }], + modules: [], + }); + let framework = await AgentFramework.create(config()); + const lines: string[] = []; + const error = console.error; + try { + await framework.quiesce({ reason: 'surgery' }); + const before = framework.getAgent('agent')!.getContextManager().getAllMessages().length; + const add = (framework as unknown as { addMessage(p: string, c: unknown[]): unknown }).addMessage.bind(framework); + add('user', [{ type: 'text', text: 'first' }]); + add('user', [{ type: 'text', text: 'second' }]); + await framework.stop(); + + // Entries no write of the framework's makes, as other code could leave + // them in the recovery file: a null block, and content that isn't blocks. + const file = join(storePath, 'recovery', 'deferred-writes.json'); + const doc = JSON.parse(readFileSync(file, 'utf8')) as { pending: Array> }; + assert.equal(doc.pending.length, 2); + doc.pending.splice(1, 0, + { id: 'null-block', seq: 100, participant: 'user', content: [{ type: 'text', text: 'x' }, null] }, + { id: 'not-blocks', seq: 101, participant: 'user', content: 'x' }, + ); + writeFileSync(file, JSON.stringify(doc)); + + console.error = (...args: unknown[]) => { lines.push(args.map(String).join(' ')); }; + framework = await AgentFramework.create(config()); + console.error = error; + assert.equal(framework.getHostModeStatus().deferredWrites, 2, 'the well-formed entries are restored'); + for (const entry of ['null-block (user)', 'not-blocks (user)']) { + assert.ok(lines.some((line) => line.includes(`malformed and was dropped: ${entry}`)), `${entry} is named:\n${lines.join('\n')}`); + } + const rewritten = readFileSync(file, 'utf8'); + assert.ok(!rewritten.includes('null-block') && !rewritten.includes('not-blocks'), 'the recovery file is rewritten without them'); + await framework.resume(); + const texts = framework.getAgent('agent')!.getContextManager().getAllMessages().slice(before) + .map((m) => (m.content[0] as { text: string }).text); + assert.deepEqual(texts, ['first', 'second']); + } finally { + console.error = error; + await framework.stop(); + rmSync(dir, { recursive: true, force: true }); + } +}); + test('context writes deferred while quiesced survive a restart and land at resume', async () => { const dir = mkdtempSync(join(tmpdir(), 'host-quiesce-deferred-')); const storePath = join(dir, 'store'); diff --git a/test/operator-surgery.test.ts b/test/operator-surgery.test.ts index d38379d17..dd732b96f 100644 --- a/test/operator-surgery.test.ts +++ b/test/operator-surgery.test.ts @@ -257,7 +257,7 @@ describe('live operator surgery', () => { } }); - it("the post-surgery flush lands what waited behind the hold while another agent's tool cycle is pending, and acknowledges it", async () => { + it("the post-surgery flush lands what waited behind the hold while another agent's tool cycle is pending, and acknowledges it", { timeout: 60_000 }, async () => { const c = cm() as unknown as { switchBranch: (name: string) => Promise }; const originalSwitch = c.switchBranch; let releaseSwitch!: () => void; @@ -290,6 +290,43 @@ describe('live operator surgery', () => { } }); + it("the post-surgery flush stores several agents' writes in the order they arrived", { timeout: 60_000 }, async () => { + // Residents share one message slot, so the order writes arrived in is the + // order every resident reads. + await framework.stop(); + framework = await AgentFramework.create({ + storePath, + membrane: membrane.asMembrane(), + agents: [ + { name: 'scout', model: 'test-model', systemPrompt: 'You are scout.' }, + { name: 'builder', model: 'test-model', systemPrompt: 'You are builder.' }, + ], + modules: [], + }); + const c = cm() as unknown as { switchBranch: (name: string) => Promise }; + const originalSwitch = c.switchBranch; + let releaseSwitch!: () => void; + const gate = new Promise((resolve) => { releaseSwitch = resolve; }); + let entered!: () => void; + const inSwitch = new Promise((resolve) => { entered = resolve; }); + c.switchBranch = async (name: string) => { entered(); await gate; return originalSwitch.call(c, name); }; + const add = (framework as unknown as { + addMessage(participant: string, content: unknown[], metadata?: undefined, opts?: { forAgent?: string }): string; + }).addMessage.bind(framework); + try { + const rollback = framework.rollbackToMessage('scout', { messageId: ids[2] }); + await inSwitch; + assert.equal(add('user', [{ type: 'text', text: 'one' }]), '', 'deferred'); + assert.equal(add('user', [{ type: 'text', text: 'two' }], undefined, { forAgent: 'builder' }), '', 'deferred'); + assert.equal(add('user', [{ type: 'text', text: 'three' }]), '', 'deferred'); + releaseSwitch(); + await rollback; + assert.deepEqual(remainingTexts().filter((t) => ['one', 'two', 'three'].includes(t)), ['one', 'two', 'three']); + } finally { + c.switchBranch = originalSwitch; + } + }); + // --- strategy-initialization failure ----------------------------------------- // switchBranch()/fork() move the chronicle branch BEFORE awaiting strategy // initialization, so a rejection there leaves the store on the new branch. From 0ed2f4c48fdeb42ecb8e4708bc0264eb6393cf3c Mon Sep 17 00:00:00 2001 From: Duncan-2410 Date: Tue, 13 Oct 2026 13:20:01 -0400 Subject: [PATCH 09/34] fix(deferral): a re-deferred entry keeps its seq whether or not the queue is persisted Found by both readers of the previous round: Tobias-2430 with a helper fixture, Ansel-2428 end to end with a real store and a reopen. A drained entry that addMessage defers again took its seq from the un-acked set, which handOffDeferredWrites fills only for a persisted queue. With the usual memory-only queue it got a fresh seq, so a turn-end flush that met a successor's turn moved the agent's entries behind later arrivals for other agents, and the next flush of several agents' entries (stop(), resume, after surgery) stored them out of the order they arrived. addMessage's deferredWriteId option becomes fromQueue: { id, seq }. A write coming back from the queue is re-deferred under its own id and seq, and leaves the un-acked set as before. readdDrained is its only caller. Tests (deferred-window-only): the two-resident stop() test first re-defers the assistant's entries through a turn-end flush while its turn is held, and still expects one, two, three; the re-deferral test asserts the seq it passed. The fragment says a re-deferred write keeps its place in arrival order. Not yet typechecked or run: written as the house stop began (room-203 #88162), which starts no builds. Co-Authored-By: Claude Opus 5.5 --- .../mid-turn-window-only-held.fixed.md | 7 ++-- src/framework.ts | 35 +++++++++---------- test/deferred-window-only.test.ts | 20 +++++++---- 3 files changed, 35 insertions(+), 27 deletions(-) diff --git a/changelog.d/mid-turn-window-only-held.fixed.md b/changelog.d/mid-turn-window-only-held.fixed.md index 698cb1cb1..3208b9e5d 100644 --- a/changelog.d/mid-turn-window-only-held.fixed.md +++ b/changelog.d/mid-turn-window-only-held.fixed.md @@ -19,9 +19,10 @@ branch the watermark only advances: a compile that started before such a mark doesn't roll it back when it finishes. - A deferred write that goes back into the queue does so whatever its - content. That happens while its target's own turn, stream or tool cycle is - alive, or the host is quiesced. A `tool_result` used to land regardless, - apart from the `tool_use` it answers, which went back into the queue. + content, under its id and in its place in the order writes arrived. That + happens while its target's own turn, stream or tool cycle is alive, or the + host is quiesced. A `tool_result` used to land regardless, apart from the + `tool_use` it answers, which went back into the queue. - A write that waits now waits on its own target's state: the target's turn, stream and tool cycle, or a quiesced host. (A fresh `tool_result` still doesn't wait.) Before, a write for the primary agent waited out every diff --git a/src/framework.ts b/src/framework.ts index 227cf80ce..672bfecbb 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -13135,13 +13135,14 @@ export class AgentFramework { */ forAgent?: string; /** - * Set when the message is being flushed FROM the durable deferred - * queue: it is the entry's one durable identity. A write stamps it - * into `metadata.deferredWriteId` (boot dedup); a re-deferral moves - * the same logical entry back to pending under the same id instead of - * minting a second, independently replayable one. + * Set when the message comes back from the deferred queue: the entry's + * one durable identity and its place in arrival order. A write stamps + * the id into `metadata.deferredWriteId` (boot dedup). A re-deferral + * moves the same logical entry back to pending under the same id and + * seq, instead of minting a second, independently replayable one, or + * moving it behind entries that arrived after it. */ - deferredWriteId?: string; + fromQueue?: { id: string; seq: number }; /** Out-param: where the message landed (RFC-006 needs to find it again). */ placement?: { agent?: string; messageId?: MessageId; deferredId?: string }; /** @@ -13190,12 +13191,12 @@ export class AgentFramework { // append mid-surgery. Deferred here, flushed by resume(). tool_result // still lands (a draining turn's continuation depends on it). // - // A write coming back from the deferred queue (deferredWriteId) is + // A write coming back from the deferred queue (fromQueue) is // re-deferred whatever its content: its tool_use went back into the // queue just before it, in the same drain, and a tool_result stored // without it would split the pair. (A turn-end flush can run after a // successor turn has taken the agent: the 2026-07-31 interleave.) - const freshToolResult = opts?.deferredWriteId === undefined && content.some(b => b.type === 'tool_result'); + const freshToolResult = opts?.fromQueue === undefined && content.some(b => b.type === 'tool_result'); // One rule for every write, primary-bound or targeted: it waits while ITS // TARGET's own tool cycle is pending (pendingAssistantBlocks is keyed by // agent). Another agent's cycle says nothing about this window. The @@ -13215,16 +13216,14 @@ export class AgentFramework { this.activeTurnTokens.has(agent.name) || this.activeStreams.has(agent.name)) ) { - // A re-deferral of a drained entry keeps its id and leaves the un-acked - // set: one logical message, one durable identity — never two entries. - const id = opts?.deferredWriteId ?? randomUUID(); - let seq: number | undefined; - if (opts?.deferredWriteId) { - const prior = this.unackedDeferredWrites.find((m) => m.id === id); - seq = prior?.seq; + // A re-deferral of a drained entry keeps its id and seq, and leaves the + // un-acked set: one logical message, one durable identity, one place + // in arrival order, whether or not the queue is persisted. + const id = opts?.fromQueue?.id ?? randomUUID(); + if (opts?.fromQueue) { this.unackedDeferredWrites = this.unackedDeferredWrites.filter((m) => m.id !== id); } - this.deferredMessages.push({ id, seq: seq ?? ++this.deferredSeq, participant, content, metadata, forAgent: opts?.forAgent }); + this.deferredMessages.push({ id, seq: opts?.fromQueue?.seq ?? ++this.deferredSeq, participant, content, metadata, forAgent: opts?.forAgent }); if (this.quiesced || this.deferredWritesPersisted) this.persistDeferredWrites(); if (opts?.placement) { opts.placement.agent = agent.name; opts.placement.deferredId = id; } return '' as MessageId; // Deferred — flushed at the target's next boundary @@ -13233,7 +13232,7 @@ export class AgentFramework { const stored = agent.getContextManager().addMessage( participant, content, - opts?.deferredWriteId ? withDeferredWriteId(metadata, opts.deferredWriteId) : metadata, + opts?.fromQueue ? withDeferredWriteId(metadata, opts.fromQueue.id) : metadata, ); if (opts?.placement) { opts.placement.agent = agent.name; opts.placement.messageId = stored; } return stored; @@ -13354,7 +13353,7 @@ export class AgentFramework { return this.writeDrainedInOrder(entries, where, (msg) => { const target = msg.forAgent ?? this.primaryAgentName; this.addMessage(msg.participant, msg.content, msg.metadata, { - deferredWriteId: msg.id, + fromQueue: { id: msg.id, seq: msg.seq }, ...(msg.forAgent ? { forAgent: msg.forAgent } : {}), ...(opts?.bypassDeferral && target ? { bypassDeferralFor: target } : {}), }); diff --git a/test/deferred-window-only.test.ts b/test/deferred-window-only.test.ts index fe1b9fc47..485de9487 100644 --- a/test/deferred-window-only.test.ts +++ b/test/deferred-window-only.test.ts @@ -270,10 +270,11 @@ describe('mid-turn deliveries keep the window what the stream carried', () => { pendingAssistantBlocks: Map; activeTurnTokens: Map; activeStreams: Map; - deferredMessages: Array<{ id: string; participant: string }>; + deferredMessages: Array<{ id: string; seq: number; participant: string }>; deferredWritesPersisted: boolean; drainDeferredFor: (name: string, stopAt?: unknown) => unknown[]; - addMessage(participant: string, content: ContentBlock[], metadata?: undefined, opts?: { forAgent?: string; deferredWriteId?: string }): string; + readdDrained: (entries: unknown[], where: string) => unknown[]; + addMessage(participant: string, content: ContentBlock[], metadata?: undefined, opts?: { forAgent?: string; fromQueue?: { id: string; seq: number } }): string; }; } const otherCycle = (): ContentBlock[] => [{ type: 'tool_use', id: 'b1', name: 'robot--move', input: {} } as ContentBlock]; @@ -497,6 +498,11 @@ describe('mid-turn deliveries keep the window what the stream carried', () => { assert.equal(internals.addMessage('Antra', text('one')), '', 'deferred'); assert.equal(internals.addMessage('Antra', text('two'), undefined, { forAgent: 'scout' }), '', 'deferred'); assert.equal(internals.addMessage('Antra', text('three')), '', 'deferred'); + // A turn-end flush of the assistant's entries while a successor holds its + // turn defers them again (the 2026-07-31 interleave). They keep their + // places in arrival order. + internals.readdDrained(internals.drainDeferredFor('assistant'), 'turn-end flush'); + assert.equal(internals.deferredMessages.length, 3, 'deferred again'); await framework.stop(); const reopened = await createFramework({ agents }); @@ -662,12 +668,14 @@ describe('mid-turn deliveries keep the window what the stream carried', () => { // successor turn has started). const requeued = internals.addMessage('user', [ { type: 'tool_result', toolUseId: 'p1', content: 'ok' } as ContentBlock, - ], undefined, { deferredWriteId: 'w1' }); + ], undefined, { fromQueue: { id: 'w1', seq: 41 } }); assert.equal(requeued, ''); - assert.ok(internals.deferredMessages.some((m) => m.id === 'w1')); + // It goes back under its id and its place in arrival order, though this + // queue is held in memory only. + assert.ok(internals.deferredMessages.some((m) => m.id === 'w1' && m.seq === 41)); // So does plain text. - assert.equal(internals.addMessage('Antra', text('plain'), undefined, { deferredWriteId: 'w2' }), ''); - assert.ok(internals.deferredMessages.some((m) => m.id === 'w2')); + assert.equal(internals.addMessage('Antra', text('plain'), undefined, { fromQueue: { id: 'w2', seq: 42 } }), ''); + assert.ok(internals.deferredMessages.some((m) => m.id === 'w2' && m.seq === 42)); // A fresh tool_result write keeps landing as before. const fresh = internals.addMessage('user', [ { type: 'tool_result', toolUseId: 'p2', content: 'ok' } as ContentBlock, From dee13683b4766e45556552bd30c673193e206751 Mon Sep 17 00:00:00 2001 From: Duncan-2410 Date: Tue, 13 Oct 2026 20:56:04 -0400 Subject: [PATCH 10/34] fix(deferral): a turn stop() refuses before its compile ends its ephemeral run too #281, merged from main in the previous commit, refuses a turn before its compile once stop() has closed provider admission, beside the existing identity check. That refusal returned without ending an ephemeral run waiting on the turn, so the run waited for its idle watchdog (fifteen minutes by default) where this branch ends a run refused at the top of startAgentStream or after the compile at once, with "the framework is stopping". It now ends there too, through endEphemeralRunRefused, which is a no-op for an agent that was replaced. Test (ephemeral-settle): a module's gatherContext holds the run's turn until stop() has begun; the run rejects with the refusal and the agent never compiled. On the merge's source the run instead ends by its 20 s idle watchdog, whose message the test's pattern doesn't match. Co-Authored-By: Claude Opus 5.5 --- src/framework.ts | 1 + test/ephemeral-settle.test.ts | 41 +++++++++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+) diff --git a/src/framework.ts b/src/framework.ts index a701a9c26..d2f2eeba4 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -10006,6 +10006,7 @@ export class AgentFramework { if (this.agents.get(agent.name) !== agent || this.providerAdmissionClosed) { this.channelRegistry?.stopTyping(); this.eventGate?.onInferenceEnded(agent.name); + if (this.providerAdmissionClosed) this.endEphemeralRunRefused(agent); return false; } diff --git a/test/ephemeral-settle.test.ts b/test/ephemeral-settle.test.ts index 8ebe8d870..dcb73a0db 100644 --- a/test/ephemeral-settle.test.ts +++ b/test/ephemeral-settle.test.ts @@ -170,6 +170,47 @@ describe('runEphemeralToCompletion settle signal', () => { } }); + it('a run whose turn stop() refuses before its compile ends at once', { timeout: 60_000 }, async () => { + // The turn waits on its modules' context when stop() begins, so it is + // refused before it compiles, not after. + const { tempDir, storePath } = tempStorePath('ephemeral-stopping-precompile-'); + let gathering!: () => void; + const inGather = new Promise((resolve) => { gathering = resolve; }); + let release!: () => void; + const released = new Promise((resolve) => { release = resolve; }); + const contextModule: Module = new EphemeralToolModule(); + contextModule.gatherContext = async () => { + gathering(); + await released; + return []; + }; + const framework = await AgentFramework.create({ + storePath, + membrane: new MockMembrane().asMembrane(), + agents: [], + modules: [contextModule], + syncIntervalMs: 0, + }); + try { + const { agent, contextManager } = await createEphemeral(framework); + let compiles = 0; + const start = agent.startStreamWithInjections.bind(agent); + (agent as unknown as { startStreamWithInjections: typeof start }).startStreamWithInjections = async (...args) => { + compiles++; + return start(...args); + }; + const run = runStartedEphemeral(framework, agent, contextManager, STOPPED_RUN_WATCHDOGS); + await inGather; + const stopped = framework.stop(); + release(); + await assert.rejects(run, /refused: the framework is stopping/); + await stopped; + assert.equal(compiles, 0, 'the turn was refused before its compile'); + } finally { + rmSync(tempDir, { recursive: true, force: true }); + } + }); + it('a run waiting to retry when stop() begins gives up before compiling again', { timeout: 60_000 }, async () => { // An ephemeral run doesn't hold the primary's provider gate, so the // refusal at the top of startAgentStream is the only check its retry From bc267743fd99a5e664fb153fb1cbaa09bc90efe2 Mon Sep 17 00:00:00 2001 From: Duncan-2410 Date: Tue, 13 Oct 2026 20:56:04 -0400 Subject: [PATCH 11/34] test(deferral): the interleave holds in both queue modes; say which blocks the log line can't read and what a persisted queue does with a stuck write From both readers of c9d612e (Ansel-2428, Tobias-2430). - deferred-window-only: the two-resident stop() test, which first re-defers the assistant's entries through a turn-end flush, now runs with a memory-only queue and with a persisted one. The persisted queue kept each entry's seq before 0ed2f4c too (its un-acked set held it), so that run is the control for the memory-only one. - describeDeferredWrite's doc said every queued entry's blocks are objects. addMessage reads each block's type before it defers a fresh write, which throws only on a null or undefined block; a primitive block gets past it and can be queued, and is named with no type. The doc now says that no queued entry has a null or undefined block, the one kind the log line can't read, and why. No new validation. - The fragment says what a persisted queue (after a quiesce, or once boot recovery restored one) does with a write that keeps failing: it outlives restarts, each boot restores it and tries it again with the rest of the queue, the queue stays persisted while it's held, and only the host log names it. Co-Authored-By: Claude Opus 5.5 --- .../mid-turn-window-only-held.fixed.md | 7 ++- src/framework.ts | 8 +-- test/deferred-window-only.test.ts | 63 ++++++++++--------- 3 files changed, 44 insertions(+), 34 deletions(-) diff --git a/changelog.d/mid-turn-window-only-held.fixed.md b/changelog.d/mid-turn-window-only-held.fixed.md index 3208b9e5d..fa897bbae 100644 --- a/changelog.d/mid-turn-window-only-held.fixed.md +++ b/changelog.d/mid-turn-window-only-held.fixed.md @@ -70,7 +70,12 @@ the next acknowledgement dropped it from the persisted queue (at a turn start or a flush it was lost even from memory). At a tool boundary, nothing from it on is injected. While the framework runs it is kept; a - memory-only queue still loses it when the process exits. + memory-only queue still loses it when the process exits. A persisted queue + (one a quiesce persisted, or boot recovery restored) keeps it across + restarts: each boot restores it and tries it again with the rest of the + queue, at once or at the resume after a quiesced boot, and logs it again. + While it's held the queue stays persisted, so every later deferral is + written to the recovery file. Only the host log names it. - Boot recovery drops a recovered deferred write whose content isn't an array of block objects, which the store could never take, and logs it by id and participant, rather than holding it in the queue. Before, an entry diff --git a/src/framework.ts b/src/framework.ts index d2f2eeba4..ceb4a0998 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -484,10 +484,10 @@ function withDeferredWriteId(metadata: MessageMetadata | undefined, id: string): } /** A deferred entry as a log line names it: its id, participant and block - * types. Every queued entry's blocks are objects: a fresh write with a null - * block throws in addMessage before it could be queued, and a recovered - * entry whose content isn't an array of objects is dropped at load - * (restorePersistedDeferredWrites). */ + * types. Reading a block's type throws only on a null or undefined block, + * and no queued entry has one: addMessage reads every block's type before + * it defers a fresh write, and a recovered entry whose content isn't an + * array of objects is dropped at load (restorePersistedDeferredWrites). */ function describeDeferredWrite(msg: { id: string; participant: string; content: ContentBlock[] }): string { return `${msg.id} (${msg.participant}: ${[...new Set(msg.content.map((b) => b.type))].join('+') || 'empty'})`; } diff --git a/test/deferred-window-only.test.ts b/test/deferred-window-only.test.ts index 485de9487..212e47d4f 100644 --- a/test/deferred-window-only.test.ts +++ b/test/deferred-window-only.test.ts @@ -483,35 +483,40 @@ describe('mid-turn deliveries keep the window what the stream carried', () => { await reopened.stop(); }); - it("stop() stores several agents' deferred writes in the order they arrived", async () => { - // Residents share one message slot, so the order writes arrived in is the - // order every resident reads. - const agents = [ - { name: 'assistant', model: 'test-model', systemPrompt: 'You are a robot pilot.' }, - { name: 'scout', model: 'test-model', systemPrompt: 'You scout.' }, - ]; - const framework = await createFramework({ agents }); - const internals = internalsOf(framework); - // Frames stop() doesn't wait for hold both agents' turns. - internals.activeTurnTokens.set('assistant', 1); - internals.activeTurnTokens.set('scout', 2); - assert.equal(internals.addMessage('Antra', text('one')), '', 'deferred'); - assert.equal(internals.addMessage('Antra', text('two'), undefined, { forAgent: 'scout' }), '', 'deferred'); - assert.equal(internals.addMessage('Antra', text('three')), '', 'deferred'); - // A turn-end flush of the assistant's entries while a successor holds its - // turn defers them again (the 2026-07-31 interleave). They keep their - // places in arrival order. - internals.readdDrained(internals.drainDeferredFor('assistant'), 'turn-end flush'); - assert.equal(internals.deferredMessages.length, 3, 'deferred again'); - await framework.stop(); - - const reopened = await createFramework({ agents }); - assert.deepEqual( - windowOf(reopened).filter((line) => /^Antra: (one|two|three)$/.test(line)), - ['Antra: one', 'Antra: two', 'Antra: three'], - ); - await reopened.stop(); - }); + for (const persisted of [false, true]) { + it(`stop() stores several agents' deferred writes in the order they arrived (${persisted ? 'persisted' : 'memory-only'} queue)`, async () => { + // Residents share one message slot, so the order writes arrived in is + // the order every resident reads. + const agents = [ + { name: 'assistant', model: 'test-model', systemPrompt: 'You are a robot pilot.' }, + { name: 'scout', model: 'test-model', systemPrompt: 'You scout.' }, + ]; + const framework = await createFramework({ agents }); + const internals = internalsOf(framework); + // A queue boot recovery restored, or one a quiesce persisted, stays + // persisted until it empties. + internals.deferredWritesPersisted = persisted; + // Frames stop() doesn't wait for hold both agents' turns. + internals.activeTurnTokens.set('assistant', 1); + internals.activeTurnTokens.set('scout', 2); + assert.equal(internals.addMessage('Antra', text('one')), '', 'deferred'); + assert.equal(internals.addMessage('Antra', text('two'), undefined, { forAgent: 'scout' }), '', 'deferred'); + assert.equal(internals.addMessage('Antra', text('three')), '', 'deferred'); + // A turn-end flush of the assistant's entries while a successor holds + // its turn defers them again (the 2026-07-31 interleave). They keep + // their places in arrival order. + internals.readdDrained(internals.drainDeferredFor('assistant'), 'turn-end flush'); + assert.equal(internals.deferredMessages.length, 3, 'deferred again'); + await framework.stop(); + + const reopened = await createFramework({ agents }); + assert.deepEqual( + windowOf(reopened).filter((line) => /^Antra: (one|two|three)$/.test(line)), + ['Antra: one', 'Antra: two', 'Antra: three'], + ); + await reopened.stop(); + }); + } it('a turn waiting to retry when the framework stops gives up rather than compile again', { timeout: 60_000 }, async () => { let waiting!: () => void; From d32d8cb83044b24950570ddbc11a087ff42e4c17 Mon Sep 17 00:00:00 2001 From: Duncan-2410 Date: Tue, 13 Oct 2026 21:47:01 -0400 Subject: [PATCH 12/34] test(deferral): an ephemeral run's retry refused once stop() begins doesn't start its turn The removal run at 411310a found that taking out the refusal at the top of startAgentStream no longer failed a test. #281, merged from main, refuses a turn before its compile once stop() has closed admission, and dee1368 ends a waiting ephemeral run there too, so the ephemeral retry test's checks (the run ends with the refusal, one compile) held without the top refusal. What the top refusal still does is keep the retry's turn from starting at all: no turn token, no turn-start flush, no context gathered and no hooks run while stop() tears down. The test now counts the module's gatherContext calls and expects the first attempt's alone, and is renamed for it. With the top refusal removed, that assertion fails. Co-Authored-By: Claude Opus 5.5 --- test/ephemeral-settle.test.ts | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/test/ephemeral-settle.test.ts b/test/ephemeral-settle.test.ts index dcb73a0db..7d75812a5 100644 --- a/test/ephemeral-settle.test.ts +++ b/test/ephemeral-settle.test.ts @@ -211,18 +211,25 @@ describe('runEphemeralToCompletion settle signal', () => { } }); - it('a run waiting to retry when stop() begins gives up before compiling again', { timeout: 60_000 }, async () => { + it('a run waiting to retry when stop() begins gives up before its turn starts again', { timeout: 60_000 }, async () => { // An ephemeral run doesn't hold the primary's provider gate, so the - // refusal at the top of startAgentStream is the only check its retry - // meets before it would compile, and start a stream, again. + // refusal at the top of startAgentStream is the first check its retry + // meets. The refusal before the compile would stop it too, but only once + // the turn had started again: its context gathered, its hooks run. const { tempDir, storePath } = tempStorePath('ephemeral-retry-stopping-'); let waiting!: () => void; const inDelay = new Promise((resolve) => { waiting = resolve; }); + let gathered = 0; + const contextModule: Module = new EphemeralToolModule(); + contextModule.gatherContext = async () => { + gathered++; + return []; + }; const framework = await AgentFramework.create({ storePath, membrane: new MockMembrane().asMembrane(), agents: [], - modules: [new EphemeralToolModule()], + modules: [contextModule], syncIntervalMs: 0, errorPolicy: { maxRetries: 1, @@ -245,6 +252,7 @@ describe('runEphemeralToCompletion settle signal', () => { await assert.rejects(run, /refused: the framework is stopping/); await stopped; assert.equal(compiles, 1, 'the retry was refused, not compiled'); + assert.equal(gathered, 1, 'the retry was refused before its turn started'); } finally { rmSync(tempDir, { recursive: true, force: true }); } From 69a9b607ce4d6f66aed2a2bd5cc00ac603a99be4 Mon Sep 17 00:00:00 2001 From: Duncan-2410 Date: Wed, 14 Oct 2026 04:54:11 -0400 Subject: [PATCH 13/34] fix(deferral): the log line names any queued block; the text says what the code does at its edges From an internal review of the final round. - describeDeferredWrite read each block's type directly. Nothing checks a fresh write's blocks beyond addMessage reading each type, so a block that isn't an object (a string or number) can be queued, and the line named such content "empty", or "text+" beside a typed block. It now names a block that isn't an object by its JavaScript type, and an object whose type isn't a string as untyped. Its doc says so, in place of the argument that no queued entry has a null block. host-quiesce's two-resident resume test queues its stuck write with a bare string beside its text block and expects "text+string"; reading the type directly fails it. - The stop() comment: past the deferral a write that throws stays queued (a quiesced host never stores here), and an entry for a target no longer registered is logged and dropped by the boot or resume flush that meets it, which at a quiesced boot is the resume. - The changelog fragment: - a write a flush defers again is one case of going back into the queue; a write whose store throws goes back too; - only the agent's own state (turn, stream, tool cycle) holds its flushes, not only its tool cycle; - stop(): main's refusals before and after a compile (#281) are main's; this change refuses a fork's, subagent's or retry's turn before it starts, and a new ephemeral run, and ends an ephemeral run whose turn is refused because stop() closed admission; - writes that aren't deferred include a message assembled into a turn before its compile (RFC-006 coalescing); - a persisted queue keeps a stuck write across restarts unless boot drops it as an entry the store could never take, and persists later deferrals up to the recovery file's size cap; - boot also drops an entry whose participant isn't text, as its loader always did. Co-Authored-By: Claude Opus 5.5 --- .../mid-turn-window-only-held.fixed.md | 101 +++++++++--------- src/framework.ts | 26 +++-- test/host-quiesce.test.ts | 7 +- 3 files changed, 74 insertions(+), 60 deletions(-) diff --git a/changelog.d/mid-turn-window-only-held.fixed.md b/changelog.d/mid-turn-window-only-held.fixed.md index fa897bbae..b622b16fa 100644 --- a/changelog.d/mid-turn-window-only-held.fixed.md +++ b/changelog.d/mid-turn-window-only-held.fixed.md @@ -18,11 +18,12 @@ subject could remove a message the agent had heard and answered. Within a branch the watermark only advances: a compile that started before such a mark doesn't roll it back when it finishes. -- A deferred write that goes back into the queue does so whatever its - content, under its id and in its place in the order writes arrived. That - happens while its target's own turn, stream or tool cycle is alive, or the - host is quiesced. A `tool_result` used to land regardless, apart from the - `tool_use` it answers, which went back into the queue. +- A deferred write that a flush defers again does so whatever its content, + under its id and in its place in the order writes arrived. That happens + while its target's own turn, stream or tool cycle is alive, or the host is + quiesced. (A write whose store throws goes back into the queue too; see + below.) A `tool_result` used to land regardless, apart from the `tool_use` + it answers, which went back into the queue. - A write that waits now waits on its own target's state: the target's turn, stream and tool cycle, or a quiesced host. (A fresh `tool_result` still doesn't wait.) Before, a write for the primary agent waited out every @@ -33,51 +34,55 @@ the primary's window. Each flush, at a turn's end, after surgery, after a withdrawn turn and after a puppet call, now delivers the agent's deferred messages while another agent's tool cycle is pending; only the agent's own - cycle holds them. The flush after surgery stores several agents' entries in + state holds them. The flush after surgery stores several agents' entries in the order they arrived, where it went agent by agent: residents share one message slot, so that reordered their shared transcript. - `stop()` now stores what is still deferred for each registered agent before the store closes, in the order it arrived; before, a memory-only - queue lost it. Once `stop()` begins, no agent turn starts and no stream - registers: a turn that was mid-compile when the streams were cancelled, or - one waiting to retry, is refused rather than left running, and an ephemeral - run refused this way ends at once rather than waiting for a watchdog. Since - no reply can then follow a stored entry in a stream, `stop()` stores each - one even while a frame it doesn't wait for still holds its target's turn: - a turn started outside the event loop, or a puppet call. `stop()` neither - refuses nor waits for a puppet call, as before. One still running stores - its pair when its tool returns. If that's before `stop()` stores the - deferred entries, the pair lands first and the puppet's own flush stores - what was deferred behind it; if it's later, the pair lands after them, and - on a store `stop()` has closed that write fails. A quiesced host keeps what - is deferred in its persisted queue for boot recovery. An entry for a target - no longer registered (a disposed fork) is dropped from a memory-only queue - at exit, as before. -- A deferred write whose store throws goes back into the queue under its - id, and so does every entry queued behind it for that agent, so none of - them lands ahead of an earlier one, and a queued `tool_use` that can't be - stored keeps the `tool_result` queued behind it from being stored alone. (A - stored `tool_use` whose `tool_result` then can't be stored stays alone + queue lost it. Once `stop()` begins, no agent turn starts: a fork's, + subagent's or retry's turn is refused before it starts, as a resident's + already was, and a new ephemeral run is refused. An ephemeral run whose + turn is refused because `stop()` has closed admission, at any of these + checks or at the ones before and after a compile, ends at once rather than + waiting for a watchdog. Since no reply can then follow a stored entry in a + stream, `stop()` stores each one even while a frame it doesn't wait for + still holds its target's turn: a turn started outside the event loop, or a + puppet call. `stop()` neither refuses nor waits for a puppet call, as + before. One still running stores its pair when its tool returns. If that's + before `stop()` stores the deferred entries, the pair lands first and the + puppet's own flush stores what was deferred behind it; if it's later, the + pair lands after them, and on a store `stop()` has closed that write + fails. A quiesced host keeps what is deferred in its persisted queue for + boot recovery. An entry for a target no longer registered (a disposed + fork) is dropped from a memory-only queue at exit, as before. +- A deferred write whose store throws goes back into the queue under its id, + and so does every entry queued behind it for that agent, so none of them + lands ahead of an earlier one, and a queued `tool_use` that can't be + stored keeps the `tool_result` queued behind it from being stored alone. + (A stored `tool_use` whose `tool_result` then can't be stored stays alone until the result lands at a later flush.) Other agents' entries go on, in - the order they arrived. The log line names the entry at each flush, and one - that keeps throwing holds what is deferred behind it for that agent until - it can be stored. The hold is the queue's: a write that isn't deferred is - stored at once and can land ahead of a held entry. That's a fresh - `tool_result`, and any write while the agent's turn, stream and tool cycle - are idle and the host isn't quiesced. So an entry that keeps throwing holds - only what is deferred while the agent's turn, stream or tool cycle is - alive, or the host is quiesced. Before, it stayed in the un-acked set and - the next acknowledgement dropped it from the persisted queue (at a turn - start or a flush it was lost even from memory). At a tool boundary, - nothing from it on is injected. While the framework runs it is kept; a - memory-only queue still loses it when the process exits. A persisted queue - (one a quiesce persisted, or boot recovery restored) keeps it across - restarts: each boot restores it and tries it again with the rest of the - queue, at once or at the resume after a quiesced boot, and logs it again. - While it's held the queue stays persisted, so every later deferral is - written to the recovery file. Only the host log names it. -- Boot recovery drops a recovered deferred write whose content isn't an - array of block objects, which the store could never take, and logs it by - id and participant, rather than holding it in the queue. Before, an entry - whose content wasn't an array was dropped without a line, and one with a - null block was kept, where it could never be stored. + the order they arrived. The log line names the entry at each flush, and + one that keeps throwing holds what is deferred behind it for that agent + until it can be stored. The hold is the queue's: a write that isn't + deferred is stored at once and can land ahead of a held entry. That's a + fresh `tool_result`, a message assembled into the agent's turn before its + compile (RFC-006 coalescing), and any write while the agent's turn, stream + and tool cycle are idle and the host isn't quiesced. So an entry that + keeps throwing holds only what is deferred while the agent's turn, stream + or tool cycle is alive, or the host is quiesced. Before, it stayed in the + un-acked set and the next acknowledgement dropped it from the persisted + queue (at a turn start or a flush it was lost even from memory). At a tool + boundary, nothing from it on is injected. While the framework runs it is + kept; a memory-only queue still loses it when the process exits. A + persisted queue (one a quiesce persisted, or boot recovery restored) keeps + it across restarts: each boot restores it and tries it again with the rest + of the queue, at once or at the resume after a quiesced boot, and logs it + again, unless it's one boot drops (below). While it's held the queue stays + persisted, so every later deferral is written to the recovery file, up to + the file's size cap. Only the host log names it. +- Boot recovery drops a recovered deferred write whose participant isn't + text or whose content isn't an array of block objects, which the store + could never take, and logs it by id and participant, rather than holding + it in the queue. Before, such an entry was dropped without a line, except + one whose blocks weren't all objects, such as a null block, which was kept, + where it could never be stored. diff --git a/src/framework.ts b/src/framework.ts index b6a254fdf..034bf6e97 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -484,12 +484,17 @@ function withDeferredWriteId(metadata: MessageMetadata | undefined, id: string): } /** A deferred entry as a log line names it: its id, participant and block - * types. Reading a block's type throws only on a null or undefined block, - * and no queued entry has one: addMessage reads every block's type before - * it defers a fresh write, and a recovered entry whose content isn't an - * array of objects is dropped at load (restorePersistedDeferredWrites). */ + * types. A block that isn't an object is named by its JavaScript type, and + * an object whose type isn't a string as untyped, so the line names whatever + * a queued entry holds. */ function describeDeferredWrite(msg: { id: string; participant: string; content: ContentBlock[] }): string { - return `${msg.id} (${msg.participant}: ${[...new Set(msg.content.map((b) => b.type))].join('+') || 'empty'})`; + const kind = (b: unknown): string => { + if (b === null) return 'null'; + if (typeof b !== 'object') return typeof b; + const type = (b as { type?: unknown }).type; + return typeof type === 'string' ? type : 'untyped'; + }; + return `${msg.id} (${msg.participant}: ${[...new Set(msg.content.map(kind))].join('+') || 'empty'})`; } /** Whether a deferred message can join a live stream at a tool boundary: @@ -2337,11 +2342,12 @@ export class AgentFramework { // fails. // // A quiesced host defers each entry again instead, into the persisted - // queue, for boot recovery. A write that throws stays queued with what - // follows it for its target, and a memory-only queue still loses it at - // exit. An entry for a target no longer registered (a disposed fork) is - // left: dropped with a memory-only queue at exit, and logged and dropped - // from a persisted one at boot. + // queue, for boot recovery. Past the deferral, a write that throws stays + // queued with what follows it for its target, and a memory-only queue + // still loses it at exit. An entry for a target no longer registered (a + // disposed fork) is left: dropped with a memory-only queue at exit, and + // logged and dropped from a persisted one by the boot or resume flush + // that meets it. if (this.deferredMessages.length > 0) { const entries = [...this.agents.keys()].flatMap((name) => this.drainDeferredFor(name)); this.readdDrained(entries, 'stop', { bypassDeferral: !this.quiesced }); diff --git a/test/host-quiesce.test.ts b/test/host-quiesce.test.ts index fe3b81676..812ea60bb 100644 --- a/test/host-quiesce.test.ts +++ b/test/host-quiesce.test.ts @@ -752,7 +752,10 @@ test("resume stores several agents' writes in the order they arrived, and cuts e add('one', 'agent'); add('two', 'scout'); add('three', 'agent'); - add('stuck', 'agent'); + // Nothing checks a fresh write's blocks beyond reading each one's type, + // so a block that isn't an object can be queued; the log line names it + // by its JavaScript type. + internals.addMessage('user', [{ type: 'text', text: 'stuck' }, 'a bare string'], undefined, { forAgent: 'agent' }); add('for scout', 'scout'); add('behind the stuck one', 'agent'); add('also for scout', 'scout'); @@ -767,7 +770,7 @@ test("resume stores several agents' writes in the order they arrived, and cuts e return realAdd(participant, content, metadata, causedBy); }) as typeof agentCm.addMessage; const named = () => lines.filter((line) => - line.includes(`deferred write ${stuckId} (user: text) for agent could not be stored`) + line.includes(`deferred write ${stuckId} (user: text+string) for agent could not be stored`) && line.includes('a thrown value that has no text')).length; let status = await framework.resume(); From 57943f6e5d16cbf21fa4da20343156d5fb67bc22 Mon Sep 17 00:00:00 2001 From: Duncan-2410 Date: Wed, 14 Oct 2026 05:12:46 -0400 Subject: [PATCH 14/34] fix(deferral): #251's retry cancellations end a waiting ephemeral run; one stop refusal at startAgentStream's top #251 (89a51fab), merged from main in the previous commit, made stop() end a retry's wait and cancel the retry there, at both retry sites, and added a refusal at the top of startAgentStream for every agent once admission has closed. The merge put that refusal right after this branch's own, which did the same and also ended a waiting ephemeral run, and its two cancellation points left such a run to its idle watchdog. - startAgentStream keeps one refusal, main's, which now also ends a waiting ephemeral run (endEphemeralRunRefused). This branch's copy and its comment are gone. - beginAgentTurn's catch and driveStream's error case end a waiting ephemeral run when stop() has cancelled their retry. Tests: - ephemeral-settle: the retry test waits 60 s, so only stop() ends the wait, and now pins the ending at beginAgentTurn's cancellation (the retry never re-enters). New: a stream that fails and waits to retry ends its run at once when stop() begins (driveStream's cancellation); a turn asked for once stop() has begun, for a run whose request never reached the loop, doesn't start and ends the run at once (the refusal at the top; a store the framework doesn't own, so the run's cleanup can still read it). Removing each of the three endings fails its test. - deferred-window-only: the primary's retry test waits 60 s and now expects no re-entry: stop() ends the wait and cancels the retry. Co-Authored-By: Claude Opus 5.5 --- src/framework.ts | 20 ++++--- test/deferred-window-only.test.ts | 13 ++--- test/ephemeral-settle.test.ts | 88 ++++++++++++++++++++++++++++--- 3 files changed, 98 insertions(+), 23 deletions(-) diff --git a/src/framework.ts b/src/framework.ts index 802196187..e266111cf 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -10903,21 +10903,15 @@ export class AgentFramework { attempt = 0, providerGateAlreadyHeld = false, ): Promise { - // Once stop() has begun, no turn starts, for any agent: a fork's or a - // subagent's retry after its delay, or a wake the loop dequeued before it - // ended, would otherwise begin a turn stop() never cancels, and the - // deferred drain at stop's end would meet it live. - if (this.providerAdmissionClosed) { - if (providerGateAlreadyHeld) this.releasePrimaryProviderGate(agent.name); - this.endEphemeralRunRefused(agent); - return; - } const ownsProviderGate = !this.ephemeralRuns.has(agent.name) && !this.conversationAgentHomes.has(agent.name); // A stopping framework starts no turn, whoever owns provider admission - // (conversation agents and ephemeral runs do not take the gate). + // (conversation agents and ephemeral runs do not take the gate). An + // ephemeral run waiting on the turn ends now, as at every refusal of + // stop()'s. if (this.providerAdmissionClosed) { if (ownsProviderGate && providerGateAlreadyHeld) this.releasePrimaryProviderGate(agent.name); + this.endEphemeralRunRefused(agent); return; } if (ownsProviderGate && !providerGateAlreadyHeld) { @@ -11446,11 +11440,13 @@ export class AgentFramework { if (this.providerAdmissionClosed) { // stop() ended the wait: that cancels the retry, it does not hasten // it. Nothing is written (the store is closing); this frame still - // owns its token, since no retry frame replaced it. + // owns its token, since no retry frame replaced it. An ephemeral + // run waiting on the turn ends now, as at every refusal of stop()'s. if (this.activeTurnTokens.get(agent.name) === turnToken) { this.activeTurnTokens.delete(agent.name); this.activeTurnTriggers.delete(agent.name); } + this.endEphemeralRunRefused(agent); return false; } // The retry re-enters startAgentStream, which replaces this frame's @@ -12672,7 +12668,9 @@ export class AgentFramework { await this.waitForRetry(action.delayMs); // stop() ending the wait cancels the retry (this frame's finally // clears the token); it is never permission to retry at once. + // An ephemeral run waiting on the turn ends now. if (!this.providerAdmissionClosed) await this.startAgentStream(agent, trigger, attempt + 1); + else this.endEphemeralRunRefused(agent); } else { this.settleAgent(agent.name, { stopReason: 'exhausted', diff --git a/test/deferred-window-only.test.ts b/test/deferred-window-only.test.ts index 212e47d4f..e0500a338 100644 --- a/test/deferred-window-only.test.ts +++ b/test/deferred-window-only.test.ts @@ -526,7 +526,8 @@ describe('mid-turn deliveries keep the window what the stream carried', () => { maxRetries: 2, onInferenceError: (_error: Error, _agent: string, attempt: number) => { waiting(); - return attempt < 2 ? { retry: true as const, delayMs: 50 } : { retry: false as const }; + // Long enough that only stop() ends the wait. + return attempt < 2 ? { retry: true as const, delayMs: 60_000 } : { retry: false as const }; }, }, }); @@ -537,9 +538,9 @@ describe('mid-turn deliveries keep the window what the stream carried', () => { compiles++; throw new Error('boom'); }; - // The retry re-enters startAgentStream after its delay; count those - // re-entries, so a retry that was never granted can't pass for one that - // was refused. + // A retry re-enters startAgentStream after its wait. stop() ends the + // wait, and that cancels the retry before it re-enters: count re-entries, + // so a retry that went ahead can't pass for one that was cancelled. const retries: Array<{ attempt: number; stopping: boolean }> = []; const frames = framework as unknown as { providerAdmissionClosed: boolean; @@ -557,8 +558,8 @@ describe('mid-turn deliveries keep the window what the stream carried', () => { const error = console.error; console.error = () => {}; try { await framework.stop(); } finally { console.error = error; } - assert.deepEqual(retries, [{ attempt: 1, stopping: true }], 'the retry was granted, and re-entered once stop() had begun'); - assert.equal(compiles, 1, 'the retry was refused, not compiled'); + assert.deepEqual(retries, [], 'stop() ended the wait, and the retry was cancelled before it re-entered'); + assert.equal(compiles, 1, 'the retry was not compiled'); const reopened = await createFramework(); assert.ok(windowOf(reopened).includes('Antra: while it waits'), 'what was deferred for it was stored'); diff --git a/test/ephemeral-settle.test.ts b/test/ephemeral-settle.test.ts index 7d75812a5..db251e5d0 100644 --- a/test/ephemeral-settle.test.ts +++ b/test/ephemeral-settle.test.ts @@ -17,6 +17,7 @@ import type { ToolResult, } from '../src/index.js'; import { AgentFramework } from '../src/index.js'; +import { JsStore } from '@animalabs/chronicle'; import { createMockResponse, MockMembrane, MockYieldingStream } from './helpers/mock-membrane.js'; class EphemeralToolModule implements Module { @@ -211,11 +212,9 @@ describe('runEphemeralToCompletion settle signal', () => { } }); - it('a run waiting to retry when stop() begins gives up before its turn starts again', { timeout: 60_000 }, async () => { - // An ephemeral run doesn't hold the primary's provider gate, so the - // refusal at the top of startAgentStream is the first check its retry - // meets. The refusal before the compile would stop it too, but only once - // the turn had started again: its context gathered, its hooks run. + it('a run waiting to retry when stop() begins gives up before its turn starts again, and ends at once', { timeout: 60_000 }, async () => { + // stop() ends the retry's wait, which cancels the retry before its turn + // starts again; the run ends then, not when its idle watchdog fires. const { tempDir, storePath } = tempStorePath('ephemeral-retry-stopping-'); let waiting!: () => void; const inDelay = new Promise((resolve) => { waiting = resolve; }); @@ -235,7 +234,8 @@ describe('runEphemeralToCompletion settle signal', () => { maxRetries: 1, onInferenceError: (_error: Error, _agent: string, attempt: number): ErrorAction => { waiting(); - return attempt < 1 ? { retry: true, delayMs: 50 } : { retry: false }; + // Long enough that only stop() ends the wait. + return attempt < 1 ? { retry: true, delayMs: 60_000 } : { retry: false }; }, }, }); @@ -258,6 +258,82 @@ describe('runEphemeralToCompletion settle signal', () => { } }); + it('a run whose stream failed and waits to retry ends at once when stop() begins', { timeout: 60_000 }, async () => { + // The stream's own retry: stop() ends its wait and cancels it there. + const { tempDir, storePath } = tempStorePath('ephemeral-stream-retry-stopping-'); + let waiting!: () => void; + const inDelay = new Promise((resolve) => { waiting = resolve; }); + const membrane = new MockMembrane(); + membrane.streamYielding = (request) => { + membrane.calls.push(request); + // A stream that fails once it has started. + const stream = new MockYieldingStream([]); + (stream as unknown as { events: unknown[] }).events.push({ type: 'error', error: new Error('overloaded') }); + return stream; + }; + const framework = await AgentFramework.create({ + storePath, + membrane: membrane.asMembrane(), + agents: [], + modules: [new EphemeralToolModule()], + syncIntervalMs: 0, + errorPolicy: { + maxRetries: 1, + onInferenceError: (_error: Error, _agent: string, attempt: number): ErrorAction => { + waiting(); + // Long enough that only stop() ends the wait. + return attempt < 1 ? { retry: true, delayMs: 60_000 } : { retry: false }; + }, + }, + }); + try { + const { agent, contextManager } = await createEphemeral(framework); + const run = runStartedEphemeral(framework, agent, contextManager, STOPPED_RUN_WATCHDOGS); + await inDelay; + const stopped = framework.stop(); + await assert.rejects(run, /refused: the framework is stopping/); + await stopped; + assert.equal(membrane.calls.length, 1, 'the retry was not compiled'); + } finally { + rmSync(tempDir, { recursive: true, force: true }); + } + }); + + it('a turn asked for once stop() has begun does not start, and its run ends at once', { timeout: 60_000 }, async () => { + // As a wake the loop dequeued before stop() ended it would be: refused + // before the turn starts. The run, whose request never reached the loop, + // ends then, not when its startup watchdog fires. + const tempDir = mkdtempSync(join(tmpdir(), 'ephemeral-late-turn-')); + // A store the framework doesn't own stays open after stop(), as the run's + // own cleanup needs it to. + const store = JsStore.openOrCreate({ path: join(tempDir, 'store') }); + let gathered = 0; + const contextModule: Module = new EphemeralToolModule(); + contextModule.gatherContext = async () => { + gathered++; + return []; + }; + const framework = await AgentFramework.create({ + store, + membrane: new MockMembrane().asMembrane(), + agents: [], + modules: [contextModule], + syncIntervalMs: 0, + }); + try { + const { agent, contextManager } = await createEphemeral(framework); + // Asked for, but the event loop never started, so its turn waits. + const run = framework.runEphemeralToCompletion(agent, contextManager, STOPPED_RUN_WATCHDOGS); + await framework.stop(); + await (framework as unknown as { startAgentStream(a: typeof agent): Promise }).startAgentStream(agent); + await assert.rejects(run, /refused: the framework is stopping/); + assert.equal(gathered, 0, 'the turn did not start'); + } finally { + store.close(); + rmSync(tempDir, { recursive: true, force: true }); + } + }); + it('resolves plain text completion and deregisters the agent', async () => { const { tempDir, storePath } = tempStorePath('ephemeral-plain-'); const membrane = new MockMembrane(); From e9689d428dc0fac5717a94643e490eb2df8249df Mon Sep 17 00:00:00 2001 From: Duncan-2410 Date: Wed, 14 Oct 2026 05:13:42 -0400 Subject: [PATCH 15/34] docs(deferral): the fragment no longer claims the stop() turn refusal main has since #251 #251 refuses every turn at the top of startAgentStream once stop() has closed admission and cancels a retry whose wait stop() ends, so this change no longer adds the turn refusal its fragment described. The stop() bullet now says what this change adds there: a new ephemeral run is refused, and an ephemeral run whose turn is refused because stop() closed admission, or whose retry stop() cancels, ends at once. Co-Authored-By: Claude Opus 5.5 --- changelog.d/mid-turn-window-only-held.fixed.md | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/changelog.d/mid-turn-window-only-held.fixed.md b/changelog.d/mid-turn-window-only-held.fixed.md index b622b16fa..d059c6341 100644 --- a/changelog.d/mid-turn-window-only-held.fixed.md +++ b/changelog.d/mid-turn-window-only-held.fixed.md @@ -39,14 +39,13 @@ message slot, so that reordered their shared transcript. - `stop()` now stores what is still deferred for each registered agent before the store closes, in the order it arrived; before, a memory-only - queue lost it. Once `stop()` begins, no agent turn starts: a fork's, - subagent's or retry's turn is refused before it starts, as a resident's - already was, and a new ephemeral run is refused. An ephemeral run whose - turn is refused because `stop()` has closed admission, at any of these - checks or at the ones before and after a compile, ends at once rather than - waiting for a watchdog. Since no reply can then follow a stored entry in a - stream, `stop()` stores each one even while a frame it doesn't wait for - still holds its target's turn: a turn started outside the event loop, or a + queue lost it. Once `stop()` begins, a new ephemeral run is refused, and + an ephemeral run whose turn is refused because `stop()` has closed + admission, or whose retry `stop()` cancels, ends at once rather than + waiting for a watchdog. Since no turn starts or registers a stream once + `stop()` begins, no reply can then follow a stored entry in a stream, so + `stop()` stores each one even while a frame it doesn't wait for still + holds its target's turn: a turn started outside the event loop, or a puppet call. `stop()` neither refuses nor waits for a puppet call, as before. One still running stores its pair when its tool returns. If that's before `stop()` stores the deferred entries, the pair lands first and the From 739bdbac8b290b0a1ae5c0ff985760213737cf6e Mon Sep 17 00:00:00 2001 From: Duncan-2410 Date: Wed, 14 Oct 2026 05:17:51 -0400 Subject: [PATCH 16/34] docs(deferral): stop() waits for a puppet call since #325; say so where this change said it didn't #325 makes stop() refuse a puppet call once it has begun and wait for one in flight, with the other operator operations, before teardown and so before this change's drain. The stop() comment and the fragment said stop() neither refuses nor waits for one, and what happened to its pair: they now say a puppet call in flight has stored its pair and run its flush by the drain, and that only one still running after that wait timed out holds its target's turn past it, its pair landing after the drained entries on the store a timed-out wait leaves open. A test's comment named a puppet call as a frame stop() doesn't wait for; it names a turn started outside the event loop. Co-Authored-By: Claude Opus 5.5 --- changelog.d/mid-turn-window-only-held.fixed.md | 15 ++++++--------- src/framework.ts | 15 +++++++-------- test/deferred-window-only.test.ts | 4 ++-- 3 files changed, 15 insertions(+), 19 deletions(-) diff --git a/changelog.d/mid-turn-window-only-held.fixed.md b/changelog.d/mid-turn-window-only-held.fixed.md index d059c6341..b82168a59 100644 --- a/changelog.d/mid-turn-window-only-held.fixed.md +++ b/changelog.d/mid-turn-window-only-held.fixed.md @@ -45,15 +45,12 @@ waiting for a watchdog. Since no turn starts or registers a stream once `stop()` begins, no reply can then follow a stored entry in a stream, so `stop()` stores each one even while a frame it doesn't wait for still - holds its target's turn: a turn started outside the event loop, or a - puppet call. `stop()` neither refuses nor waits for a puppet call, as - before. One still running stores its pair when its tool returns. If that's - before `stop()` stores the deferred entries, the pair lands first and the - puppet's own flush stores what was deferred behind it; if it's later, the - pair lands after them, and on a store `stop()` has closed that write - fails. A quiesced host keeps what is deferred in its persisted queue for - boot recovery. An entry for a target no longer registered (a disposed - fork) is dropped from a memory-only queue at exit, as before. + holds its target's turn, such as a turn started outside the event loop, or + a puppet call still running after `stop()`'s wait for operator operations + timed out; that call's pair then lands after them. A quiesced host keeps + what is deferred in its persisted queue for boot recovery. An entry for a + target no longer registered (a disposed fork) is dropped from a + memory-only queue at exit, as before. - A deferred write whose store throws goes back into the queue under its id, and so does every entry queued behind it for that agent, so none of them lands ahead of an earlier one, and a queued `tool_use` that can't be diff --git a/src/framework.ts b/src/framework.ts index 9a39160c3..157704cd4 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -2834,15 +2834,14 @@ export class AgentFramework { // queue order (residents share one message slot), each target cut at its // own first entry that can't be stored (writeDrainedInOrder). They go // past the deferral: a frame stop() doesn't wait for (a turn started - // outside the event loop, or a puppet call) can still hold its target's - // turn token, but no reply can follow an entry in any stream. + // outside the event loop) can still hold its target's turn token, but no + // reply can follow an entry in any stream. // - // stop() neither refuses nor waits for a puppet call, as before this - // drain. One still running stores its pair when its tool returns. If - // that's before this drain runs, the pair lands first, and the puppet's - // own flush stores what was deferred behind it. If it's later, the pair - // lands after these entries, and on a store closed below that write - // fails. + // A puppet call in flight is one of the operator operations stop() waited + // for above, so its pair and the flush after it have landed by now. If + // that wait timed out, the call may still be running and holding its + // target's turn: its pair lands after these entries, on the store a + // timed-out wait leaves open. // // A quiesced host defers each entry again instead, into the persisted // queue, for boot recovery. Past the deferral, a write that throws stays diff --git a/test/deferred-window-only.test.ts b/test/deferred-window-only.test.ts index e0500a338..0ceba7846 100644 --- a/test/deferred-window-only.test.ts +++ b/test/deferred-window-only.test.ts @@ -472,8 +472,8 @@ describe('mid-turn deliveries keep the window what the stream carried', () => { it("stop() stores what is deferred for an agent whose turn frame it doesn't wait for", async () => { const framework = await createFramework(); const internals = internalsOf(framework); - // A frame stop() doesn't await, such as a puppet call still running, - // holds the agent's turn token. + // A frame stop() doesn't await, such as a turn started outside the + // event loop, holds the agent's turn token. internals.activeTurnTokens.set('assistant', 1); assert.equal(internals.addMessage('Antra', text('still to land')), '', 'deferred: the turn is alive'); await framework.stop(); From 3e89b658be4ba1d91eb7dc1fd62d21b4931be517 Mon Sep 17 00:00:00 2001 From: Duncan-2410 Date: Wed, 14 Oct 2026 06:53:54 -0400 Subject: [PATCH 17/34] test(deferral): a requeued write is in the recovery file once; the dropped-entry line names what it holds An internal review of the final round found pieces no test pinned, and two places where a line or a comment said less than it should. - requeueDrained takes the entries it puts back out of the un-acked set. Without that, a write that fails at a flush is in the recovery file twice (once handed off, once requeued) until the batch's ack, and for as long as that ack's sync keeps failing; a crash then restores it twice and stores it twice. Removing it survived every test in the twelve files that touch the deferral. host-quiesce now fails a write at resume and the ack's sync after it, and checks that the file holds the entry that landed (waiting for its ack) and the two that went back, each once. Its removal fails it. - Boot recovery's line for a dropped entry said "no participant" (or "no id") for a value that was there but wasn't text. It now names such a value by its type ("participant of type number"), and "no " only when the field is missing. The boot-recovery test adds an entry with a numeric participant; the old line fails it. - writeDrainedInOrder's doc read as if the held later entries were logged at each flush; it's the entry that keeps failing that is. - The fragment said a held queue's later deferrals are written to the recovery file up to its size cap, and not what happens past it: the file isn't rewritten, as the host log warns, and what's deferred after that is kept in memory only. Co-Authored-By: Claude Opus 5.5 --- .../mid-turn-window-only-held.fixed.md | 4 +- src/framework.ts | 9 ++-- test/host-quiesce.test.ts | 54 +++++++++++++++++-- 3 files changed, 60 insertions(+), 7 deletions(-) diff --git a/changelog.d/mid-turn-window-only-held.fixed.md b/changelog.d/mid-turn-window-only-held.fixed.md index b82168a59..a20c5346f 100644 --- a/changelog.d/mid-turn-window-only-held.fixed.md +++ b/changelog.d/mid-turn-window-only-held.fixed.md @@ -75,7 +75,9 @@ of the queue, at once or at the resume after a quiesced boot, and logs it again, unless it's one boot drops (below). While it's held the queue stays persisted, so every later deferral is written to the recovery file, up to - the file's size cap. Only the host log names it. + the file's size cap; past it the file isn't rewritten, as the host log + warns, and what's deferred after that is kept in memory only. Only the + host log names the held write. - Boot recovery drops a recovered deferred write whose participant isn't text or whose content isn't an array of block objects, which the store could never take, and logs it by id and participant, rather than holding diff --git a/src/framework.ts b/src/framework.ts index 157704cd4..133c88c55 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -4955,9 +4955,12 @@ export class AgentFramework { && (m as { content: unknown[] }).content.every((b) => !!b && typeof b === 'object'); const malformed = data.filter((m) => !wellFormed(m)); for (const m of malformed) { + // What the entry holds there: a value that isn't text by its type. const field = (key: string): string => { const value = m && typeof m === 'object' ? (m as Record)[key] : undefined; - return typeof value === 'string' ? value : `no ${key}`; + if (typeof value === 'string') return value; + if (value === undefined) return `no ${key}`; + return `${key} ${value === null ? 'null' : `of type ${typeof value}`}`; }; console.error( `[host-mode] a recovered deferred context write is malformed and was dropped: ` + @@ -15126,8 +15129,8 @@ export class AgentFramework { * being stored alone. (A tool_use that is stored, followed by a tool_result * that can't be, stays stored without it until the tool_result lands at a * later flush. The only queued pair is puppetToolCall's defensive one, - * which no path reaches today.) An entry that keeps failing holds its - * target's later queued writes, logged by id at each flush, rather than + * which no path reaches today.) An entry that keeps failing is logged by + * id at each flush, and holds its target's later queued writes rather than * letting them land out of order. * * The hold is the queue's alone. A write that isn't deferred is stored at diff --git a/test/host-quiesce.test.ts b/test/host-quiesce.test.ts index 812ea60bb..b322016cb 100644 --- a/test/host-quiesce.test.ts +++ b/test/host-quiesce.test.ts @@ -722,6 +722,52 @@ test('resume flushes deferred writes in order: a poison write stays queued with }); }); +test('a write that fails at resume goes back into the queue once: the recovery file holds each entry once while the batch waits for its ack', async () => { + const membrane = new MockMembrane(); + await withFramework(membrane, async (framework, dir) => { + await framework.quiesce({ reason: 'refold' }); + const cm = framework.getAgent('agent')!.getContextManager(); + const add = (framework as unknown as { + addMessage(participant: string, content: unknown[]): unknown; + }).addMessage.bind(framework); + add('user', [{ type: 'text', text: 'first' }]); + add('user', [{ type: 'text', text: 'poison' }]); + add('user', [{ type: 'text', text: 'third' }]); + + const realAdd = cm.addMessage.bind(cm); + (cm as unknown as { addMessage: typeof cm.addMessage }).addMessage = ((participant, content, metadata, causedBy) => { + if ((content[0] as { text?: string }).text === 'poison') throw new Error('store write failed'); + return realAdd(participant, content, metadata, causedBy); + }) as typeof cm.addMessage; + // The batch's ack syncs the store before it rewrites the recovery file + // without what landed. While that sync fails, the batch stays in the file + // as it was handed off, beside what went back into the queue. + const store = (framework as unknown as { store: { sync(): void } }).store; + const realSync = store.sync.bind(store); + let syncFails = true; + store.sync = () => { + if (syncFails) throw new Error('sync failed'); + realSync(); + }; + try { + await framework.resume(); + } finally { + syncFails = false; + } + + const file = join(dir, 'store', 'recovery', 'deferred-writes.json'); + const pending = (JSON.parse(readFileSync(file, 'utf8')) as { + pending: Array<{ id: string; content: Array<{ text: string }> }>; + }).pending; + assert.deepEqual( + pending.map((entry) => entry.content[0]!.text), + ['first', 'poison', 'third'], + 'the entry that landed waits for its ack, and the two that went back are in the queue, each once', + ); + assert.equal(new Set(pending.map((entry) => entry.id)).size, 3, 'no id is written twice'); + }); +}); + test("resume stores several agents' writes in the order they arrived, and cuts each agent at its own stuck write, named at every flush", { timeout: 60_000 }, async () => { const dir = mkdtempSync(join(tmpdir(), 'host-quiesce-targets-')); const framework = await AgentFramework.create({ @@ -817,13 +863,15 @@ test('boot recovery drops a recovered deferred write the store could never take, await framework.stop(); // Entries no write of the framework's makes, as other code could leave - // them in the recovery file: a null block, and content that isn't blocks. + // them in the recovery file: a null block, content that isn't blocks, and + // a participant that isn't text, which the line names by its type. const file = join(storePath, 'recovery', 'deferred-writes.json'); const doc = JSON.parse(readFileSync(file, 'utf8')) as { pending: Array> }; assert.equal(doc.pending.length, 2); doc.pending.splice(1, 0, { id: 'null-block', seq: 100, participant: 'user', content: [{ type: 'text', text: 'x' }, null] }, { id: 'not-blocks', seq: 101, participant: 'user', content: 'x' }, + { id: 'number-participant', seq: 102, participant: 42, content: [{ type: 'text', text: 'x' }] }, ); writeFileSync(file, JSON.stringify(doc)); @@ -831,11 +879,11 @@ test('boot recovery drops a recovered deferred write the store could never take, framework = await AgentFramework.create(config()); console.error = error; assert.equal(framework.getHostModeStatus().deferredWrites, 2, 'the well-formed entries are restored'); - for (const entry of ['null-block (user)', 'not-blocks (user)']) { + for (const entry of ['null-block (user)', 'not-blocks (user)', 'number-participant (participant of type number)']) { assert.ok(lines.some((line) => line.includes(`malformed and was dropped: ${entry}`)), `${entry} is named:\n${lines.join('\n')}`); } const rewritten = readFileSync(file, 'utf8'); - assert.ok(!rewritten.includes('null-block') && !rewritten.includes('not-blocks'), 'the recovery file is rewritten without them'); + assert.ok(!['null-block', 'not-blocks', 'number-participant'].some((id) => rewritten.includes(id)), 'the recovery file is rewritten without them'); await framework.resume(); const texts = framework.getAgent('agent')!.getContextManager().getAllMessages().slice(before) .map((m) => (m.content[0] as { text: string }).text); From db59360d25095a1520ce8c79d4d2f2fe7ccc38d7 Mon Sep 17 00:00:00 2001 From: Duncan-2410 Date: Wed, 14 Oct 2026 07:10:03 -0400 Subject: [PATCH 18/34] docs(deferral): say what boot drops and why, and that start() during stop() reopens what the drain relies on An internal review of the final round found two claims the code doesn't bear out. - Boot recovery was said to drop an entry "the store could never take", but the store keeps a string or number block as given (blobKindOf finds no blob kind and keeps the block). What it can't take is a null block: reading its type throws, at the tool boundary and in the store. A string or number block can't be sent either way: membrane's native formatter refuses any request that carries a block that isn't an object with a string type (normalizeToolPairs, at entry), and the XML, completions and Responses formatters leave it out. So the drop stays as it is, and the loader's comment, the fragment and the test's title now say what boot drops (an entry whose shape the queue's readers don't expect) and why. An entry the store refuses for another reason, such as an image without a media type, is restored and held like any write that keeps failing. - stop()'s comment and the fragment said no turn starts once stop() has begun. A start() made before stop() finishes reopens admission, so a turn can start, and the drain then writes past the deferral into it. On main that sequence already leaves a lifecycle a later stop() can't end (stopPromise stays set), so the code isn't changed for it; the comment and the fragment name the exception, and #315 is where start() after stop() is being settled. Co-Authored-By: Claude Opus 5.5 --- .../mid-turn-window-only-held.fixed.md | 29 ++++++++++--------- src/framework.ts | 14 ++++++--- test/host-quiesce.test.ts | 2 +- 3 files changed, 27 insertions(+), 18 deletions(-) diff --git a/changelog.d/mid-turn-window-only-held.fixed.md b/changelog.d/mid-turn-window-only-held.fixed.md index a20c5346f..4077e43c0 100644 --- a/changelog.d/mid-turn-window-only-held.fixed.md +++ b/changelog.d/mid-turn-window-only-held.fixed.md @@ -43,14 +43,15 @@ an ephemeral run whose turn is refused because `stop()` has closed admission, or whose retry `stop()` cancels, ends at once rather than waiting for a watchdog. Since no turn starts or registers a stream once - `stop()` begins, no reply can then follow a stored entry in a stream, so - `stop()` stores each one even while a frame it doesn't wait for still - holds its target's turn, such as a turn started outside the event loop, or - a puppet call still running after `stop()`'s wait for operator operations - timed out; that call's pair then lands after them. A quiesced host keeps - what is deferred in its persisted queue for boot recovery. An entry for a - target no longer registered (a disposed fork) is dropped from a - memory-only queue at exit, as before. + `stop()` begins (unless `start()` is called before it finishes), no reply + can then follow a stored entry in a stream, so `stop()` stores each one + even while a frame it doesn't wait for still holds its target's turn, such + as a turn started outside the event loop, or a puppet call still running + after `stop()`'s wait for operator operations timed out; that call's pair + then lands after them. A quiesced host keeps what is deferred in its + persisted queue for boot recovery. An entry for a target no longer + registered (a disposed fork) is dropped from a memory-only queue at exit, + as before. - A deferred write whose store throws goes back into the queue under its id, and so does every entry queued behind it for that agent, so none of them lands ahead of an earlier one, and a queued `tool_use` that can't be @@ -79,8 +80,10 @@ warns, and what's deferred after that is kept in memory only. Only the host log names the held write. - Boot recovery drops a recovered deferred write whose participant isn't - text or whose content isn't an array of block objects, which the store - could never take, and logs it by id and participant, rather than holding - it in the queue. Before, such an entry was dropped without a line, except - one whose blocks weren't all objects, such as a null block, which was kept, - where it could never be stored. + text or whose content isn't an array of block objects, and logs it by id + and participant, rather than holding it in the queue. The store can't take + a null block, and while it keeps a string or number block as given, + membrane's native formatter refuses any request that carries one, and the + other formatters leave it out. Before, such an entry was dropped without a + line, except one whose blocks weren't all objects, which was kept: one + with a null block could never be stored. diff --git a/src/framework.ts b/src/framework.ts index 133c88c55..deca4eb97 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -2826,6 +2826,9 @@ export class AgentFramework { // No stream is left: from the start of stop() none registers and no // agent turn starts (providerAdmissionClosed, checked for every agent), // and every stream registered before was cancelled and awaited above. + // (A start() made before this stop finishes reopens admission, so a turn + // it begins can be alive here and take one of these entries mid-stream; + // #315 asks how start() after stop() should behave.) // What is still deferred (a message held for a turn's end whose flush // found the target busy again, or one deferred behind its target's own // state) would wait for a flush that may never come. Land each at its @@ -4945,10 +4948,13 @@ export class AgentFramework { if (typeof v === 'number' && Number.isFinite(v)) scanFrom.set(k, Math.max(0, Math.floor(v))); } } - // An entry the store could never take has no place in the queue, whose - // readers trust each entry's shape: one whose participant isn't text, - // or whose content isn't an array of block objects, is dropped here - // with a line naming it, rather than held. + // An entry whose shape the queue's readers don't expect has no place in + // it: one whose participant isn't text, or whose content isn't an array + // of block objects, is dropped here with a line naming it, rather than + // held. Reading a null block's type throws, at the tool boundary and in + // the store. The store keeps a string or number block as given, but + // membrane's native formatter refuses any request that carries one (the + // other formatters leave such a block out). const wellFormed = (m: unknown): m is Omit & { id?: string; seq?: number } => !!m && typeof m === 'object' && typeof (m as { participant?: unknown }).participant === 'string' && Array.isArray((m as { content?: unknown }).content) diff --git a/test/host-quiesce.test.ts b/test/host-quiesce.test.ts index b322016cb..a198d3bc6 100644 --- a/test/host-quiesce.test.ts +++ b/test/host-quiesce.test.ts @@ -842,7 +842,7 @@ test("resume stores several agents' writes in the order they arrived, and cuts e } }); -test('boot recovery drops a recovered deferred write the store could never take, naming it, and restores the rest', { timeout: 60_000 }, async () => { +test('boot recovery drops a malformed recovered deferred write, naming it, and restores the rest', { timeout: 60_000 }, async () => { const dir = mkdtempSync(join(tmpdir(), 'host-quiesce-malformed-')); const storePath = join(dir, 'store'); const config = () => ({ From eb3b26f7424b2ed58c51f65bd254c627aa5e5d33 Mon Sep 17 00:00:00 2001 From: Duncan-2410 Date: Wed, 14 Oct 2026 07:19:59 -0400 Subject: [PATCH 19/34] fix(deferral): boot recovery drops only a null or undefined block, and restores an entry holding a string block Boot recovery dropped a recovered entry whose content held any block that isn't an object. But only a null or undefined block can't be handled: its type can't be read, so reading it throws at the tool boundary, in the store's blob extraction and in membrane's formatters. A string or number block is stored as given (blobKindOf finds no blob kind), and every formatter leaves it out of its requests: NativeFormatter's convertContent keeps a block only by its type before normalizeToolPairs sees it, as the XML, completions and Responses formatters do. Dropping such an entry lost its other blocks, which the same write would have landed had it not been deferred across a restart. So the loader now drops an entry whose participant isn't text, whose content isn't an array, or that holds a null or undefined block, and restores an entry with a string block with the rest of its blocks. The boot-recovery test adds an entry holding a text block and a bare string: it is restored, kept in the rewritten recovery file, and lands at resume with the string stored as given. The loader's comment and the fragment say what boot drops and why; the previous commit's claim that the native formatter refuses such a block was wrong. Co-Authored-By: Claude Opus 5.5 --- changelog.d/mid-turn-window-only-held.fixed.md | 14 +++++++------- src/framework.ts | 16 ++++++++-------- test/host-quiesce.test.ts | 16 ++++++++++------ 3 files changed, 25 insertions(+), 21 deletions(-) diff --git a/changelog.d/mid-turn-window-only-held.fixed.md b/changelog.d/mid-turn-window-only-held.fixed.md index 4077e43c0..dfedb7cf5 100644 --- a/changelog.d/mid-turn-window-only-held.fixed.md +++ b/changelog.d/mid-turn-window-only-held.fixed.md @@ -80,10 +80,10 @@ warns, and what's deferred after that is kept in memory only. Only the host log names the held write. - Boot recovery drops a recovered deferred write whose participant isn't - text or whose content isn't an array of block objects, and logs it by id - and participant, rather than holding it in the queue. The store can't take - a null block, and while it keeps a string or number block as given, - membrane's native formatter refuses any request that carries one, and the - other formatters leave it out. Before, such an entry was dropped without a - line, except one whose blocks weren't all objects, which was kept: one - with a null block could never be stored. + text, whose content isn't an array, or that holds a null block, whose type + can't be read, and logs it by id and participant, rather than holding it + in the queue. An entry with a string or number block is restored with the + rest of its blocks: the store keeps such a block as given, and membrane's + formatters leave it out of requests. Before, an entry whose participant + isn't text or whose content isn't an array was dropped without a line, and + one with a null block was kept, where it could never be stored. diff --git a/src/framework.ts b/src/framework.ts index deca4eb97..5ff1ebf39 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -4948,17 +4948,17 @@ export class AgentFramework { if (typeof v === 'number' && Number.isFinite(v)) scanFrom.set(k, Math.max(0, Math.floor(v))); } } - // An entry whose shape the queue's readers don't expect has no place in - // it: one whose participant isn't text, or whose content isn't an array - // of block objects, is dropped here with a line naming it, rather than - // held. Reading a null block's type throws, at the tool boundary and in - // the store. The store keeps a string or number block as given, but - // membrane's native formatter refuses any request that carries one (the - // other formatters leave such a block out). + // An entry the queue's readers can't handle has no place in it: one whose + // participant isn't text, whose content isn't an array, or that holds a + // null or undefined block, whose type can't be read (at the tool + // boundary, in the store, or in membrane's formatters), is dropped here + // with a line naming it, rather than held. A string or number block is + // restored with the rest of its entry: the store keeps it as given, and + // every formatter leaves it out of its requests. const wellFormed = (m: unknown): m is Omit & { id?: string; seq?: number } => !!m && typeof m === 'object' && typeof (m as { participant?: unknown }).participant === 'string' && Array.isArray((m as { content?: unknown }).content) - && (m as { content: unknown[] }).content.every((b) => !!b && typeof b === 'object'); + && (m as { content: unknown[] }).content.every((b) => b !== null && b !== undefined); const malformed = data.filter((m) => !wellFormed(m)); for (const m of malformed) { // What the entry holds there: a value that isn't text by its type. diff --git a/test/host-quiesce.test.ts b/test/host-quiesce.test.ts index a198d3bc6..bc95ddc0b 100644 --- a/test/host-quiesce.test.ts +++ b/test/host-quiesce.test.ts @@ -842,7 +842,7 @@ test("resume stores several agents' writes in the order they arrived, and cuts e } }); -test('boot recovery drops a malformed recovered deferred write, naming it, and restores the rest', { timeout: 60_000 }, async () => { +test('boot recovery drops a malformed recovered deferred write, naming it, and restores the rest, a string block among them', { timeout: 60_000 }, async () => { const dir = mkdtempSync(join(tmpdir(), 'host-quiesce-malformed-')); const storePath = join(dir, 'store'); const config = () => ({ @@ -864,7 +864,9 @@ test('boot recovery drops a malformed recovered deferred write, naming it, and r // Entries no write of the framework's makes, as other code could leave // them in the recovery file: a null block, content that isn't blocks, and - // a participant that isn't text, which the line names by its type. + // a participant that isn't text, which the line names by its type. An + // entry with a string block beside a text block is restored and lands: + // the store keeps the string as given. const file = join(storePath, 'recovery', 'deferred-writes.json'); const doc = JSON.parse(readFileSync(file, 'utf8')) as { pending: Array> }; assert.equal(doc.pending.length, 2); @@ -872,22 +874,24 @@ test('boot recovery drops a malformed recovered deferred write, naming it, and r { id: 'null-block', seq: 100, participant: 'user', content: [{ type: 'text', text: 'x' }, null] }, { id: 'not-blocks', seq: 101, participant: 'user', content: 'x' }, { id: 'number-participant', seq: 102, participant: 42, content: [{ type: 'text', text: 'x' }] }, + { id: 'string-block', seq: 103, participant: 'user', content: [{ type: 'text', text: 'kept' }, 'a bare string'] }, ); writeFileSync(file, JSON.stringify(doc)); console.error = (...args: unknown[]) => { lines.push(args.map(String).join(' ')); }; framework = await AgentFramework.create(config()); console.error = error; - assert.equal(framework.getHostModeStatus().deferredWrites, 2, 'the well-formed entries are restored'); + assert.equal(framework.getHostModeStatus().deferredWrites, 3, 'the well-formed entries are restored, the one with a string block among them'); for (const entry of ['null-block (user)', 'not-blocks (user)', 'number-participant (participant of type number)']) { assert.ok(lines.some((line) => line.includes(`malformed and was dropped: ${entry}`)), `${entry} is named:\n${lines.join('\n')}`); } const rewritten = readFileSync(file, 'utf8'); assert.ok(!['null-block', 'not-blocks', 'number-participant'].some((id) => rewritten.includes(id)), 'the recovery file is rewritten without them'); + assert.ok(rewritten.includes('string-block'), 'and keeps the entry with a string block'); await framework.resume(); - const texts = framework.getAgent('agent')!.getContextManager().getAllMessages().slice(before) - .map((m) => (m.content[0] as { text: string }).text); - assert.deepEqual(texts, ['first', 'second']); + const landed = framework.getAgent('agent')!.getContextManager().getAllMessages().slice(before); + assert.deepEqual(landed.map((m) => (m.content[0] as { text: string }).text), ['first', 'second', 'kept']); + assert.deepEqual(landed[2]!.content, [{ type: 'text', text: 'kept' }, 'a bare string'], 'the string block is stored as given'); } finally { console.error = error; await framework.stop(); From ff6bbaf2cc253ee699291d36d2a877eabc7f1f2d Mon Sep 17 00:00:00 2001 From: Duncan-2410 Date: Wed, 14 Oct 2026 13:24:42 -0400 Subject: [PATCH 20/34] docs(deferral): rewrap the fragment line the main merge left long The merge's edit to the fragment kept the next sentence's start on the line it changed, so that line ran to 129 characters where the rest of the file wraps near 76. Rewrapped; the words are unchanged. --- changelog.d/mid-turn-window-only-held.fixed.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/changelog.d/mid-turn-window-only-held.fixed.md b/changelog.d/mid-turn-window-only-held.fixed.md index f31415fd2..6d9d66b41 100644 --- a/changelog.d/mid-turn-window-only-held.fixed.md +++ b/changelog.d/mid-turn-window-only-held.fixed.md @@ -34,9 +34,10 @@ the primary's window. Each flush, at a turn's end, after surgery, after a withdrawn turn, after a start that ends before its stream and after a puppet call, now delivers the agent's deferred messages while another - agent's tool cycle is pending; only the agent's own state holds them. The flush after surgery stores several agents' entries in - the order they arrived, where it went agent by agent: residents share one - message slot, so that reordered their shared transcript. + agent's tool cycle is pending; only the agent's own state holds them. The + flush after surgery stores several agents' entries in the order they + arrived, where it went agent by agent: residents share one message slot, + so that reordered their shared transcript. - `stop()` now stores what is still deferred for each registered agent before the store closes, in the order it arrived; before, a memory-only queue lost it. Once `stop()` begins, a new ephemeral run is refused, and From e493705972e2e2c37c6d29aa3d7cd3641df2cec3 Mon Sep 17 00:00:00 2001 From: Duncan-2410 Date: Wed, 14 Oct 2026 15:19:39 -0400 Subject: [PATCH 21/34] fix(deferral): boot recovery drops an entry holding a string or number block again, since the compile can't take one eb3b26f restored a recovered entry holding a string or number block, reasoning that the store keeps the block as given and every formatter leaves it out of its requests. Both hold, but the compile reads the blocks before any formatter does: context-manager 0.14.0's MessageStore.estimateBlockTokensRaw caches each block's estimate in a WeakMap keyed on the block, and WeakMap.set throws for a string, a number or a boolean. So once such an entry landed, every compile for its agent threw "Invalid value used as weak map key", and a restart didn't clear it. main restores such an entry too, having no block check at boot. The loader's predicate is object-only again (!!b && typeof b === 'object'), so an entry holding a block that isn't an object is dropped with a line naming it. The boot-recovery test now has entries holding a string block and a number block among those dropped and named, checks the recovery file is rewritten without them, and compiles the agent after resume. The loader's comment and the fragment say why such a block is dropped, and that what a block holds, such as a tool result's content, isn't checked, as it isn't for a fresh write. slimepriestess's review of ff6bbaf found it. Co-Authored-By: Claude Opus 5.5 --- .../mid-turn-window-only-held.fixed.md | 17 ++++++---- src/framework.ts | 15 +++++---- test/host-quiesce.test.ts | 33 ++++++++++--------- 3 files changed, 37 insertions(+), 28 deletions(-) diff --git a/changelog.d/mid-turn-window-only-held.fixed.md b/changelog.d/mid-turn-window-only-held.fixed.md index 6d9d66b41..8c5857aca 100644 --- a/changelog.d/mid-turn-window-only-held.fixed.md +++ b/changelog.d/mid-turn-window-only-held.fixed.md @@ -81,10 +81,13 @@ warns, and what's deferred after that is kept in memory only. Only the host log names the held write. - Boot recovery drops a recovered deferred write whose participant isn't - text, whose content isn't an array, or that holds a null block, whose type - can't be read, and logs it by id and participant, rather than holding it - in the queue. An entry with a string or number block is restored with the - rest of its blocks: the store keeps such a block as given, and membrane's - formatters leave it out of requests. Before, an entry whose participant - isn't text or whose content isn't an array was dropped without a line, and - one with a null block was kept, where it could never be stored. + text, whose content isn't an array, or that holds a block that isn't an + object, and logs it by id and participant, rather than holding it in the + queue. The store can't take a null block. It keeps a string or number + block as given, but the context manager's token estimate throws on such a + block, so once one is stored, every compile for that agent fails, and a + restart doesn't clear it. Before, an entry whose participant isn't text or + whose content isn't an array was dropped without a line, and one with a + block that isn't an object was kept: a null block could never be stored, + and a string or number block, once stored, made every compile for its + agent fail. diff --git a/src/framework.ts b/src/framework.ts index d2fc3ab48..47caa0040 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -4976,15 +4976,18 @@ export class AgentFramework { } // An entry the queue's readers can't handle has no place in it: one whose // participant isn't text, whose content isn't an array, or that holds a - // null or undefined block, whose type can't be read (at the tool - // boundary, in the store, or in membrane's formatters), is dropped here - // with a line naming it, rather than held. A string or number block is - // restored with the rest of its entry: the store keeps it as given, and - // every formatter leaves it out of its requests. + // block that isn't an object, is dropped here with a line naming it, + // rather than held. Reading a null block's type throws, at the tool + // boundary and in the store. The store keeps a string or number block as + // given, but the context manager's token estimate keys a WeakMap on each + // block, which such a value can't be: once one is stored, every compile + // for that agent throws, and a restart doesn't clear it. What a block + // holds, such as a tool result's content, isn't checked here, as it isn't + // for a fresh write. const wellFormed = (m: unknown): m is Omit & { id?: string; seq?: number } => !!m && typeof m === 'object' && typeof (m as { participant?: unknown }).participant === 'string' && Array.isArray((m as { content?: unknown }).content) - && (m as { content: unknown[] }).content.every((b) => b !== null && b !== undefined); + && (m as { content: unknown[] }).content.every((b) => !!b && typeof b === 'object'); const malformed = data.filter((m) => !wellFormed(m)); for (const m of malformed) { // What the entry holds there: a value that isn't text by its type. diff --git a/test/host-quiesce.test.ts b/test/host-quiesce.test.ts index bc95ddc0b..03dccd308 100644 --- a/test/host-quiesce.test.ts +++ b/test/host-quiesce.test.ts @@ -842,7 +842,7 @@ test("resume stores several agents' writes in the order they arrived, and cuts e } }); -test('boot recovery drops a malformed recovered deferred write, naming it, and restores the rest, a string block among them', { timeout: 60_000 }, async () => { +test('boot recovery drops a malformed recovered deferred write, naming it, and restores the rest, after which the agent compiles', { timeout: 60_000 }, async () => { const dir = mkdtempSync(join(tmpdir(), 'host-quiesce-malformed-')); const storePath = join(dir, 'store'); const config = () => ({ @@ -863,35 +863,38 @@ test('boot recovery drops a malformed recovered deferred write, naming it, and r await framework.stop(); // Entries no write of the framework's makes, as other code could leave - // them in the recovery file: a null block, content that isn't blocks, and - // a participant that isn't text, which the line names by its type. An - // entry with a string block beside a text block is restored and lands: - // the store keeps the string as given. + // them in the recovery file: a null block, a string block and a number + // block beside a text block, content that isn't blocks, and a participant + // that isn't text, which the line names by its type. Once stored, a string + // or number block would make every compile for the agent throw, since the + // context manager's token estimate keys a WeakMap on each block. const file = join(storePath, 'recovery', 'deferred-writes.json'); const doc = JSON.parse(readFileSync(file, 'utf8')) as { pending: Array> }; assert.equal(doc.pending.length, 2); + const dropped = ['null-block', 'string-block', 'number-block', 'not-blocks', 'number-participant']; doc.pending.splice(1, 0, { id: 'null-block', seq: 100, participant: 'user', content: [{ type: 'text', text: 'x' }, null] }, - { id: 'not-blocks', seq: 101, participant: 'user', content: 'x' }, - { id: 'number-participant', seq: 102, participant: 42, content: [{ type: 'text', text: 'x' }] }, - { id: 'string-block', seq: 103, participant: 'user', content: [{ type: 'text', text: 'kept' }, 'a bare string'] }, + { id: 'string-block', seq: 101, participant: 'user', content: [{ type: 'text', text: 'x' }, 'a bare string'] }, + { id: 'number-block', seq: 102, participant: 'user', content: [{ type: 'text', text: 'x' }, 42] }, + { id: 'not-blocks', seq: 103, participant: 'user', content: 'x' }, + { id: 'number-participant', seq: 104, participant: 42, content: [{ type: 'text', text: 'x' }] }, ); writeFileSync(file, JSON.stringify(doc)); console.error = (...args: unknown[]) => { lines.push(args.map(String).join(' ')); }; framework = await AgentFramework.create(config()); console.error = error; - assert.equal(framework.getHostModeStatus().deferredWrites, 3, 'the well-formed entries are restored, the one with a string block among them'); - for (const entry of ['null-block (user)', 'not-blocks (user)', 'number-participant (participant of type number)']) { + assert.equal(framework.getHostModeStatus().deferredWrites, 2, 'the well-formed entries are restored'); + for (const entry of ['null-block (user)', 'string-block (user)', 'number-block (user)', 'not-blocks (user)', 'number-participant (participant of type number)']) { assert.ok(lines.some((line) => line.includes(`malformed and was dropped: ${entry}`)), `${entry} is named:\n${lines.join('\n')}`); } const rewritten = readFileSync(file, 'utf8'); - assert.ok(!['null-block', 'not-blocks', 'number-participant'].some((id) => rewritten.includes(id)), 'the recovery file is rewritten without them'); - assert.ok(rewritten.includes('string-block'), 'and keeps the entry with a string block'); + assert.ok(!dropped.some((id) => rewritten.includes(id)), 'the recovery file is rewritten without them'); await framework.resume(); - const landed = framework.getAgent('agent')!.getContextManager().getAllMessages().slice(before); - assert.deepEqual(landed.map((m) => (m.content[0] as { text: string }).text), ['first', 'second', 'kept']); - assert.deepEqual(landed[2]!.content, [{ type: 'text', text: 'kept' }, 'a bare string'], 'the string block is stored as given'); + const texts = framework.getAgent('agent')!.getContextManager().getAllMessages().slice(before) + .map((m) => (m.content[0] as { text: string }).text); + assert.deepEqual(texts, ['first', 'second']); + await assert.doesNotReject(framework.getAgent('agent')!.compileContext(), 'and the agent compiles'); } finally { console.error = error; await framework.stop(); From eac77cd0a0e859e7be500bb9addde6d1a947af60 Mon Sep 17 00:00:00 2001 From: Duncan-2410 Date: Thu, 15 Oct 2026 08:42:18 -0400 Subject: [PATCH 22/34] fix(deferral): a write that throws after its message is stored isn't stored again (R1-2) writeDrainedInOrder put every entry whose write threw back into the queue, so a write that stored its message and then threw was stored twice: once by that write, and again by the flush that met the entry next. A store listener can throw after the record is written, since context-manager 0.14.1's MessageStore.append writes the record and only then emits to its listeners, and a sharded body's append loop can throw after its first shards are stored. Before an entry that threw is put back, its target's store is now read from where it stood before the write, for the entry's deferredWriteId. A copy found there counts as written, under its stored id, and the line logged says it threw after it was stored. At a tool boundary the message is injected with the rest, since a message stored between rounds without being carried is the divergence this PR removes. writeDrainedInOrder now returns each entry it wrote with its stored id, so the boundary and the turn-start flush no longer keep maps of their own. Anarchid's nightly review of e493705 found this (R1-2) and reproduced it with a listener that throws once. Co-Authored-By: Claude Opus 5.5 --- src/framework.ts | 118 ++++++++++++++++++++++-------- test/deferred-window-only.test.ts | 75 +++++++++++++++++++ 2 files changed, 161 insertions(+), 32 deletions(-) diff --git a/src/framework.ts b/src/framework.ts index 47caa0040..723a0fed6 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -8696,13 +8696,12 @@ export class AgentFramework { const deferred = this.drainDeferredFor(agent.name, (msg) => !isInjectableMidTurn(msg, agent.name)); // Stored in queue order, cut at the first that can't be stored: it // and what follows wait for the turn's end, and only what was - // stored is injected (the round's results go on either way). - const storedIds = new Map(); - const stored = this.writeDrainedInOrder(deferred, `tool boundary for ${agent.name}`, (msg) => { - storedIds.set(msg, agent.getContextManager().addMessage(msg.participant, msg.content, withDeferredWriteId(msg.metadata, msg.id))); - }); - for (const msg of stored) { - const id = storedIds.get(msg)!; + // stored is injected (the round's results go on either way). A + // write that threw after its message was stored counts as stored, + // so that copy is injected too (writeDrainedInOrder). + const stored = this.writeDrainedInOrder(deferred, `tool boundary for ${agent.name}`, (msg) => + agent.getContextManager().addMessage(msg.participant, msg.content, withDeferredWriteId(msg.metadata, msg.id))); + for (const [msg, id] of stored) { // Every other resident reads what this boundary stores; for its // own agent, the boundary weighs what it injects below. this.noteForOtherResidents(agent, msg.metadata, id); @@ -11462,16 +11461,10 @@ export class AgentFramework { // drain cuts there, and it and what follows stay queued, in order // (writeDrainedInOrder). What was stored is weighed for the turn now // starting and for every other resident's open turn. - const storedIds = new Map(); - const stored = this.writeDrainedInOrder(this.drainDeferredFor(agent.name), `turn-start flush for ${agent.name}`, (msg) => { - const id = agent.getContextManager().addMessage( - msg.participant, msg.content, withDeferredWriteId(msg.metadata, msg.id), - ); - storedIds.set(msg, id); + const stored = this.writeDrainedInOrder(this.drainDeferredFor(agent.name), `turn-start flush for ${agent.name}`, (msg) => + agent.getContextManager().addMessage(msg.participant, msg.content, withDeferredWriteId(msg.metadata, msg.id))); + for (const [msg, id] of stored) { this.emitTrace({ type: 'message:added', messageId: id, source: 'deferred-flush:turn-start' }); - }); - for (const msg of stored) { - const id = storedIds.get(msg)!; this.noteArrival(agent.name, msg.metadata, id); this.noteForOtherResidents(agent, msg.metadata, id); } @@ -15349,34 +15342,92 @@ export class AgentFramework { * cycle are all idle and the host isn't quiesced. So an entry that keeps * failing holds only what is deferred behind it. * - * Returns what was written (stored, or deferred again by addMessage), in - * queue order: for one target, a prefix of its entries. + * A write can throw after its message is stored: a store listener, the + * strategy's handling of the add, or a later shard of a sharded body can + * throw once the record is written. Before an entry that threw is put + * back, its target's store is read from where it stood before the write, + * for the entry's `deferredWriteId`. A copy found there counts as written, + * under its stored id, so the entry isn't requeued and stored a second + * time. (Written through readdDrained, such a copy misses the notes + * addMessage takes after a store, as any write that throws after storing + * does. The tool boundary and the turn-start flush note what is returned.) + * + * Returns what was written (stored, or deferred again by addMessage, whose + * id is then empty), in queue order: for one target, a prefix of its + * entries. */ - private writeDrainedInOrder(entries: DeferredWrite[], where: string, write: (msg: DeferredWrite) => void): DeferredWrite[] { + private writeDrainedInOrder( + entries: DeferredWrite[], + where: string, + write: (msg: DeferredWrite) => MessageId, + ): Map { const ordered = [...entries].sort(bySeq); const targetOf = (msg: DeferredWrite): string | null => msg.forAgent ?? this.primaryAgentName; const cut = new Set(); - const written: DeferredWrite[] = []; + const written = new Map(); for (let i = 0; i < ordered.length; i++) { const msg = ordered[i]!; const target = targetOf(msg); if (cut.has(target)) continue; + const before = this.storedCountOf(target); try { - write(msg); - written.push(msg); + written.set(msg, write(msg)); } catch (error) { + const copy = before === undefined ? undefined : this.storedCopyOf(target!, before, msg.id); + if (copy !== undefined) { + written.set(msg, copy); + this.logDeferredWriteError(msg, where, error, true); + continue; + } cut.add(target); this.requeueDrained(ordered.slice(i).filter((m) => targetOf(m) === target)); - this.logStuckDeferredWrite(msg, where, error); + this.logDeferredWriteError(msg, where, error, false); } } return written; } - /** The line naming a drained write that couldn't be stored. The entries are - * already back in the queue, and logging mustn't fail the flush that met - * it, so a thrown value that has no text is named as such. */ - private logStuckDeferredWrite(msg: DeferredWrite, where: string, error: unknown): void { + /** How many messages the agent's store holds: undefined when no such agent + * is registered, or its store can't say. */ + private storedCountOf(agentName: string | null): number | undefined { + const agent = agentName ? this.agents.get(agentName) : undefined; + if (!agent) return undefined; + try { + const cm = agent.getContextManager() as unknown as { getMessageCount?: () => number; getAllMessages: () => unknown[] }; + return typeof cm.getMessageCount === 'function' ? cm.getMessageCount() : cm.getAllMessages().length; + } catch { + return undefined; + } + } + + /** The id of the message carrying `deferredWriteId` among those the agent's + * store gained from position `from` on, read without blobs, if there is + * one. For a sharded body, its first shard. */ + private storedCopyOf(agentName: string, from: number, deferredWriteId: string): MessageId | undefined { + const agent = this.agents.get(agentName); + if (!agent) return undefined; + try { + const cm = agent.getContextManager() as unknown as { + getMessageCount?: () => number; + getMessageWindow?: (o: number, l: number, opts?: { resolveBlobs?: boolean }) => + { messages: Array<{ id: MessageId; metadata?: Record }> }; + getAllMessages: () => Array<{ id: MessageId; metadata?: Record }>; + }; + const gained = typeof cm.getMessageCount === 'function' && typeof cm.getMessageWindow === 'function' + ? cm.getMessageWindow(from, Math.max(0, cm.getMessageCount() - from), { resolveBlobs: false }).messages + : cm.getAllMessages().slice(from); + return gained.find((m) => m.metadata?.deferredWriteId === deferredWriteId)?.id; + } catch (error) { + console.error(`[deferral] could not read ${agentName}'s store for a deferred write that threw:`, error); + return undefined; + } + } + + /** The line naming a drained write that threw: one that couldn't be stored, + * which is already back in the queue, or one whose message was stored + * before the throw. Logging mustn't fail the flush that met it, so a thrown + * value that has no text is named as such. */ + private logDeferredWriteError(msg: DeferredWrite, where: string, error: unknown, stored: boolean): void { let reason: string; try { reason = error instanceof Error ? error.message : String(error); @@ -15385,8 +15436,11 @@ export class AgentFramework { } console.error( `[deferral] ${where}: deferred write ${describeDeferredWrite(msg)} for ` + - `${msg.forAgent ?? this.primaryAgentName ?? 'the primary'} could not be stored; it stays queued, ` + - `holding what is queued behind it for that agent: ${reason}`, + `${msg.forAgent ?? this.primaryAgentName ?? 'the primary'} ` + + (stored + ? `threw after it was stored; it counts as written and isn't stored again: ` + : `could not be stored; it stays queued, holding what is queued behind it for that agent: `) + + reason, ); } @@ -15394,14 +15448,14 @@ export class AgentFramework { * it again, in queue order and cut per target (writeDrainedInOrder). With * `bypassDeferral`, each is stored past its target's deferral (stop()). */ private readdDrained(entries: DeferredWrite[], where: string, opts?: { bypassDeferral?: boolean }): DeferredWrite[] { - return this.writeDrainedInOrder(entries, where, (msg) => { + return [...this.writeDrainedInOrder(entries, where, (msg) => { const target = msg.forAgent ?? this.primaryAgentName; - this.addMessage(msg.participant, msg.content, msg.metadata, { + return this.addMessage(msg.participant, msg.content, msg.metadata, { fromQueue: { id: msg.id, seq: msg.seq }, ...(msg.forAgent ? { forAgent: msg.forAgent } : {}), ...(opts?.bypassDeferral && target ? { bypassDeferralFor: target } : {}), }); - }); + }).keys()]; } private editMessage(id: MessageId, content: ContentBlock[]): void { diff --git a/test/deferred-window-only.test.ts b/test/deferred-window-only.test.ts index 0ceba7846..f195e8161 100644 --- a/test/deferred-window-only.test.ts +++ b/test/deferred-window-only.test.ts @@ -436,6 +436,81 @@ describe('mid-turn deliveries keep the window what the stream carried', () => { await framework.stop(); }); + /** A store listener that throws once, after the record with this text is + * written: the write throws though its message is stored. */ + function throwOnceAfterStoring(framework: AgentFramework, textOf: string): void { + let thrown = false; + framework.getAgent('assistant')!.getContextManager().onMessage((event) => { + if (thrown || event.type !== 'add') return; + if (!event.message.content.some((b) => (b as { text?: string }).text === textOf)) return; + thrown = true; + throw new Error('listener failed'); + }); + } + + it('at a tool boundary, a write that throws after its message is stored counts as stored: injected, and not stored again', async () => { + membrane.pushResponse(createMockResponse([ + { type: 'text', text: 'Moving.' }, + { type: 'tool_use', id: 'c1', name: 'robot--move', input: {} }, + ] as ContentBlock[], 'tool_use')); + membrane.pushResponse(createMockResponse(text('Done.'))); + const framework = await createFramework(); + throwOnceAfterStoring(framework, 'look left'); + module.interjections = [ + { participant: 'Antra', content: text('look left') }, + { participant: 'Antra', content: text('look right') }, + ]; + const errors: string[] = []; + const error = console.error; + console.error = (...args: unknown[]) => { errors.push(args.map(String).join(' ')); }; + try { + framework.pushEvent({ type: 'external-message', source: 'test', content: 'go' } as unknown as ProcessEvent); + await framework.runUntilIdle(); + } finally { console.error = error; } + assert.deepEqual(injected(), ['Antra: look left', 'Antra: look right'], 'the stored copy is carried too'); + assert.deepEqual(windowOf(framework).slice(2), [ + 'user: ', + 'Antra: look left', + 'Antra: look right', + 'assistant: Done.', + ], 'stored once, where the stream carried it'); + assert.ok(errors.some((e) => e.includes("threw after it was stored; it counts as written and isn't stored again") + && e.includes('listener failed')), errors.join('\n')); + await framework.stop(); + }); + + it('at the turn end, a write that throws after its message is stored is not queued again', async () => { + membrane.pushResponse(createMockResponse([ + { type: 'text', text: 'Moving.' }, + { type: 'tool_use', id: 'c1', name: 'robot--move', input: {} }, + ] as ContentBlock[], 'tool_use')); + membrane.pushResponse(createMockResponse(text('Done.'))); + const framework = await createFramework(); + throwOnceAfterStoring(framework, 'look left'); + // Held behind a message named as the agent, both wait for the turn end. + module.interjections = [ + { participant: 'assistant', content: text('my own words') }, + { participant: 'Antra', content: text('look left') }, + ]; + const error = console.error; + console.error = () => {}; + try { + framework.pushEvent({ type: 'external-message', source: 'test', content: 'go' } as unknown as ProcessEvent); + await framework.runUntilIdle(); + } finally { console.error = error; } + assert.equal(internalsOf(framework).deferredMessages.length, 0, 'nothing went back into the queue'); + await framework.stop(); // stop() lands whatever is still queued + + const reopened = await createFramework(); + assert.deepEqual(windowOf(reopened).slice(2), [ + 'user: ', + 'assistant: Done.', + 'assistant: my own words', + 'Antra: look left', + ], 'stored once, at the turn end'); + await reopened.stop(); + }); + it("a queued tool_use whose store throws keeps its tool_result from being stored alone", async () => { const framework = await createFramework(); const internals = internalsOf(framework); From c55eee9131b9c0596a0c4936b95f7d824ebb570f Mon Sep 17 00:00:00 2001 From: Duncan-2410 Date: Thu, 15 Oct 2026 08:42:39 -0400 Subject: [PATCH 23/34] fix(deferral): mark injected messages read only when the stream says it carries them (R1-1) At a tool boundary this PR stored the drained messages, passed them to provideToolResults as injectedMessages, and moved the agent's watermark over them, as messages the live request carried. Membrane 0.5.87's yielding XML runner ignores injectedMessages, with a warning that they go to the next turn, and connectome-host runs every anthropic-xml recipe in XML tool mode, on that path. So there they were stored and marked read without being carried, and a coalesced replacement for their subject then appended instead of replacing them. On main they stayed unread. The tool-calls event now says whether the stream can carry injected messages: membrane's ToolContext.supportsInjectedMessages, on membrane's main since f52be9f and in no release yet, is true on the native path and false on the XML prefill path. driveStream passes it to enterWaitingForTools, and the agent's state carries it to the boundary. - true: the boundary drains, stores and injects as before, and marks what it injected read. - false: the boundary drains nothing. What is queued waits for the turn-end flush, stored after the turn's last reply, as a held message does, so it never sits in the window between rounds the stream didn't carry it into. - absent, as on 0.5.87 on either path: the boundary injects as before and marks nothing, as on main. Checking only for false would leave the mark on every XML agent on 0.5.87. The cost of checking for true is the native path's mark, which waits for a membrane release with the field, as does XML's no-drain. The mock membrane's streams can declare the field. The test that pinned the mark now declares true, and two new tests pin false and absent. Anarchid's nightly review of e493705 found this (R1-1). Co-Authored-By: Claude Opus 5.5 --- src/agent.ts | 14 ++++++--- src/framework.ts | 41 ++++++++++++++++++++------ src/types/agent.ts | 9 ++++-- test/deferred-window-only.test.ts | 49 +++++++++++++++++++++++++------ test/helpers/mock-membrane.ts | 14 +++++++-- 5 files changed, 101 insertions(+), 26 deletions(-) diff --git a/src/agent.ts b/src/agent.ts index 9c831b0c1..b98c14e81 100644 --- a/src/agent.ts +++ b/src/agent.ts @@ -1070,7 +1070,12 @@ export class Agent { // If all tools done, transition to ready if (this._state.pending.size === 0) { - this._state = { status: 'ready', toolResults: this._state.completed, stream: this._state.stream }; + this._state = { + status: 'ready', + toolResults: this._state.completed, + stream: this._state.stream, + supportsInjectedMessages: this._state.supportsInjectedMessages, + }; } } @@ -1361,9 +1366,10 @@ export class Agent { /** * Transition to waiting_for_tools when stream yields tool calls. - * Called by framework's driveStream. + * Called by framework's driveStream, with what the stream declared about + * carrying injected messages with this round's results (see AgentState). */ - enterWaitingForTools(calls: ToolCall[], stream: YieldingStream): void { + enterWaitingForTools(calls: ToolCall[], stream: YieldingStream, supportsInjectedMessages?: boolean): void { const pending = new Map(); for (const call of calls) { pending.set(call.id, { @@ -1373,7 +1379,7 @@ export class Agent { startedAt: Date.now(), }); } - this._state = { status: 'waiting_for_tools', pending, completed: [], stream }; + this._state = { status: 'waiting_for_tools', pending, completed: [], stream, supportsInjectedMessages }; } /** diff --git a/src/framework.ts b/src/framework.ts index 723a0fed6..6b36b176b 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -8688,11 +8688,18 @@ export class AgentFramework { // #155). So the drain stops at the first such message, which waits // with everything queued behind it — the queue keeps its order — // for the turn-end flush, where no reply follows it in this stream. + // + // A stream that declares it can't carry injected messages + // (supportsInjectedMessages false: membrane's XML prefill path) + // gets none, so nothing is drained here. Everything queued for + // the agent waits for the turn-end flush, as a held message does, + // rather than sitting in the window between rounds the stream + // never showed it. const midTurnInjections: Array<{ participant: string; content: ContentBlock[]; metadata?: MessageMetadata }> = []; // The injected messages' ids, for a boundary that leaves them to the // turn's next start as arrivals. const injectedIds = new Map(); - { + if (currentState.supportsInjectedMessages !== false) { const deferred = this.drainDeferredFor(agent.name, (msg) => !isInjectableMidTurn(msg, agent.name)); // Stored in queue order, cut at the first that can't be stored: it // and what follows wait for the turn's end, and only what was @@ -8830,7 +8837,7 @@ export class AgentFramework { } if (shouldEndTurn) { - // endTurn: messages already stored above, cancel stream, reset to idle. + // endTurn: what this boundary stored stays; cancel stream, reset to idle. // The batch is never submitted, so nothing can refuse it: admit // it now instead of leaving it pending across the idle gap. agent.toolResultGuard.settleTurnEnded(); @@ -8897,11 +8904,17 @@ export class AgentFramework { midTurnInjections.length > 0 ? { injectedMessages: midTurnInjections } : undefined, ); // RFC-006 §3.3: consumption is determined at assembly, and this - // is assembly too — the injected messages are in the live - // request. Without this they would sit above the watermark of - // the turn's compile and read as unread, so a coalesced - // replacement could remove a message the agent heard and - // answered (and that its later replies were minted after). + // is assembly too when the stream carries the injected messages + // into its next request. Without the mark they would sit above + // the watermark of the turn's compile and read as unread, so a + // coalesced replacement could remove a message the agent heard + // and answered (and that its later replies were minted after). + // Only a stream that declares it carries them is marked + // (supportsInjectedMessages true: membrane's native path). One + // that doesn't say may be membrane 0.5.87's XML path, which + // drops them with a warning, and a message marked read there + // could no longer be replaced though the agent never saw it. So + // without the declaration they stay unread, as before this mark. // The mark is the store's head, a scalar, as a compile's is: it // also covers what was stored since the compile without being // carried, such as another slot's messages, and a write that @@ -8910,7 +8923,9 @@ export class AgentFramework { // Under context-manager#160 such a write already gives the // replies stored after it stamps their requests didn't carry, // whatever the mark says; deferring those writers is the fix. - if (midTurnInjections.length > 0) agent.markContextConsumed(); + if (midTurnInjections.length > 0 && currentState.supportsInjectedMessages === true) { + agent.markContextConsumed(); + } agent.setStreaming(currentState.stream); this.emitTrace({ type: 'inference:stream_resumed', @@ -12281,7 +12296,15 @@ export class AgentFramework { // Truncated responses yield 'complete' with stop_reason 'max_tokens' directly. // So all calls here have valid input — {} is legitimate for parameterless tools. - agent.enterWaitingForTools(event.calls, stream); + // Whether this stream can carry messages injected with the round's + // results: true on membrane's native path, false on its XML + // prefill path, absent before membrane declared it. The tool + // boundary reads it from the agent's state. + agent.enterWaitingForTools( + event.calls, + stream, + (event.context as { supportsInjectedMessages?: boolean }).supportsInjectedMessages, + ); for (const call of event.calls) { // RFC-007: register at the one dispatch point for model-issued diff --git a/src/types/agent.ts b/src/types/agent.ts index dce7b9a81..065542751 100644 --- a/src/types/agent.ts +++ b/src/types/agent.ts @@ -304,13 +304,18 @@ export interface InferenceOptions { /** * Internal state of an agent. + * + * `supportsInjectedMessages` is what the stream declared with the round's + * tool calls (membrane's `ToolContext.supportsInjectedMessages`): whether it + * can carry messages injected with that round's results. It's absent when + * the stream doesn't say, as on membrane 0.5.87 and earlier. */ export type AgentState = | { status: 'idle' } | { status: 'inferring'; promise: Promise; abortController: AbortController } | { status: 'streaming'; stream: YieldingStream } - | { status: 'waiting_for_tools'; pending: Map; completed: CompletedToolCall[]; stream?: YieldingStream } - | { status: 'ready'; toolResults: CompletedToolCall[]; stream?: YieldingStream }; + | { status: 'waiting_for_tools'; pending: Map; completed: CompletedToolCall[]; stream?: YieldingStream; supportsInjectedMessages?: boolean } + | { status: 'ready'; toolResults: CompletedToolCall[]; stream?: YieldingStream; supportsInjectedMessages?: boolean }; /** * A tool call that's in progress. diff --git a/test/deferred-window-only.test.ts b/test/deferred-window-only.test.ts index f195e8161..062c6bf97 100644 --- a/test/deferred-window-only.test.ts +++ b/test/deferred-window-only.test.ts @@ -12,7 +12,11 @@ * whose stored prefix holds a message its request didn't. * * A message the stream carried has been read: RFC-006's consumed watermark - * covers it, so a coalesced replacement can no longer remove it. + * covers it, so a coalesced replacement can no longer remove it. Only a + * stream that declares it carries injected messages (membrane's native + * path) is known to have carried them. One that declares it can't (membrane's + * XML prefill path) gets none, and what is queued waits for the turn's end. + * One that doesn't say (membrane 0.5.87) gets them, and they stay unread. */ import { describe, it, beforeEach, afterEach } from 'node:test'; @@ -239,16 +243,43 @@ describe('mid-turn deliveries keep the window what the stream carried', () => { await framework.stop(); }); - it('a message the stream carried is read: the consumed watermark covers it', async () => { - const framework = await runTurn('move', [{ participant: 'Antra', content: text('look left') }]); - assert.deepEqual(injected(), ['Antra: look left']); + /** Whether the stored message with this text reads as unread (RFC-006). */ + function unread(framework: AgentFramework, textOf: string): boolean { const agent = framework.getAgent('assistant')!; - const heard = agent.getContextManager().getAllMessages() - .find((m) => m.content.some((b) => (b as { text?: string }).text === 'look left'))!; - const isUnread = (framework as unknown as { + const stored = agent.getContextManager().getAllMessages() + .find((m) => m.content.some((b) => (b as { text?: string }).text === textOf))!; + assert.ok(stored, `'${textOf}' is stored`); + return (framework as unknown as { isUnreadStoredMessage: (a: typeof agent, id: string) => boolean; - }).isUnreadStoredMessage.bind(framework); - assert.equal(isUnread(agent, heard.id), false, 'a coalesced replacement must not remove what the agent heard'); + }).isUnreadStoredMessage.call(framework, agent, stored.id); + } + + it('a message injected into a stream that declares it carries injections is read: the consumed watermark covers it', async () => { + membrane.supportsInjectedMessages = true; // membrane's native path declares it + const framework = await runTurn('move', [{ participant: 'Antra', content: text('look left') }]); + assert.deepEqual(injected(), ['Antra: look left']); + assert.equal(unread(framework, 'look left'), false, 'a coalesced replacement must not remove what the agent heard'); + await framework.stop(); + }); + + it('a stream that doesn\'t say whether it carries injections gets them, and they stay unread', async () => { + // membrane 0.5.87 doesn't declare it, on either path, and its XML path + // drops what's injected: marked read there, a message the agent never + // saw could no longer be replaced. + const framework = await runTurn('move', [{ participant: 'Antra', content: text('look left') }]); + assert.deepEqual(injected(), ['Antra: look left']); + assert.deepEqual(windowOf(framework).slice(2), ['user: ', 'Antra: look left', 'assistant: Done.']); + assert.equal(unread(framework, 'look left'), true); + await framework.stop(); + }); + + it('a stream that can\'t carry injections gets none: what is queued waits for the turn to end', async () => { + membrane.supportsInjectedMessages = false; // membrane's XML prefill path declares it + const framework = await runTurn('move', [{ participant: 'Antra', content: text('look left') }]); + assert.deepEqual(atBoundary(), undefined, 'the boundary drained nothing'); + assert.equal(membrane.lastStream!.receivedToolResultOptions[0], undefined, 'nothing was injected'); + assert.deepEqual(windowOf(framework).slice(2), ['user: ', 'assistant: Done.', 'Antra: look left']); + assert.equal(unread(framework, 'look left'), true); await framework.stop(); }); diff --git a/test/helpers/mock-membrane.ts b/test/helpers/mock-membrane.ts index 21774f3d8..96cbc92b5 100644 --- a/test/helpers/mock-membrane.ts +++ b/test/helpers/mock-membrane.ts @@ -56,7 +56,12 @@ export class MockYieldingStream implements YieldingStream { { injectedMessages?: Array<{ participant?: string; content: unknown[] }> } | undefined > = []; - constructor(private responses: NormalizedResponse[]) { + /** + * @param supportsInjectedMessages what each tool-calls event declares, as + * membrane does after 0.5.87 (true on its native path, false on its XML + * prefill path); left out, the event doesn't say, as on 0.5.87. + */ + constructor(private responses: NormalizedResponse[], private supportsInjectedMessages?: boolean) { this.processResponse(0); } @@ -108,6 +113,9 @@ export class MockYieldingStream implements YieldingStream { // provider order — lets the framework store the assistant turn // verbatim and (≥0.5.72) route the round's prose live. roundContent: response.content, + ...(this.supportsInjectedMessages !== undefined + ? { supportsInjectedMessages: this.supportsInjectedMessages } + : {}), }, } as StreamEvent); } else { @@ -167,6 +175,8 @@ export class MockMembrane { responses: NormalizedResponse[] = []; calls: NormalizedRequest[] = []; lastStream: MockYieldingStream | null = null; + /** What the streams' tool-calls events declare (see MockYieldingStream). */ + supportsInjectedMessages: boolean | undefined = undefined; private responseIndex = 0; pushResponse(response: NormalizedResponse): void { @@ -185,7 +195,7 @@ export class MockMembrane { this.calls.push(request); const remaining = this.responses.slice(this.responseIndex); this.responseIndex = this.responses.length; - const stream = new MockYieldingStream(remaining); + const stream = new MockYieldingStream(remaining, this.supportsInjectedMessages); this.lastStream = stream; return stream; } From 88db26a508071036bf2a9dbf0f7968b579d557bc Mon Sep 17 00:00:00 2001 From: Duncan-2410 Date: Thu, 15 Oct 2026 08:42:59 -0400 Subject: [PATCH 24/34] fix(deferral): what is held for an agent that's going away lands before it goes (R1-4) The turn-end flush in driveStream's finally is where held entries land: a message that can't be injected and everything queued behind it. Two agents leave the agent map before that flush runs, so it found no agent to take their entries, and addMessage dropped them as addressed to an agent that isn't registered: - a conversation fork whose closure turn just ended, disposed a few lines above the flush; - an ephemeral run, which runEphemeralToCompletion's finally deregisters once the run settles, before the stream's finally runs. The ordering is main's. This PR holds more for that flush: a message held at a tool boundary, and what is queued behind it. A closing fork is now disposed after its turn-end flush. An ephemeral run's finally drains what is queued for the run and stores it before deregistering it, past the run's deferral. The run's namespace stays in the store, so the messages stay with it. A quiesced host defers them again instead, as stop() does, and a flush that meets them after the run has gone drops them, as addMessage drops any write for an agent that isn't registered. A message reaches an ephemeral run's or a fork's queue when a write addresses that agent by name, as the event gate's do. Each test addresses the agent while its turn is alive, as the gate would, and finds the messages after its last reply. Anarchid's nightly review of e493705 found this (R1-4). Co-Authored-By: Claude Opus 5.5 --- src/framework.ts | 26 +++++++++--- test/conversation-routing.test.ts | 35 ++++++++++++++++ test/ephemeral-settle.test.ts | 67 +++++++++++++++++++++++++++++++ 3 files changed, 122 insertions(+), 6 deletions(-) diff --git a/src/framework.ts b/src/framework.ts index 6b36b176b..5915e264c 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -6016,6 +6016,18 @@ export class AgentFramework { agent.cancelStream(); if (this.ephemeralRuns.get(agent.name) === run) this.ephemeralRuns.delete(agent.name); if (this.agents.get(agent.name) === agent) { + // What is still queued for the run, such as a message held for its + // turn's end, goes to its store now, past its deferral: once the run + // is deregistered, a flush finds no agent to take it and drops it. + // The run's namespace stays in the store for investigation, so the + // messages stay with it. A quiesced host defers them again instead, + // as stop() does, and with the run gone, the flush that meets them + // later drops them. + const queued = this.drainDeferredFor(agent.name); + if (queued.length > 0) { + this.readdDrained(queued, `end of ephemeral run ${agent.name}`, { bypassDeferral: !this.quiesced }); + this.ackDeferredWrites(); + } // Exact-generation disposal owns these name-keyed entries. Remove them // before deregistration; the late driveStream finally will see the // generation mismatch and leave any future owner untouched. @@ -13478,12 +13490,6 @@ export class AgentFramework { this.pendingAssistantBlocks.delete(agent.name); } - // A conversation fork whose TTL closure turn just finished is done for - // good — dispose it so the agent map doesn't grow monotonically. - if (ownsPhysicalStream && this.closingConversationAgents.has(agent.name)) { - this.disposeConversationAgent(agent.name); - } - // The turn is torn down — release the turn-alive marker BEFORE the // deferred flush below, so the flush appends at the settled tail. // Token-matched: if a successor turn already owns the agent (endTurn @@ -13509,6 +13515,14 @@ export class AgentFramework { this.readdDrained(this.drainDeferredFor(agent.name), 'turn-end flush'); this.ackDeferredWrites(); } + + // A conversation fork whose TTL closure turn just finished is done for + // good — dispose it so the agent map doesn't grow monotonically. After + // the flush above, so what was held for the turn's end lands in the + // fork's store instead of being dropped as addressed to no agent. + if (ownsPhysicalStream && this.closingConversationAgents.has(agent.name)) { + this.disposeConversationAgent(agent.name); + } } } diff --git a/test/conversation-routing.test.ts b/test/conversation-routing.test.ts index c6c6c6b0a..6622b54b6 100644 --- a/test/conversation-routing.test.ts +++ b/test/conversation-routing.test.ts @@ -629,6 +629,41 @@ describe('Conversation routing', () => { await framework.stop(); }); + it('a write deferred for a fork during its closure turn is stored in the fork before it is disposed', async () => { + const IDLE_TTL_MS = 60_000; + membrane.pushResponse(createMockResponse([{ type: 'text', text: 'hello!' }])); + const framework = await makeFramework({ idleTtlMs: IDLE_TTL_MS }); + framework.pushEvent(incomingEvent({ channelId: 'slack:D4', text: 'hi', channelType: 'im' })); + await framework.runUntilIdle(); + const g1 = framework.getAgent('conversation-slack-D4-g1')!; + assert.ok(g1, 'g1 fork should exist after spawn'); + + forceExpiry(framework, 'slack:D4', IDLE_TTL_MS); + membrane.pushResponse(createMockResponse([{ type: 'text', text: 'Final report posted.' }])); + // While the closure turn is alive, a write addressed to the fork is + // deferred to that turn's end. + const internals = framework as unknown as { + addMessage(participant: string, content: ContentBlock[], metadata?: undefined, opts?: { forAgent?: string }): string; + }; + const streamYielding = membrane.streamYielding.bind(membrane); + let deferred: string | undefined; + membrane.streamYielding = (request, options) => { + deferred = internals.addMessage('alice', [{ type: 'text', text: 'one more thing' }], undefined, + { forAgent: 'conversation-slack-D4-g1' }); + return streamYielding(request, options); + }; + framework.pushEvent({ type: 'external-message', source: 'test', content: 'tick', metadata: {} } as unknown as ProcessEvent); + await framework.runUntilIdle(); + + assert.equal(deferred, '', 'deferred: the closure turn was alive'); + assert.equal(framework.getAgent('conversation-slack-D4-g1'), null, 'disposed once its closure turn finished'); + const stored = g1.getContextManager().getAllMessages().map((m) => JSON.stringify(m.content)); + assert.ok(stored[stored.length - 2]!.includes('Final report posted.'), stored.join('\n')); + assert.ok(stored[stored.length - 1]!.includes('one more thing'), 'stored after the closure reply, before the fork was disposed'); + + await framework.stop(); + }); + it('restart does not reuse generation names or re-seed an existing namespace', async () => { membrane.pushResponse(createMockResponse([{ type: 'text', text: 'noted' }])); const fw1 = await makeFramework({}, 'restart.chronicle'); diff --git a/test/ephemeral-settle.test.ts b/test/ephemeral-settle.test.ts index db251e5d0..58bb2ddcc 100644 --- a/test/ephemeral-settle.test.ts +++ b/test/ephemeral-settle.test.ts @@ -63,6 +63,30 @@ class EphemeralToolModule implements Module { } } +/** A tool that, while its call runs, has `duringCall` run: there, a test + * addresses the run as the event gate does, through the framework's + * addMessage with `forAgent`. */ +class DuringCallToolModule implements Module { + readonly name = 'test'; + duringCall: (() => void) | null = null; + + async start(_ctx: ModuleContext): Promise {} + async stop(): Promise {} + + getTools(): ToolDefinition[] { + return [{ name: 'echo', description: 'Echo input', inputSchema: { type: 'object', properties: {} } }]; + } + + async handleToolCall(_call: ToolCall): Promise { + this.duringCall?.(); + return { success: true, data: { ok: true } }; + } + + async onProcess(_event: ProcessEvent, _state: ProcessState): Promise { + return {}; + } +} + class ThrowingMembrane extends MockMembrane { readonly errors: Error[]; @@ -406,6 +430,49 @@ describe('runEphemeralToCompletion settle signal', () => { } }); + it('what is queued for a run when it ends is stored in its namespace, not dropped once it is deregistered', async () => { + const { tempDir, storePath } = tempStorePath('ephemeral-queued-'); + const membrane = new MockMembrane(); + membrane.pushResponse(createMockResponse([ + { type: 'tool_use', id: 'call_1', name: 'test--echo', input: {} }, + ], 'tool_use')); + membrane.pushResponse(createMockResponse([{ type: 'text', text: 'Done.' }])); + + const tools = new DuringCallToolModule(); + const framework = await AgentFramework.create({ + storePath, + membrane: membrane.asMembrane(), + agents: [], + modules: [tools], + syncIntervalMs: 0, + }); + // While the run's tool call is pending, two messages for it are deferred: + // one named as the run itself, which waits for its turn's end, and one + // queued behind it. + const internals = framework as unknown as { + addMessage(participant: string, content: Array<{ type: 'text'; text: string }>, metadata?: undefined, opts?: { forAgent?: string }): string; + }; + const returned: string[] = []; + tools.duringCall = () => { + returned.push(internals.addMessage('worker', [{ type: 'text', text: 'my own words' }], undefined, { forAgent: 'worker' })); + returned.push(internals.addMessage('Antra', [{ type: 'text', text: 'look left' }], undefined, { forAgent: 'worker' })); + }; + + try { + const { agent, contextManager } = await createEphemeral(framework, 'worker'); + const result = await runStartedEphemeral(framework, agent, contextManager); + assert.strictEqual(result.speech, 'Done.'); + assert.deepStrictEqual(returned, ['', ''], 'both were deferred: the run was waiting on its tool'); + assert.strictEqual(framework.getAgent(agent.name), null, 'deregistered'); + const stored = contextManager.getAllMessages().map((m) => + `${m.participant}: ${m.content.map((b) => (b.type === 'text' ? (b as { text: string }).text : `<${b.type}>`)).join(' ')}`); + assert.deepStrictEqual(stored.slice(-3), ['worker: Done.', 'worker: my own words', 'Antra: look left'], + 'held to the turn end, and stored after the run\'s last reply'); + } finally { + await stopAndRemove(framework, tempDir); + } + }); + it('resolves through the endTurn tool-result path', async () => { const { tempDir, storePath } = tempStorePath('ephemeral-endturn-'); const membrane = new MockMembrane(); From fc04450ec3e078b90154dcb5cfd7d1e4f170547c Mon Sep 17 00:00:00 2001 From: Duncan-2410 Date: Thu, 15 Oct 2026 08:43:23 -0400 Subject: [PATCH 25/34] fix(deferral): boot recovery checks each entry's seq, id and target (R1-5) Boot recovery took any number as a seq, any string as an id, and any value as forAgent. The framework writes none of the values that break them, so each comes from damage or a hand edit, which is what this check is for: - JSON reads 1e400 as Infinity, and boot set deferredSeq to the largest seq, so it became Infinity: every later deferral took that seq, since Infinity plus one is Infinity, and the next persist wrote it as null. A negative seq put its entry ahead of everything. - The queue takes an id as one entry: boot skipped every entry whose id had landed, both entries with an id when one had, and a requeue put back only one entry for an id. - An empty or zero forAgent was routed to the primary, and any other value that isn't text was dropped by addMessage with a line. Now a seq must be a whole number from 1 up to Number.MAX_SAFE_INTEGER. An entry without one, from a file that predates seq or from damage, is numbered after every valid seq, in file order, so the counter stays finite and the rest keep their order, and one whose seq is present but isn't such a number is named in a line. The first entry with an id is kept, and a later one is dropped and named. An entry whose forAgent is present but empty or not text is dropped as malformed, named as such entries are: the agent it meant can't be known, and sent to the primary it could put another agent's message in the primary's window. The recovery file is rewritten whenever any of this changed the queue. Anarchid suggested writing dropped entries to a sidecar file. It isn't here: these entries come only from damage, each line names its entry by id and participant, and a sidecar would be a second recovery format that nothing reads. Anarchid's nightly review of e493705 found this (R1-5). Co-Authored-By: Claude Opus 5.5 --- src/framework.ts | 89 ++++++++++++++++++++++++++++----------- test/host-quiesce.test.ts | 76 ++++++++++++++++++++++++++++++--- 2 files changed, 134 insertions(+), 31 deletions(-) diff --git a/src/framework.ts b/src/framework.ts index 5915e264c..43a8c090a 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -4975,19 +4975,26 @@ export class AgentFramework { } } // An entry the queue's readers can't handle has no place in it: one whose - // participant isn't text, whose content isn't an array, or that holds a - // block that isn't an object, is dropped here with a line naming it, - // rather than held. Reading a null block's type throws, at the tool - // boundary and in the store. The store keeps a string or number block as - // given, but the context manager's token estimate keys a WeakMap on each - // block, which such a value can't be: once one is stored, every compile - // for that agent throws, and a restart doesn't clear it. What a block - // holds, such as a tool result's content, isn't checked here, as it isn't - // for a fresh write. - const wellFormed = (m: unknown): m is Omit & { id?: string; seq?: number } => - !!m && typeof m === 'object' && typeof (m as { participant?: unknown }).participant === 'string' - && Array.isArray((m as { content?: unknown }).content) - && (m as { content: unknown[] }).content.every((b) => !!b && typeof b === 'object'); + // participant isn't text, whose content isn't an array, that holds a + // block that isn't an object, or whose `forAgent` is present but not a + // name, is dropped here with a line naming it, rather than held. Reading + // a null block's type throws, at the tool boundary and in the store. The + // store keeps a string or number block as given, but the context + // manager's token estimate keys a WeakMap on each block, which such a + // value can't be: once one is stored, every compile for that agent + // throws, and a restart doesn't clear it. What a block holds, such as a + // tool result's content, isn't checked here, as it isn't for a fresh + // write. The framework writes `forAgent` only as a non-empty name, so + // another value comes from damage or a hand edit, and the agent it + // meant can't be known: sent to the primary, as a falsy one would be, it + // could put another agent's message in the primary's window. + const wellFormed = (m: unknown): m is Omit & { id?: unknown; seq?: unknown } => { + if (!m || typeof m !== 'object') return false; + const { participant, content, forAgent } = m as { participant?: unknown; content?: unknown; forAgent?: unknown }; + return typeof participant === 'string' + && Array.isArray(content) && content.every((b) => !!b && typeof b === 'object') + && (forAgent === undefined || (typeof forAgent === 'string' && forAgent !== '')); + }; const malformed = data.filter((m) => !wellFormed(m)); for (const m of malformed) { // What the entry holds there: a value that isn't text by its type. @@ -5002,16 +5009,48 @@ export class AgentFramework { `${field('id')} (${field('participant')})`, ); } - const candidates = data - .filter(wellFormed) - .map((m, i) => ({ - ...m, - id: typeof m.id === 'string' ? m.id : randomUUID(), - // Files that predate `seq` are in write order already. - seq: typeof m.seq === 'number' ? m.seq : i + 1, - })) + // One entry per id: the boot dedup and the ack each take an id as one + // write, so an id that comes again is dropped with a line, and the first + // entry with it is kept. An entry with no id is given a fresh one. + const seen = new Set(); + const kept: Array & { id: string; seq?: unknown }> = []; + let duplicates = 0; + for (const m of data.filter(wellFormed)) { + const id = typeof m.id === 'string' && m.id !== '' ? m.id : randomUUID(); + if (seen.has(id)) { + duplicates++; + console.error( + `[host-mode] a recovered deferred context write repeats id ${id} (${m.participant}) and was dropped; ` + + `the first entry with that id is kept`, + ); + continue; + } + seen.add(id); + kept.push({ ...m, id }); + } + // A seq is a positive whole number, the entry's place in arrival order. + // An entry without one (a file that predates seq, which is in write + // order already) or with any other value (damage or a hand edit: + // JSON's 1e400 reads as Infinity) is numbered after every valid seq, in + // file order, so the counter stays finite and the rest keep their order. + const validSeq = (seq: unknown): seq is number => Number.isSafeInteger(seq) && (seq as number) > 0; + let lastSeq = 0; + for (const m of kept) if (validSeq(m.seq)) lastSeq = Math.max(lastSeq, m.seq); + let renumbered = 0; + const candidates: DeferredWrite[] = kept + .map((m) => { + if (validSeq(m.seq)) return { ...m, seq: m.seq }; + renumbered++; + if (m.seq !== undefined) { + console.error( + `[host-mode] a recovered deferred context write (${m.id}) has seq ${String(m.seq)}, not a positive ` + + `whole number; it is queued after the others`, + ); + } + return { ...m, seq: ++lastSeq }; + }) .sort(bySeq); - this.deferredSeq = Math.max(this.deferredSeq, ...candidates.map((m) => m.seq)); + this.deferredSeq = Math.max(this.deferredSeq, lastSeq); // Exactly-once: a message that landed before its ack was written is // already in the store under its deferredWriteId — do not replay it. // The scan covers every slot the interrupted batch could occupy (from @@ -5028,9 +5067,9 @@ export class AgentFramework { this.deferredMessages.push(...restored); this.deferredWritesPersisted = true; // Rewrite the durable queue without the already-landed entries so a - // second interrupted boot does not re-scan them, and without the - // malformed ones. - if (landed.size > 0 || malformed.length > 0) this.persistDeferredWrites(); + // second interrupted boot does not re-scan them, without the malformed + // and repeated ones, and with the seqs given here. + if (landed.size > 0 || malformed.length > 0 || duplicates > 0 || renumbered > 0) this.persistDeferredWrites(); return restored.length; } catch (err) { console.error('[host-mode] failed to restore persisted deferred context writes:', err); diff --git a/test/host-quiesce.test.ts b/test/host-quiesce.test.ts index 03dccd308..bc65ce740 100644 --- a/test/host-quiesce.test.ts +++ b/test/host-quiesce.test.ts @@ -864,20 +864,23 @@ test('boot recovery drops a malformed recovered deferred write, naming it, and r // Entries no write of the framework's makes, as other code could leave // them in the recovery file: a null block, a string block and a number - // block beside a text block, content that isn't blocks, and a participant - // that isn't text, which the line names by its type. Once stored, a string - // or number block would make every compile for the agent throw, since the - // context manager's token estimate keys a WeakMap on each block. + // block beside a text block, content that isn't blocks, a participant + // that isn't text, which the line names by its type, and a target that + // is empty or isn't text. Once stored, a string or number block would + // make every compile for the agent throw, since the context manager's + // token estimate keys a WeakMap on each block. const file = join(storePath, 'recovery', 'deferred-writes.json'); const doc = JSON.parse(readFileSync(file, 'utf8')) as { pending: Array> }; assert.equal(doc.pending.length, 2); - const dropped = ['null-block', 'string-block', 'number-block', 'not-blocks', 'number-participant']; + const dropped = ['null-block', 'string-block', 'number-block', 'not-blocks', 'number-participant', 'empty-target', 'number-target']; doc.pending.splice(1, 0, { id: 'null-block', seq: 100, participant: 'user', content: [{ type: 'text', text: 'x' }, null] }, { id: 'string-block', seq: 101, participant: 'user', content: [{ type: 'text', text: 'x' }, 'a bare string'] }, { id: 'number-block', seq: 102, participant: 'user', content: [{ type: 'text', text: 'x' }, 42] }, { id: 'not-blocks', seq: 103, participant: 'user', content: 'x' }, { id: 'number-participant', seq: 104, participant: 42, content: [{ type: 'text', text: 'x' }] }, + { id: 'empty-target', seq: 105, participant: 'user', content: [{ type: 'text', text: 'x' }], forAgent: '' }, + { id: 'number-target', seq: 106, participant: 'user', content: [{ type: 'text', text: 'x' }], forAgent: 7 }, ); writeFileSync(file, JSON.stringify(doc)); @@ -885,7 +888,7 @@ test('boot recovery drops a malformed recovered deferred write, naming it, and r framework = await AgentFramework.create(config()); console.error = error; assert.equal(framework.getHostModeStatus().deferredWrites, 2, 'the well-formed entries are restored'); - for (const entry of ['null-block (user)', 'string-block (user)', 'number-block (user)', 'not-blocks (user)', 'number-participant (participant of type number)']) { + for (const entry of ['null-block (user)', 'string-block (user)', 'number-block (user)', 'not-blocks (user)', 'number-participant (participant of type number)', 'empty-target (user)', 'number-target (user)']) { assert.ok(lines.some((line) => line.includes(`malformed and was dropped: ${entry}`)), `${entry} is named:\n${lines.join('\n')}`); } const rewritten = readFileSync(file, 'utf8'); @@ -902,6 +905,67 @@ test('boot recovery drops a malformed recovered deferred write, naming it, and r } }); +test('boot recovery keeps one entry per id, and numbers a seq that isn\'t a positive whole number after the others', { timeout: 60_000 }, async () => { + const dir = mkdtempSync(join(tmpdir(), 'host-quiesce-identity-')); + const storePath = join(dir, 'store'); + const config = () => ({ + storePath, + membrane: new MockMembrane().asMembrane(), + agents: [{ name: 'agent', model: 'test-model', systemPrompt: 'test' }], + modules: [], + }); + let framework = await AgentFramework.create(config()); + const lines: string[] = []; + const error = console.error; + try { + await framework.quiesce({ reason: 'surgery' }); + const before = framework.getAgent('agent')!.getContextManager().getAllMessages().length; + const add = (framework as unknown as { addMessage(p: string, c: unknown[]): unknown }).addMessage.bind(framework); + add('user', [{ type: 'text', text: 'first' }]); + add('user', [{ type: 'text', text: 'second' }]); + await framework.stop(); + + // What the framework never writes, as damage or a hand edit could leave + // it: an id that comes twice, a seq of 1e400 (JSON reads it as Infinity) + // and a negative one. The entries with a bad seq come before 'second' in + // the file. Nothing here is malformed, so only these make boot rewrite it. + const file = join(storePath, 'recovery', 'deferred-writes.json'); + const doc = JSON.parse(readFileSync(file, 'utf8')) as { pending: Array> }; + assert.equal(doc.pending.length, 2); + const entry = (id: string, seq: unknown, text: string) => + ({ id, seq, participant: 'user', content: [{ type: 'text', text }] }); + doc.pending.splice(1, 0, + entry('infinite-seq', '__INFINITY__', 'infinite seq'), + entry('negative-seq', -3, 'negative seq'), + entry('twice', 50, 'first with its id'), + entry('twice', 51, 'second with its id'), + ); + writeFileSync(file, JSON.stringify(doc).replace('"__INFINITY__"', '1e400')); + + console.error = (...args: unknown[]) => { lines.push(args.map(String).join(' ')); }; + framework = await AgentFramework.create(config()); + console.error = error; + const named = (text: string) => assert.ok(lines.some((line) => line.includes(text)), `${text}:\n${lines.join('\n')}`); + named('repeats id twice (user) and was dropped; the first entry with that id is kept'); + named('(infinite-seq) has seq Infinity, not a positive whole number; it is queued after the others'); + named('(negative-seq) has seq -3, not a positive whole number; it is queued after the others'); + assert.equal(framework.getHostModeStatus().deferredWrites, 5); + + const rewritten = JSON.parse(readFileSync(file, 'utf8')) as { pending: Array<{ id: string; seq: unknown }> }; + assert.equal(rewritten.pending.length, 5, 'the recovery file is rewritten without the repeated id'); + assert.ok(rewritten.pending.every((m) => Number.isSafeInteger(m.seq) && (m.seq as number) > 0), JSON.stringify(rewritten.pending)); + + await framework.resume(); + const texts = framework.getAgent('agent')!.getContextManager().getAllMessages().slice(before) + .map((m) => (m.content[0] as { text: string }).text); + assert.deepEqual(texts, ['first', 'second', 'first with its id', 'infinite seq', 'negative seq']); + } finally { + console.error = error; + await framework.stop(); + rmSync(dir, { recursive: true, force: true }); + } +}); + test('context writes deferred while quiesced survive a restart and land at resume', async () => { const dir = mkdtempSync(join(tmpdir(), 'host-quiesce-deferred-')); const storePath = join(dir, 'store'); From 323229fc5bfbab676d46cb51c2267b5c4f6bec9c Mon Sep 17 00:00:00 2001 From: Duncan-2410 Date: Thu, 15 Oct 2026 08:43:32 -0400 Subject: [PATCH 26/34] docs(deferral): say how a flush acknowledges, and how long deferrals live only in memory (R1-6, R1-3) flushDeferredWrites' docstring said it acknowledged each write as it landed, rewriting the persisted queue after every message. It acknowledges the batch once: the chronicle is synced, then the persisted queue is rewritten without it. A crash mid-flush leaves the batch in the file, and boot skips each entry the store already holds under its deferredWriteId (R1-6). persistDeferredWrites' docstring said that deferrals outside a quiesce window flush within the turn, as before. They still land within the turn, but a message held for the turn's end, and what is queued behind it, wait for that end. On a stream that can't carry injected messages, so does everything deferred during the turn. A crash before then loses them. The docstring now says so (R1-3). Nothing new is persisted: mirroring the queue while a cut holds would rewrite the recovery file on every deferral for the rest of the turn, which on such a stream would be every turn. Anarchid's nightly review of e493705 found both. Co-Authored-By: Claude Opus 5.5 --- src/framework.ts | 28 +++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/src/framework.ts b/src/framework.ts index 43a8c090a..558e6586b 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -4739,16 +4739,18 @@ export class AgentFramework { } /** - * Land every deferred write whose target has no turn alive, one at a time, - * acknowledging each durably as it lands (the persisted queue is rewritten - * after every message). A crash mid-flush therefore replays exactly the - * un-acked remainder at the next boot — no loss, no duplicates. The batch - * is stored in queue order, whatever its targets (writeDrainedInOrder). A - * write the store rejects stays queued, logged, with what is queued behind - * it for its target, so the flush never stores that target's later entries - * ahead of it, and other targets' entries go on. (A write that isn't - * deferred can still land ahead of it, such as one for an idle agent once - * the host has resumed.) A boot that meets it again logs it and goes on. + * Land every deferred write whose target has no turn alive, as one batch + * acknowledged once: the chronicle is synced, then the persisted queue is + * rewritten without the batch (ackDeferredWrites). Until then the batch + * stays in the durable queue, so a crash mid-flush replays it at the next + * boot, which skips each entry that already landed, by its deferredWriteId + * — no loss, no duplicates. The batch is stored in queue order, whatever + * its targets (writeDrainedInOrder). A write the store rejects stays + * queued, logged, with what is queued behind it for its target, so the + * flush never stores that target's later entries ahead of it, and other + * targets' entries go on. (A write that isn't deferred can still land + * ahead of it, such as one for an idle agent once the host has resumed.) + * A boot that meets it again logs it and goes on. * * Per-target: an agent whose turn is still alive keeps ITS messages * deferred for its own turn boundary; everyone else's flush now. @@ -4911,7 +4913,11 @@ export class AgentFramework { * window is operator-length, not turn-length. Lives in the * branch-independent recovery file so a rollback cannot orphan it. * Deferrals outside a quiesce window (turn-alive, mid-tool-cycle) are - * still memory-only: they flush within the turn, as before. + * still memory-only: they land within the turn, at a tool boundary or at + * its end, and a crash before then loses them. That window runs to the + * turn's end for a message held there and what is queued behind it, and, + * on a stream that can't carry injected messages, for every message + * deferred during the turn. */ private persistDeferredWrites(): void { try { From 1cc89243cf8e1b068c952241bae869961e65126f Mon Sep 17 00:00:00 2001 From: Duncan-2410 Date: Thu, 15 Oct 2026 08:44:57 -0400 Subject: [PATCH 27/34] docs(changelog): the fragment covers this round's changes The fragment now says that an injected message counts as read only when the stream says it carries injected messages, and what a stream that says it can't, or doesn't say, gets (R1-1). It says that a write that throws after its message was stored counts as written (R1-2), that a held message lives only in memory until the turn's end (R1-3), that what is deferred for a closing fork or an ending ephemeral run is stored before it goes (R1-4), and what boot recovery now checks of each entry's seq, id and target (R1-5). Co-Authored-By: Claude Opus 5.5 --- .../mid-turn-window-only-held.fixed.md | 48 ++++++++++++++----- 1 file changed, 35 insertions(+), 13 deletions(-) diff --git a/changelog.d/mid-turn-window-only-held.fixed.md b/changelog.d/mid-turn-window-only-held.fixed.md index 8c5857aca..de0b98973 100644 --- a/changelog.d/mid-turn-window-only-held.fixed.md +++ b/changelog.d/mid-turn-window-only-held.fixed.md @@ -5,19 +5,30 @@ behind it wait with it, so the queue keeps its order; messages queued before it are still heard at the boundary. The cost: a message addressed to the agent but queued behind a held one isn't heard mid-turn, and waits for - the turn's end. Stored between rounds, a held message sat before the turn's - later replies in every later compile although their requests never held - it: a prompt-cache divergence, and on a provider that binds signed thinking - to the prefix it was minted under (anima-research/context-manager#155, - whose binding anima-research/context-manager#160 walks), a reply whose - thinking is refused. + the turn's end, and a held message and what is queued behind it live only + in memory until then, so a crash during the turn loses them. Stored + between rounds, a held message sat before the turn's later replies in + every later compile although their requests never held it: a + prompt-cache divergence, and on a provider that binds signed thinking to + the prefix it was minted under (anima-research/context-manager#155, whose + binding anima-research/context-manager#160 walks), a reply whose thinking + is refused. - A message injected into the live stream at a tool boundary now counts as - read for RFC-006 coalescing: the consumed watermark moves when messages - are injected, not only at a compile. Before, an injected message sat above - the watermark of the turn's compile, so a coalesced replacement for its - subject could remove a message the agent had heard and answered. Within a - branch the watermark only advances: a compile that started before such a - mark doesn't roll it back when it finishes. + read for RFC-006 coalescing when the stream says it carries injected + messages (membrane's `ToolContext.supportsInjectedMessages`, true on its + native path in releases after 0.5.87): the consumed watermark moves when + messages are injected, not only at a compile. Before, an injected message + sat above the watermark of the turn's compile, so a coalesced replacement + for its subject could remove a message the agent had heard and answered. + Within a branch the watermark only advances: a compile that started before + such a mark doesn't roll it back when it finishes. A stream that doesn't + say, as on membrane 0.5.87, still gets injected messages, and they stay + unread, as before: 0.5.87's XML tool path drops them with a warning. +- A stream that says it can't carry injected messages (membrane's XML + prefill path, in releases after 0.5.87) gets none: everything deferred + for the agent during the turn waits for the turn's end, after its last + round, as a held message does, instead of being stored between rounds the + stream never showed it. - A deferred write that a flush defers again does so whatever its content, under its id and in its place in the order writes arrived. That happens while its target's own turn, stream or tool cycle is alive, or the host is @@ -53,6 +64,10 @@ persisted queue for boot recovery. An entry for a target no longer registered (a disposed fork) is dropped from a memory-only queue at exit, as before. +- What is deferred for a conversation fork whose closure turn ends, or for + an ephemeral run that ends, is now stored in that agent's store before the + fork is disposed or the run deregistered. Before, the turn-end flush ran + after, found no agent to take it, and dropped it. - A deferred write whose store throws goes back into the queue under its id, and so does every entry queued behind it for that agent, so none of them lands ahead of an earlier one, and a queued `tool_use` that can't be @@ -80,6 +95,10 @@ the file's size cap; past it the file isn't rewritten, as the host log warns, and what's deferred after that is kept in memory only. Only the host log names the held write. +- A deferred write that throws after its message was stored, such as one + whose store listener throws, counts as written: it isn't put back into the + queue and stored a second time, and at a tool boundary it's injected with + the rest. The log line says it was stored. - Boot recovery drops a recovered deferred write whose participant isn't text, whose content isn't an array, or that holds a block that isn't an object, and logs it by id and participant, rather than holding it in the @@ -90,4 +109,7 @@ whose content isn't an array was dropped without a line, and one with a block that isn't an object was kept: a null block could never be stored, and a string or number block, once stored, made every compile for its - agent fail. + agent fail. Boot recovery also drops an entry whose `forAgent` is present + but empty or not text, keeps the first entry with an id and drops and + names a later one, and numbers an entry whose `seq` isn't a positive whole + number after the others, naming it unless it has none. From eadf9a90ec91d2a4dfb953730bb460e81d757de5 Mon Sep 17 00:00:00 2001 From: Duncan-2410 Date: Thu, 15 Oct 2026 22:09:25 -0400 Subject: [PATCH 28/34] fix(deferral): a write that threw after storing carries what landed; a closing fork waits for the frame that ends its turn From an internal review of this round: - A deferred write that throws after storing counts as written, and now what landed is what the tool boundary injects, read back from the store with blobs resolved. For a body the strategy shards, the throw stops the append loop, so what landed can be its first shards alone; before, the boundary injected the whole body while the store held only those shards, so the request carried more than every later compile. The log names how many shards landed. At most once, as boot's scan for landed writes already counts a landed shard: requeued, the landed shards would be stored twice, and the rest alone couldn't be appended later without splitting the body. - A copy found that way through readdDrained, which every flush but the tool boundary's and the turn start's goes through (turn end, resume, post-surgery, withdrawn turn, failed start, puppet call, an ending ephemeral run and stop()), now gets the notes addMessage takes after a store, such as an arrival for the agent's open turn. The copy check and addMessage now resolve an entry's target, and take those notes, through helpers the two share. - A closing conversation fork is disposed only by the frame that ends its closure turn (frameReachedTerminal), after its turn-end flush. A frame that hands the turn to a successor, at a guard or budget restart, used to dispose it too, so the successor found no agent and the turn never finished. This is a change from main, where the disposal had no such gate. - The copy check's getAllMessages fallbacks go: the context-manager floor is ^0.14.1, which has getMessageCount and getMessageWindow. - Boot recovery names an entry it gives an id to, gives a seq that isn't a number by its type rather than its value, and says the bound (Number.MAX_SAFE_INTEGER) in the line. - Text: the fragment and docstrings say when deferrals are persisted (while quiesced, or while a restored queue is pending), that 0.5.87's XML path doesn't carry injected messages and warns that they come next turn, that a write is requeued when its store throws before storing it, that boot keeps the first well-formed entry with an id, the seq bound, and what a quiesced host does with what's queued for an ending run or closing fork. Tests: a sharded body whose listener throws after its first shard; a copy found through readdDrained is noted as an arrival; the turn-end copy's log line; a closure turn that restarts finishes in its fork and stores what was held for it; boot names an entry with no id, a zero seq and a string seq; a recovery file that predates seq keeps its order and is rewritten with seqs. Co-Authored-By: Claude Opus 5.5 --- .../mid-turn-window-only-held.fixed.md | 106 +++++---- src/framework.ts | 215 +++++++++++------- test/conversation-routing.test.ts | 69 +++++- test/deferred-window-only.test.ts | 82 ++++++- test/host-quiesce.test.ts | 78 ++++++- 5 files changed, 404 insertions(+), 146 deletions(-) diff --git a/changelog.d/mid-turn-window-only-held.fixed.md b/changelog.d/mid-turn-window-only-held.fixed.md index de0b98973..5c558d56c 100644 --- a/changelog.d/mid-turn-window-only-held.fixed.md +++ b/changelog.d/mid-turn-window-only-held.fixed.md @@ -3,16 +3,17 @@ waits for the turn's end, after its last round, instead of being stored at the boundary between rounds the stream never showed it. Messages queued behind it wait with it, so the queue keeps its order; messages queued - before it are still heard at the boundary. The cost: a message addressed to - the agent but queued behind a held one isn't heard mid-turn, and waits for - the turn's end, and a held message and what is queued behind it live only - in memory until then, so a crash during the turn loses them. Stored - between rounds, a held message sat before the turn's later replies in - every later compile although their requests never held it: a - prompt-cache divergence, and on a provider that binds signed thinking to - the prefix it was minted under (anima-research/context-manager#155, whose - binding anima-research/context-manager#160 walks), a reply whose thinking - is refused. + before it are still heard at the boundary. The cost: a message addressed + to the agent but queued behind a held one isn't heard mid-turn, and waits + for the turn's end, and unless the queue is persisted (below), a held + message and what is queued behind it live only in memory until then, so a + crash during the turn loses them. Stored between rounds, a held message + sat before the turn's later replies in every later compile although their + requests never held it: a prompt-cache divergence, and on a provider that + binds signed thinking to the prefix it was minted under + (anima-research/context-manager#155, whose binding + anima-research/context-manager#160 walks), a reply whose thinking is + refused. - A message injected into the live stream at a tool boundary now counts as read for RFC-006 coalescing when the stream says it carries injected messages (membrane's `ToolContext.supportsInjectedMessages`, true on its @@ -23,7 +24,8 @@ Within a branch the watermark only advances: a compile that started before such a mark doesn't roll it back when it finishes. A stream that doesn't say, as on membrane 0.5.87, still gets injected messages, and they stay - unread, as before: 0.5.87's XML tool path drops them with a warning. + unread, as before: 0.5.87's XML tool path doesn't carry them, and warns + that they'll come next turn instead. - A stream that says it can't carry injected messages (membrane's XML prefill path, in releases after 0.5.87) gets none: everything deferred for the agent during the turn waits for the turn's end, after its last @@ -32,9 +34,9 @@ - A deferred write that a flush defers again does so whatever its content, under its id and in its place in the order writes arrived. That happens while its target's own turn, stream or tool cycle is alive, or the host is - quiesced. (A write whose store throws goes back into the queue too; see - below.) A `tool_result` used to land regardless, apart from the `tool_use` - it answers, which went back into the queue. + quiesced. (A write whose store throws before storing it goes back into the + queue too; see below.) A `tool_result` used to land regardless, apart from + the `tool_use` it answers, which went back into the queue. - A write that waits now waits on its own target's state: the target's turn, stream and tool cycle, or a quiesced host. (A fresh `tool_result` still doesn't wait.) Before, a write for the primary agent waited out every @@ -67,38 +69,47 @@ - What is deferred for a conversation fork whose closure turn ends, or for an ephemeral run that ends, is now stored in that agent's store before the fork is disposed or the run deregistered. Before, the turn-end flush ran - after, found no agent to take it, and dropped it. -- A deferred write whose store throws goes back into the queue under its id, - and so does every entry queued behind it for that agent, so none of them - lands ahead of an earlier one, and a queued `tool_use` that can't be - stored keeps the `tool_result` queued behind it from being stored alone. - (A stored `tool_use` whose `tool_result` then can't be stored stays alone - until the result lands at a later flush.) Other agents' entries go on, in - the order they arrived. The log line names the entry at each flush, and - one that keeps throwing holds what is deferred behind it for that agent - until it can be stored. The hold is the queue's: a write that isn't - deferred is stored at once and can land ahead of a held entry. That's a - fresh `tool_result`, a message assembled into the agent's turn before its - compile (RFC-006 coalescing), and any write while the agent's turn, stream - and tool cycle are idle and the host isn't quiesced. So an entry that - keeps throwing holds only what is deferred while the agent's turn, stream - or tool cycle is alive, or the host is quiesced. Before, it stayed in the - un-acked set and the next acknowledgement dropped it from the persisted - queue (at a turn start or a flush it was lost even from memory). At a tool - boundary, nothing from it on is injected. While the framework runs it is - kept; a memory-only queue still loses it when the process exits. A - persisted queue (one a quiesce persisted, or boot recovery restored) keeps - it across restarts: each boot restores it and tries it again with the rest - of the queue, at once or at the resume after a quiesced boot, and logs it - again, unless it's one boot drops (below). While it's held the queue stays - persisted, so every later deferral is written to the recovery file, up to - the file's size cap; past it the file isn't rewritten, as the host log - warns, and what's deferred after that is kept in memory only. Only the - host log names the held write. + after, found no agent to take it, and dropped it. A closing fork is now + disposed by the frame that ends its closure turn: one that hands the turn + to a successor (a guard or budget restart) leaves it. Before, that frame + disposed it, so the successor found no agent and the turn never finished. + A quiesced host defers what is queued for an ending run or a closing fork + again instead, and with the agent gone, the flush that meets it later + drops it. +- A deferred write whose store throws without storing it goes back into the + queue under its id, and so does every entry queued behind it for that + agent, so none of them lands ahead of an earlier one, and a queued + `tool_use` that can't be stored keeps the `tool_result` queued behind it + from being stored alone. (A stored `tool_use` whose `tool_result` then + can't be stored stays alone until the result lands at a later flush.) + Other agents' entries go on, in the order they arrived. The log line names + the entry at each flush, and one that keeps throwing holds what is + deferred behind it for that agent until it can be stored. The hold is the + queue's: a write that isn't deferred is stored at once and can land ahead + of a held entry. That's a fresh `tool_result`, a message assembled into + the agent's turn before its compile (RFC-006 coalescing), and any write + while the agent's turn, stream and tool cycle are idle and the host isn't + quiesced. So an entry that keeps throwing holds only what is deferred + while the agent's turn, stream or tool cycle is alive, or the host is + quiesced. Before, it stayed in the un-acked set and the next + acknowledgement dropped it from the persisted queue (at a turn start or a + flush it was lost even from memory). At a tool boundary, nothing from it + on is injected. While the framework runs it is kept; a memory-only queue + still loses it when the process exits. A persisted queue (one a quiesce + persisted, or boot recovery restored) keeps it across restarts: each boot + restores it and tries it again with the rest of the queue, at once or at + the resume after a quiesced boot, and logs it again, unless it's one boot + drops (below). While it's held the queue stays persisted, so every later + deferral is written to the recovery file, up to the file's size cap; past + it the file isn't rewritten, as the host log warns, and what's deferred + after that is kept in memory only. Only the host log names the held write. - A deferred write that throws after its message was stored, such as one whose store listener throws, counts as written: it isn't put back into the - queue and stored a second time, and at a tool boundary it's injected with - the rest. The log line says it was stored. + queue and stored a second time, and at a tool boundary what was stored is + injected with the rest. For a body the strategy shards, that can be its + first shards alone, since the throw stops the store's append loop: the + rest of the body isn't stored, and the request carries what the store + holds. The log line says it was stored, or how many shards were. - Boot recovery drops a recovered deferred write whose participant isn't text, whose content isn't an array, or that holds a block that isn't an object, and logs it by id and participant, rather than holding it in the @@ -110,6 +121,7 @@ block that isn't an object was kept: a null block could never be stored, and a string or number block, once stored, made every compile for its agent fail. Boot recovery also drops an entry whose `forAgent` is present - but empty or not text, keeps the first entry with an id and drops and - names a later one, and numbers an entry whose `seq` isn't a positive whole - number after the others, naming it unless it has none. + but empty or not text, keeps the first well-formed entry with an id and + drops and names a later one, names an entry with no id as it gives it one, + and numbers an entry whose `seq` isn't a positive whole number up to + `Number.MAX_SAFE_INTEGER` after the others, naming it unless it has none. diff --git a/src/framework.ts b/src/framework.ts index a57ff9990..e6ff01e98 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -489,6 +489,18 @@ interface DeferredWrite { } const bySeq = (a: { seq: number }, b: { seq: number }): number => a.seq - b.seq; +/** A drained entry as writeDrainedInOrder wrote it: the id its store gave it + * (empty when addMessage deferred it again; a sharded body's first shard) + * and what the store holds of it. `afterThrow` marks an entry whose write + * threw after storing it, found in the store and counted as written: its + * content is what landed, which for a sharded body can be its first shards + * alone. */ +interface DrainedWrite { + id: MessageId; + content: ContentBlock[]; + afterThrow?: true; +} + /** Stamp a deferred write's durable id into the metadata it is stored with * (boot recovery dedups replays by it). Idempotent for an already-stamped * message. */ @@ -5093,11 +5105,12 @@ export class AgentFramework { * window is operator-length, not turn-length. Lives in the * branch-independent recovery file so a rollback cannot orphan it. * Deferrals outside a quiesce window (turn-alive, mid-tool-cycle) are - * still memory-only: they land within the turn, at a tool boundary or at - * its end, and a crash before then loses them. That window runs to the - * turn's end for a message held there and what is queued behind it, and, - * on a stream that can't carry injected messages, for every message - * deferred during the turn. + * mirrored only while such a queue is still persisted; otherwise they are + * memory-only: they land within the turn, at a tool boundary or at its + * end, and a crash before then loses them. That window runs to the turn's + * end for a message held there and what is queued behind it, and, on a + * stream that can't carry injected messages, for every message deferred + * during the turn. */ private persistDeferredWrites(): void { try { @@ -5197,12 +5210,18 @@ export class AgentFramework { } // One entry per id: the boot dedup and the ack each take an id as one // write, so an id that comes again is dropped with a line, and the first - // entry with it is kept. An entry with no id is given a fresh one. + // entry with it is kept. An entry with no id is given a fresh one, with + // a line: a copy of it that already landed carries the id it lost, so + // the boot dedup can't find it, and it may be stored again. const seen = new Set(); const kept: Array & { id: string; seq?: unknown }> = []; let duplicates = 0; for (const m of data.filter(wellFormed)) { - const id = typeof m.id === 'string' && m.id !== '' ? m.id : randomUUID(); + const hasId = typeof m.id === 'string' && m.id !== ''; + const id = hasId ? (m.id as string) : randomUUID(); + if (!hasId) { + console.error(`[host-mode] a recovered deferred context write (${m.participant}) has no id; it is given ${id}`); + } if (seen.has(id)) { duplicates++; console.error( @@ -5214,7 +5233,8 @@ export class AgentFramework { seen.add(id); kept.push({ ...m, id }); } - // A seq is a positive whole number, the entry's place in arrival order. + // A seq is a positive whole number no larger than + // Number.MAX_SAFE_INTEGER, the entry's place in arrival order. // An entry without one (a file that predates seq, which is in write // order already) or with any other value (damage or a hand edit: // JSON's 1e400 reads as Infinity) is numbered after every valid seq, in @@ -5228,9 +5248,12 @@ export class AgentFramework { if (validSeq(m.seq)) return { ...m, seq: m.seq }; renumbered++; if (m.seq !== undefined) { + // A value that isn't a number is named by its type: it could be + // any size. + const seq = typeof m.seq === 'number' ? String(m.seq) : m.seq === null ? 'null' : `of type ${typeof m.seq}`; console.error( - `[host-mode] a recovered deferred context write (${m.id}) has seq ${String(m.seq)}, not a positive ` + - `whole number; it is queued after the others`, + `[host-mode] a recovered deferred context write (${m.id}) has seq ${seq}, not a positive ` + + `whole number up to Number.MAX_SAFE_INTEGER; it is queued after the others`, ); } return { ...m, seq: ++lastSeq }; @@ -9039,17 +9062,18 @@ export class AgentFramework { // Stored in queue order, cut at the first that can't be stored: it // and what follows wait for the turn's end, and only what was // stored is injected (the round's results go on either way). A - // write that threw after its message was stored counts as stored, - // so that copy is injected too (writeDrainedInOrder). + // write that threw after storing counts as stored, and what landed + // of it is what's injected, so the request carries what the store + // holds (writeDrainedInOrder). const stored = this.writeDrainedInOrder(deferred, `tool boundary for ${agent.name}`, (msg) => agent.getContextManager().addMessage(msg.participant, msg.content, withDeferredWriteId(msg.metadata, msg.id))); - for (const [msg, id] of stored) { + for (const [msg, { id, content }] of stored) { // Every other resident reads what this boundary stores; for its // own agent, the boundary weighs what it injects below. this.noteForOtherResidents(agent, msg.metadata, id); const injection = { participant: msg.participant, - content: msg.content, + content, ...(msg.metadata ? { metadata: msg.metadata } : {}), }; midTurnInjections.push(injection); @@ -11846,7 +11870,7 @@ export class AgentFramework { // starting and for every other resident's open turn. const stored = this.writeDrainedInOrder(this.drainDeferredFor(agent.name), `turn-start flush for ${agent.name}`, (msg) => agent.getContextManager().addMessage(msg.participant, msg.content, withDeferredWriteId(msg.metadata, msg.id))); - for (const [msg, id] of stored) { + for (const [msg, { id }] of stored) { this.emitTrace({ type: 'message:added', messageId: id, source: 'deferred-flush:turn-start' }); this.noteArrival(agent.name, msg.metadata, id); this.noteForOtherResidents(agent, msg.metadata, id); @@ -13902,10 +13926,12 @@ export class AgentFramework { } // A conversation fork whose TTL closure turn just finished is done for - // good — dispose it so the agent map doesn't grow monotonically. After - // the flush above, so what was held for the turn's end lands in the - // fork's store instead of being dropped as addressed to no agent. - if (ownsPhysicalStream && this.closingConversationAgents.has(agent.name)) { + // good — dispose it so the agent map doesn't grow monotonically. The + // frame that ends the turn disposes it, after the flush above, so what + // was held for the turn's end lands in the fork's store instead of + // being dropped as addressed to no agent. A frame that hands the turn + // to a successor (a guard or budget restart) leaves the fork to it. + if (frameReachedTerminal && ownsPhysicalStream && this.closingConversationAgents.has(agent.name)) { this.disposeConversationAgent(agent.name); } } @@ -15606,12 +15632,7 @@ export class AgentFramework { bypassDeferralFor?: string; } ): MessageId { - // Route to the named agent, else the primary (not ephemeral subagents). - const agent = opts?.forAgent - ? this.agents.get(opts.forAgent) - : this.primaryAgentName - ? this.agents.get(this.primaryAgentName) - : this.agents.values().next().value; + const agent = this.deliveryAgent(opts?.forAgent); if (!agent) { if (opts?.forAgent) { console.error( @@ -15688,13 +15709,29 @@ export class AgentFramework { opts?.fromQueue ? withDeferredWriteId(metadata, opts.fromQueue.id) : metadata, ); if (opts?.placement) { opts.placement.agent = agent.name; opts.placement.messageId = stored; } - // Stored while none of the agent's turns is alive, or assembled for the - // start now beginning: no tool boundary will weigh it. - if (!this.activeTurnTokens.has(agent.name) || opts?.bypassDeferralFor === agent.name) { + this.noteStored(agent, metadata, stored, opts?.bypassDeferralFor); + return stored; + } + + /** Where addMessage delivers: the named agent, else the primary, else the + * first registered agent. Undefined when there is no such agent. */ + private deliveryAgent(forAgent?: string): Agent | undefined { + return forAgent + ? this.agents.get(forAgent) + : this.primaryAgentName + ? this.agents.get(this.primaryAgentName) + : this.agents.values().next().value; + } + + /** The notes addMessage takes for a message it stored: an arrival for the + * agent when none of its turns is alive or the message was assembled for + * the start now beginning (no tool boundary will weigh it), and one for + * every other resident's open turn. */ + private noteStored(agent: Agent, metadata: MessageMetadata | undefined, stored: MessageId, bypassDeferralFor?: string): void { + if (!this.activeTurnTokens.has(agent.name) || bypassDeferralFor === agent.name) { this.noteArrival(agent.name, metadata, stored); } this.noteForOtherResidents(agent, metadata, stored); - return stored; } /** @@ -15766,13 +15803,16 @@ export class AgentFramework { * * A write can throw after its message is stored: a store listener, the * strategy's handling of the add, or a later shard of a sharded body can - * throw once the record is written. Before an entry that threw is put - * back, its target's store is read from where it stood before the write, - * for the entry's `deferredWriteId`. A copy found there counts as written, - * under its stored id, so the entry isn't requeued and stored a second - * time. (Written through readdDrained, such a copy misses the notes - * addMessage takes after a store, as any write that throws after storing - * does. The tool boundary and the turn-start flush note what is returned.) + * throw once a record is written. Before an entry that threw is put back, + * its target's store is read from where it stood before the write, for + * messages carrying the entry's `deferredWriteId`. What it finds counts as + * written, at most once: the entry isn't requeued and stored a second + * time, and what landed is what it returns as the entry's content. For a + * sharded body, that can be its first shards alone, since the append loop + * stops at the throw; the rest of the body isn't stored. (Requeued, the + * landed shards would be stored twice, and only the rest could not be + * appended later without splitting the body.) Boot's scan for landed + * writes counts a landed shard the same way. * * Returns what was written (stored, or deferred again by addMessage, whose * id is then empty), in queue order: for one target, a prefix of its @@ -15782,74 +15822,72 @@ export class AgentFramework { entries: DeferredWrite[], where: string, write: (msg: DeferredWrite) => MessageId, - ): Map { + ): Map { const ordered = [...entries].sort(bySeq); - const targetOf = (msg: DeferredWrite): string | null => msg.forAgent ?? this.primaryAgentName; + // An entry's target as addMessage resolves it, keyed by name for the cut. + const targetOf = (msg: DeferredWrite): string | null => msg.forAgent ?? this.deliveryAgent()?.name ?? null; const cut = new Set(); - const written = new Map(); + const written = new Map(); for (let i = 0; i < ordered.length; i++) { const msg = ordered[i]!; const target = targetOf(msg); if (cut.has(target)) continue; - const before = this.storedCountOf(target); + const agent = this.deliveryAgent(msg.forAgent); + const before = agent ? this.storedCountOf(agent) : undefined; try { - written.set(msg, write(msg)); + written.set(msg, { id: write(msg), content: msg.content }); } catch (error) { - const copy = before === undefined ? undefined : this.storedCopyOf(target!, before, msg.id); - if (copy !== undefined) { - written.set(msg, copy); - this.logDeferredWriteError(msg, where, error, true); + const landed = agent && before !== undefined ? this.storedCopiesOf(agent, before, msg.id) : []; + if (landed.length > 0) { + written.set(msg, { id: landed[0]!.id, content: landed.flatMap((m) => m.content), afterThrow: true }); + this.logDeferredWriteError(msg, where, error, { count: landed.length, sharded: landed[0]!.bodyGroupId !== undefined }); continue; } cut.add(target); this.requeueDrained(ordered.slice(i).filter((m) => targetOf(m) === target)); - this.logDeferredWriteError(msg, where, error, false); + this.logDeferredWriteError(msg, where, error); } } return written; } - /** How many messages the agent's store holds: undefined when no such agent - * is registered, or its store can't say. */ - private storedCountOf(agentName: string | null): number | undefined { - const agent = agentName ? this.agents.get(agentName) : undefined; - if (!agent) return undefined; + /** How many messages the agent's store holds: undefined when its store + * can't say. */ + private storedCountOf(agent: Agent): number | undefined { try { - const cm = agent.getContextManager() as unknown as { getMessageCount?: () => number; getAllMessages: () => unknown[] }; - return typeof cm.getMessageCount === 'function' ? cm.getMessageCount() : cm.getAllMessages().length; + return agent.getContextManager().getMessageCount(); } catch { return undefined; } } - /** The id of the message carrying `deferredWriteId` among those the agent's - * store gained from position `from` on, read without blobs, if there is - * one. For a sharded body, its first shard. */ - private storedCopyOf(agentName: string, from: number, deferredWriteId: string): MessageId | undefined { - const agent = this.agents.get(agentName); - if (!agent) return undefined; + /** The messages carrying `deferredWriteId` among those the agent's store + * gained from position `from` on, in store order, with their blobs + * resolved, as a request carries them. A sharded body's carry its + * `bodyGroupId`. */ + private storedCopiesOf(agent: Agent, from: number, deferredWriteId: string): Array<{ id: MessageId; content: ContentBlock[]; bodyGroupId?: string }> { try { - const cm = agent.getContextManager() as unknown as { - getMessageCount?: () => number; - getMessageWindow?: (o: number, l: number, opts?: { resolveBlobs?: boolean }) => - { messages: Array<{ id: MessageId; metadata?: Record }> }; - getAllMessages: () => Array<{ id: MessageId; metadata?: Record }>; - }; - const gained = typeof cm.getMessageCount === 'function' && typeof cm.getMessageWindow === 'function' - ? cm.getMessageWindow(from, Math.max(0, cm.getMessageCount() - from), { resolveBlobs: false }).messages - : cm.getAllMessages().slice(from); - return gained.find((m) => m.metadata?.deferredWriteId === deferredWriteId)?.id; + const cm = agent.getContextManager(); + return cm.getMessageWindow(from, Math.max(0, cm.getMessageCount() - from)).messages + .filter((m) => m.metadata?.deferredWriteId === deferredWriteId) + .map((m) => ({ id: m.id, content: m.content, ...(m.bodyGroupId !== undefined ? { bodyGroupId: m.bodyGroupId } : {}) })); } catch (error) { - console.error(`[deferral] could not read ${agentName}'s store for a deferred write that threw:`, error); - return undefined; + console.error(`[deferral] could not read ${agent.name}'s store for a deferred write that threw:`, error); + return []; } } /** The line naming a drained write that threw: one that couldn't be stored, - * which is already back in the queue, or one whose message was stored - * before the throw. Logging mustn't fail the flush that met it, so a thrown - * value that has no text is named as such. */ - private logDeferredWriteError(msg: DeferredWrite, where: string, error: unknown, stored: boolean): void { + * which is already back in the queue, or, with `landed`, one stored before + * the throw (for a sharded body, how many of its shards). Logging mustn't + * fail the flush that met it, so a thrown value that has no text is named + * as such. */ + private logDeferredWriteError( + msg: DeferredWrite, + where: string, + error: unknown, + landed?: { count: number; sharded: boolean }, + ): void { let reason: string; try { reason = error instanceof Error ? error.message : String(error); @@ -15859,8 +15897,10 @@ export class AgentFramework { console.error( `[deferral] ${where}: deferred write ${describeDeferredWrite(msg)} for ` + `${msg.forAgent ?? this.primaryAgentName ?? 'the primary'} ` + - (stored - ? `threw after it was stored; it counts as written and isn't stored again: ` + (landed + ? `threw after ${landed.sharded + ? `${landed.count} shard${landed.count === 1 ? '' : 's'} of its body ${landed.count === 1 ? 'was' : 'were'} stored` + : 'it was stored'}; what was stored counts as written and isn't stored again: ` : `could not be stored; it stays queued, holding what is queued behind it for that agent: `) + reason, ); @@ -15868,16 +15908,23 @@ export class AgentFramework { /** Re-add drained entries through addMessage, which stores each or defers * it again, in queue order and cut per target (writeDrainedInOrder). With - * `bypassDeferral`, each is stored past its target's deferral (stop()). */ + * `bypassDeferral`, each is stored past its target's deferral (stop()). A + * write that threw after storing missed the notes addMessage takes after a + * store, so they're taken here for what landed. */ private readdDrained(entries: DeferredWrite[], where: string, opts?: { bypassDeferral?: boolean }): DeferredWrite[] { - return [...this.writeDrainedInOrder(entries, where, (msg) => { - const target = msg.forAgent ?? this.primaryAgentName; - return this.addMessage(msg.participant, msg.content, msg.metadata, { + const bypassFor = (msg: DeferredWrite): string | undefined => + opts?.bypassDeferral ? this.deliveryAgent(msg.forAgent)?.name : undefined; + const written = this.writeDrainedInOrder(entries, where, (msg) => + this.addMessage(msg.participant, msg.content, msg.metadata, { fromQueue: { id: msg.id, seq: msg.seq }, ...(msg.forAgent ? { forAgent: msg.forAgent } : {}), - ...(opts?.bypassDeferral && target ? { bypassDeferralFor: target } : {}), - }); - }).keys()]; + ...(bypassFor(msg) ? { bypassDeferralFor: bypassFor(msg) } : {}), + })); + for (const [msg, w] of written) { + const agent = w.afterThrow ? this.deliveryAgent(msg.forAgent) : undefined; + if (agent) this.noteStored(agent, msg.metadata, w.id, bypassFor(msg)); + } + return [...written.keys()]; } private editMessage(id: MessageId, content: ContentBlock[]): void { diff --git a/test/conversation-routing.test.ts b/test/conversation-routing.test.ts index 6622b54b6..50db20df3 100644 --- a/test/conversation-routing.test.ts +++ b/test/conversation-routing.test.ts @@ -10,7 +10,7 @@ import { tmpdir } from 'node:os'; import { mkdtempSync, rmSync } from 'node:fs'; import { join } from 'node:path'; import { AgentFramework } from '../src/index.js'; -import type { ProcessEvent, ConversationRouterConfig, TraceEvent } from '../src/index.js'; +import type { ProcessEvent, ConversationRouterConfig, TraceEvent, Module, ModuleContext, ToolCall, ToolResult } from '../src/index.js'; import { Membrane, OpenAIResponsesAPIAdapter, OpenAIResponsesFormatter } from '@animalabs/membrane'; import type { ContentBlock } from '@animalabs/membrane'; import { MockMembrane, createMockResponse } from './helpers/mock-membrane.js'; @@ -664,6 +664,73 @@ describe('Conversation routing', () => { await framework.stop(); }); + it('a closure turn that restarts is disposed by the frame that ends it, after storing what was held for it', async () => { + const IDLE_TTL_MS = 60_000; + // One tool, for the closure turn's tool round. + const tools: Module = { + name: 'test', + async start(_ctx: ModuleContext) {}, + async stop() {}, + getTools: () => [{ name: 'echo', description: 'Echo input', inputSchema: { type: 'object', properties: {} } }], + async handleToolCall(_call: ToolCall): Promise { return { success: true, data: 'echoed' }; }, + async onProcess() { return {}; }, + } as unknown as Module; + const framework = await AgentFramework.create({ + storePath: join(tempDir, 'test.chronicle'), + membrane: membrane.asMembrane(), + // The mock's tool round reports 10 input tokens, over this budget, + // which the fork inherits from its template: the closure turn's tool + // round restarts its stream. + agents: [{ name: 'trunk', model: 'test-model', systemPrompt: 'You are the trunk.', maxStreamTokens: 5 }], + modules: [tools], + conversations: { templateAgent: 'trunk', idleTtlMs: IDLE_TTL_MS }, + }); + membrane.pushResponse(createMockResponse([{ type: 'text', text: 'hello!' }])); + framework.pushEvent(incomingEvent({ channelId: 'slack:D6', text: 'hi', channelType: 'im' })); + await framework.runUntilIdle(); + const g1 = framework.getAgent('conversation-slack-D6-g1')!; + assert.ok(g1, 'g1 fork should exist after spawn'); + + forceExpiry(framework, 'slack:D6', IDLE_TTL_MS); + membrane.pushResponse(createMockResponse([ + { type: 'text', text: 'Checking.' }, + { type: 'tool_use', id: 'e1', name: 'test--echo', input: {} }, + ] as ContentBlock[], 'tool_use')); + const internals = framework as unknown as { + addMessage(participant: string, content: ContentBlock[], metadata?: undefined, opts?: { forAgent?: string }): string; + }; + const streamYielding = membrane.streamYielding.bind(membrane); + let streams = 0; + let deferred: string | undefined; + membrane.streamYielding = (request, options) => { + const first = streams++ === 0; + if (first) { + // While the closure turn is alive, a write addressed to the fork is + // deferred to that turn's end. + deferred = internals.addMessage('alice', [{ type: 'text', text: 'one more thing' }], undefined, + { forAgent: 'conversation-slack-D6-g1' }); + } + const stream = streamYielding(request, options); + // A mock stream takes every response queued when it starts, so the + // restart's stream gets its reply only once the first has started. + if (first) membrane.pushResponse(createMockResponse([{ type: 'text', text: 'Final report posted.' }])); + return stream; + }; + framework.pushEvent({ type: 'external-message', source: 'test', content: 'tick', metadata: {} } as unknown as ProcessEvent); + await framework.runUntilIdle(); + + assert.equal(deferred, '', 'deferred: the closure turn was alive'); + assert.equal(streams, 2, 'the closure turn restarted once'); + assert.equal(framework.getAgent('conversation-slack-D6-g1'), null, 'disposed once its closure turn finished'); + const stored = g1.getContextManager().getAllMessages().map((m) => JSON.stringify(m.content)); + // The restart's turn start stores what was held for the turn, before its + // compile, so the restarted stream carries it and its reply follows. + assert.ok(stored[stored.length - 2]!.includes('one more thing'), stored.join('\n')); + assert.ok(stored[stored.length - 1]!.includes('Final report posted.'), 'the restarted closure turn finished in the fork'); + + await framework.stop(); + }); + it('restart does not reuse generation names or re-seed an existing namespace', async () => { membrane.pushResponse(createMockResponse([{ type: 'text', text: 'noted' }])); const fw1 = await makeFramework({}, 'restart.chronicle'); diff --git a/test/deferred-window-only.test.ts b/test/deferred-window-only.test.ts index 062c6bf97..caf5348e6 100644 --- a/test/deferred-window-only.test.ts +++ b/test/deferred-window-only.test.ts @@ -505,11 +505,85 @@ describe('mid-turn deliveries keep the window what the stream carried', () => { 'Antra: look right', 'assistant: Done.', ], 'stored once, where the stream carried it'); - assert.ok(errors.some((e) => e.includes("threw after it was stored; it counts as written and isn't stored again") + assert.ok(errors.some((e) => e.includes("threw after it was stored; what was stored counts as written and isn't stored again") && e.includes('listener failed')), errors.join('\n')); await framework.stop(); }); + /** Shard a message whose first block is `marker` into one shard per block, + * as a strategy's ingress chunking does. */ + function shardOn(framework: AgentFramework, marker: string): void { + const cm = framework.getAgent('assistant')!.getContextManager() as unknown as { strategy: Record }; + cm.strategy.chunkIngressMessage = (_participant: string, content: ContentBlock[]) => + (content[0] as { text?: string } | undefined)?.text === marker + ? { bodyGroupId: `body-${marker}`, shards: content.map((block, shardIndex) => ({ content: [block], shardIndex })) } + : null; + } + + it("at a tool boundary, a sharded body that throws after its first shard: what landed is carried and counts as written, and the rest isn't stored", async () => { + membrane.pushResponse(createMockResponse([ + { type: 'text', text: 'Moving.' }, + { type: 'tool_use', id: 'c1', name: 'robot--move', input: {} }, + ] as ContentBlock[], 'tool_use')); + membrane.pushResponse(createMockResponse(text('Done.'))); + const framework = await createFramework(); + shardOn(framework, 'part one'); + // The listener throws on the first shard's add, which stops the append + // loop before the second shard. + throwOnceAfterStoring(framework, 'part one'); + module.interjections = [ + { participant: 'Antra', content: [...text('part one'), ...text('part two')] }, + { participant: 'Antra', content: text('look right') }, + ]; + const errors: string[] = []; + const error = console.error; + console.error = (...args: unknown[]) => { errors.push(args.map(String).join(' ')); }; + try { + framework.pushEvent({ type: 'external-message', source: 'test', content: 'go' } as unknown as ProcessEvent); + await framework.runUntilIdle(); + } finally { console.error = error; } + assert.deepEqual(injected(), ['Antra: part one', 'Antra: look right'], 'the request carries what the store holds'); + assert.deepEqual(windowOf(framework).slice(2), [ + 'user: ', + 'Antra: part one', + 'Antra: look right', + 'assistant: Done.', + ], 'the first shard alone, stored once'); + assert.equal(internalsOf(framework).deferredMessages.length, 0, 'nothing went back into the queue'); + assert.ok(errors.some((e) => e.includes("threw after 1 shard of its body was stored; what was stored counts as written and isn't stored again") + && e.includes('listener failed')), errors.join('\n')); + await framework.stop(); + }); + + it("a write that throws after it's stored, re-added from the queue, is weighed for its agent's open turn as a stored write is", async () => { + const framework = await createFramework(); + throwOnceAfterStoring(framework, 'from beta'); + const internals = framework as unknown as { + openLogical: Map; + readdDrained(entries: Array<{ id: string; seq: number; participant: string; content: ContentBlock[]; metadata?: Record }>, where: string): unknown[]; + }; + // A logical turn still open between its streams, engaged with chan-beta: + // no turn is alive, so no tool boundary weighs what is stored now, and + // addMessage notes it as one of the turn's arrivals. + const open = { engaged: new Set(['chan-beta']), arrivals: [] as Array<{ messageId: string; channelId: string }> }; + internals.openLogical.set('assistant', open); + const error = console.error; + console.error = () => {}; + let written: unknown[]; + try { + written = internals.readdDrained( + [{ id: 'w1', seq: 1, participant: 'Antra', content: text('from beta'), metadata: { channelId: 'chan-beta' } }], + 'test flush', + ); + } finally { console.error = error; } + assert.equal(written.length, 1, 'it counts as written'); + const copies = framework.getAgent('assistant')!.getContextManager().getAllMessages() + .filter((m) => (m.metadata as { deferredWriteId?: string } | undefined)?.deferredWriteId === 'w1'); + assert.equal(copies.length, 1, 'stored once'); + assert.deepEqual(open.arrivals, [{ messageId: copies[0]!.id, channelId: 'chan-beta' }], 'and noted as one of the open turn\'s arrivals'); + await framework.stop(); + }); + it('at the turn end, a write that throws after its message is stored is not queued again', async () => { membrane.pushResponse(createMockResponse([ { type: 'text', text: 'Moving.' }, @@ -523,13 +597,17 @@ describe('mid-turn deliveries keep the window what the stream carried', () => { { participant: 'assistant', content: text('my own words') }, { participant: 'Antra', content: text('look left') }, ]; + const errors: string[] = []; const error = console.error; - console.error = () => {}; + console.error = (...args: unknown[]) => { errors.push(args.map(String).join(' ')); }; try { framework.pushEvent({ type: 'external-message', source: 'test', content: 'go' } as unknown as ProcessEvent); await framework.runUntilIdle(); } finally { console.error = error; } assert.equal(internalsOf(framework).deferredMessages.length, 0, 'nothing went back into the queue'); + assert.ok(errors.some((e) => e.includes("turn-end flush: deferred write") + && e.includes("threw after it was stored; what was stored counts as written and isn't stored again") + && e.includes('listener failed')), errors.join('\n')); await framework.stop(); // stop() lands whatever is still queued const reopened = await createFramework(); diff --git a/test/host-quiesce.test.ts b/test/host-quiesce.test.ts index fd88a4e9c..74951cd3e 100644 --- a/test/host-quiesce.test.ts +++ b/test/host-quiesce.test.ts @@ -1001,19 +1001,23 @@ test('boot recovery keeps one entry per id, and numbers a seq that isn\'t a posi await framework.stop(); // What the framework never writes, as damage or a hand edit could leave - // it: an id that comes twice, a seq of 1e400 (JSON reads it as Infinity) - // and a negative one. The entries with a bad seq come before 'second' in - // the file. Nothing here is malformed, so only these make boot rewrite it. + // it: an id that comes twice, an entry with no id, a seq of 1e400 (JSON + // reads it as Infinity), a negative one, zero and a string. The entries + // with a bad seq come before 'second' in the file. Nothing here is + // malformed, so only these make boot rewrite it. const file = join(storePath, 'recovery', 'deferred-writes.json'); const doc = JSON.parse(readFileSync(file, 'utf8')) as { pending: Array> }; assert.equal(doc.pending.length, 2); - const entry = (id: string, seq: unknown, text: string) => - ({ id, seq, participant: 'user', content: [{ type: 'text', text }] }); + const entry = (id: string | undefined, seq: unknown, text: string) => + ({ ...(id === undefined ? {} : { id }), seq, participant: 'user', content: [{ type: 'text', text }] }); doc.pending.splice(1, 0, entry('infinite-seq', '__INFINITY__', 'infinite seq'), entry('negative-seq', -3, 'negative seq'), entry('twice', 50, 'first with its id'), entry('twice', 51, 'second with its id'), + entry('zero-seq', 0, 'zero seq'), + entry('string-seq', 'seven', 'string seq'), + entry(undefined, 52, 'no id'), ); writeFileSync(file, JSON.stringify(doc).replace('"__INFINITY__"', '1e400')); @@ -1021,19 +1025,69 @@ test('boot recovery keeps one entry per id, and numbers a seq that isn\'t a posi framework = await AgentFramework.create(config()); console.error = error; const named = (text: string) => assert.ok(lines.some((line) => line.includes(text)), `${text}:\n${lines.join('\n')}`); + const late = 'not a positive whole number up to Number.MAX_SAFE_INTEGER; it is queued after the others'; named('repeats id twice (user) and was dropped; the first entry with that id is kept'); - named('(infinite-seq) has seq Infinity, not a positive whole number; it is queued after the others'); - named('(negative-seq) has seq -3, not a positive whole number; it is queued after the others'); - assert.equal(framework.getHostModeStatus().deferredWrites, 5); - - const rewritten = JSON.parse(readFileSync(file, 'utf8')) as { pending: Array<{ id: string; seq: unknown }> }; - assert.equal(rewritten.pending.length, 5, 'the recovery file is rewritten without the repeated id'); + named(`(infinite-seq) has seq Infinity, ${late}`); + named(`(negative-seq) has seq -3, ${late}`); + named(`(zero-seq) has seq 0, ${late}`); + named(`(string-seq) has seq of type string, ${late}`); + named('a recovered deferred context write (user) has no id; it is given '); + assert.equal(framework.getHostModeStatus().deferredWrites, 8); + + const rewritten = JSON.parse(readFileSync(file, 'utf8')) as { pending: Array<{ id: unknown; seq: unknown }> }; + assert.equal(rewritten.pending.length, 8, 'the recovery file is rewritten without the repeated id'); assert.ok(rewritten.pending.every((m) => Number.isSafeInteger(m.seq) && (m.seq as number) > 0), JSON.stringify(rewritten.pending)); await framework.resume(); const texts = framework.getAgent('agent')!.getContextManager().getAllMessages().slice(before) .map((m) => (m.content[0] as { text: string }).text); - assert.deepEqual(texts, ['first', 'second', 'first with its id', 'infinite seq', 'negative seq']); + assert.deepEqual(texts, ['first', 'second', 'first with its id', 'no id', 'infinite seq', 'negative seq', 'zero seq', 'string seq']); + } finally { + console.error = error; + await framework.stop(); + rmSync(dir, { recursive: true, force: true }); + } +}); + +test('boot recovery restores a recovery file that predates seq in file order, and rewrites it with seqs', { timeout: 60_000 }, async () => { + const dir = mkdtempSync(join(tmpdir(), 'host-quiesce-noseq-')); + const storePath = join(dir, 'store'); + const config = () => ({ + storePath, + membrane: new MockMembrane().asMembrane(), + agents: [{ name: 'agent', model: 'test-model', systemPrompt: 'test' }], + modules: [], + }); + let framework = await AgentFramework.create(config()); + const lines: string[] = []; + const error = console.error; + try { + await framework.quiesce({ reason: 'surgery' }); + const before = framework.getAgent('agent')!.getContextManager().getAllMessages().length; + const add = (framework as unknown as { addMessage(p: string, c: unknown[]): unknown }).addMessage.bind(framework); + for (const text of ['first', 'second', 'third']) add('user', [{ type: 'text', text }]); + await framework.stop(); + + // A file written before entries carried a seq is in write order. Here + // that order is third, first, second, so only the file can give it. + const file = join(storePath, 'recovery', 'deferred-writes.json'); + const doc = JSON.parse(readFileSync(file, 'utf8')) as { pending: Array> }; + assert.equal(doc.pending.length, 3); + const [first, second, third] = doc.pending.map(({ seq: _seq, ...rest }) => rest); + writeFileSync(file, JSON.stringify({ version: 1, pending: [third, first, second] })); + + console.error = (...args: unknown[]) => { lines.push(args.map(String).join(' ')); }; + framework = await AgentFramework.create(config()); + console.error = error; + assert.ok(!lines.some((line) => line.includes('has seq')), `an entry with no seq isn't named:\n${lines.join('\n')}`); + const rewritten = JSON.parse(readFileSync(file, 'utf8')) as { pending: Array<{ seq: unknown; content: Array<{ text: string }> }> }; + assert.deepEqual(rewritten.pending.map((m) => [m.content[0]!.text, m.seq]), [['third', 1], ['first', 2], ['second', 3]], + 'rewritten with seqs, in file order'); + + await framework.resume(); + const texts = framework.getAgent('agent')!.getContextManager().getAllMessages().slice(before) + .map((m) => (m.content[0] as { text: string }).text); + assert.deepEqual(texts, ['third', 'first', 'second']); } finally { console.error = error; await framework.stop(); From 127e13b9024caa1bb585af70cf2c6fe20d4255ae Mon Sep 17 00:00:00 2001 From: Duncan-2410 Date: Thu, 15 Oct 2026 22:44:51 -0400 Subject: [PATCH 29/34] fix(deferral): shards that landed before a throw go as the compile joins them A tool boundary injects what landed of a write that threw after storing (eadf9a9). Two or more landed shards of a sharded body were injected one block each, while context-manager's autobiographical strategy, the one that shards on ingest, joins a body's consecutive shards when it compiles: their non-text blocks, then one text block of their joined text. So the request and every later compile disagreed in shape. The boundary now injects them as that join renders them. context-manager doesn't export the join (the strategy's mergeAdjacentBodyGroupRaw), so the rule is copied, and a test compares the boundary's injection with the next turn's request under that strategy, throwing at the second and the third shard. Found by Ansel-2428's read and probe. Co-Authored-By: Claude Opus 5.5 --- .../mid-turn-window-only-held.fixed.md | 5 +- src/framework.ts | 27 +++++++-- test/deferred-window-only.test.ts | 56 +++++++++++++++++++ 3 files changed, 80 insertions(+), 8 deletions(-) diff --git a/changelog.d/mid-turn-window-only-held.fixed.md b/changelog.d/mid-turn-window-only-held.fixed.md index 5c558d56c..af83c14d6 100644 --- a/changelog.d/mid-turn-window-only-held.fixed.md +++ b/changelog.d/mid-turn-window-only-held.fixed.md @@ -108,8 +108,9 @@ queue and stored a second time, and at a tool boundary what was stored is injected with the rest. For a body the strategy shards, that can be its first shards alone, since the throw stops the store's append loop: the - rest of the body isn't stored, and the request carries what the store - holds. The log line says it was stored, or how many shards were. + rest of the body isn't stored, and the request carries what landed, joined + as the strategy joins a body's shards when it compiles. The log line says + it was stored, or how many shards were. - Boot recovery drops a recovered deferred write whose participant isn't text, whose content isn't an array, or that holds a block that isn't an object, and logs it by id and participant, rather than holding it in the diff --git a/src/framework.ts b/src/framework.ts index e6ff01e98..836fdfa61 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -493,14 +493,28 @@ const bySeq = (a: { seq: number }, b: { seq: number }): number => a.seq - b.seq; * (empty when addMessage deferred it again; a sharded body's first shard) * and what the store holds of it. `afterThrow` marks an entry whose write * threw after storing it, found in the store and counted as written: its - * content is what landed, which for a sharded body can be its first shards - * alone. */ + * content is what landed, as later compiles render it, which for a sharded + * body can be its first shards alone. */ interface DrainedWrite { id: MessageId; content: ContentBlock[]; afterThrow?: true; } +/** What later compiles render from the store for the copies of one write + * that landed: one copy as it is, and two or more, the shards of a sharded + * body, as context-manager's autobiographical strategy (the one that + * shards on ingest) joins a body's consecutive shards: their non-text + * blocks, then one text block of their joined text. The join is that + * strategy's `mergeAdjacentBodyGroupRaw`, which context-manager doesn't + * export, so the rule is copied here, and a test compares the two. */ +function asCompiled(landed: ReadonlyArray<{ content: ContentBlock[] }>): ContentBlock[] { + if (landed.length === 1) return landed[0]!.content; + const blocks = landed.flatMap((m) => m.content); + const text = blocks.filter((b) => b.type === 'text').map((b) => (b as { text: string }).text).join(''); + return [...blocks.filter((b) => b.type !== 'text'), { type: 'text', text } as ContentBlock]; +} + /** Stamp a deferred write's durable id into the metadata it is stored with * (boot recovery dedups replays by it). Idempotent for an already-stamped * message. */ @@ -15807,9 +15821,10 @@ export class AgentFramework { * its target's store is read from where it stood before the write, for * messages carrying the entry's `deferredWriteId`. What it finds counts as * written, at most once: the entry isn't requeued and stored a second - * time, and what landed is what it returns as the entry's content. For a - * sharded body, that can be its first shards alone, since the append loop - * stops at the throw; the rest of the body isn't stored. (Requeued, the + * time, and what landed, as later compiles render it (asCompiled), is what + * it returns as the entry's content. For a sharded body, that can be its + * first shards alone, since the append loop stops at the throw; the rest + * of the body isn't stored. (Requeued, the * landed shards would be stored twice, and only the rest could not be * appended later without splitting the body.) Boot's scan for landed * writes counts a landed shard the same way. @@ -15839,7 +15854,7 @@ export class AgentFramework { } catch (error) { const landed = agent && before !== undefined ? this.storedCopiesOf(agent, before, msg.id) : []; if (landed.length > 0) { - written.set(msg, { id: landed[0]!.id, content: landed.flatMap((m) => m.content), afterThrow: true }); + written.set(msg, { id: landed[0]!.id, content: asCompiled(landed), afterThrow: true }); this.logDeferredWriteError(msg, where, error, { count: landed.length, sharded: landed[0]!.bodyGroupId !== undefined }); continue; } diff --git a/test/deferred-window-only.test.ts b/test/deferred-window-only.test.ts index caf5348e6..dd01c8d48 100644 --- a/test/deferred-window-only.test.ts +++ b/test/deferred-window-only.test.ts @@ -35,6 +35,7 @@ import type { ToolResult, } from '../src/index.js'; import { AgentFramework } from '../src/index.js'; +import { AutobiographicalStrategy } from '@animalabs/context-manager'; import { MockMembrane, createMockResponse } from './helpers/mock-membrane.js'; import type { ContentBlock } from '@animalabs/membrane'; @@ -555,6 +556,61 @@ describe('mid-turn deliveries keep the window what the stream carried', () => { await framework.stop(); }); + it('at a tool boundary, the shards that landed before a throw are carried as the next compile renders them', async () => { + // Probe by Ansel-2428: the autobiographical strategy shards a long + // message on ingest and joins a body's consecutive shards when it + // compiles, so the boundary must carry what landed in that shape. + const long = Array.from({ length: 60 }, (_, i) => `Sentence ${i} of a long message from Antra.`).join(' '); + const ofLong = (messages: Array<{ content: unknown }>) => + messages.filter((m) => JSON.stringify(m.content).includes('Sentence 0 ')); + for (const throwAt of [2, 3]) { + membrane = new MockMembrane(); + membrane.pushResponse(createMockResponse([ + { type: 'text', text: 'Moving.' }, + { type: 'tool_use', id: 'c1', name: 'robot--move', input: {} }, + ] as ContentBlock[], 'tool_use')); + membrane.pushResponse(createMockResponse(text('Done.'))); + const framework = await createFramework({ + storePath: join(tempDir, `shards-${throwAt}.chronicle`), + membrane: membrane.asMembrane(), + agents: [{ + name: 'assistant', model: 'test-model', systemPrompt: 'You are a robot pilot.', + strategy: new AutobiographicalStrategy({ + adaptiveResolution: true, foldingStrategy: 'kv-stable', recentWindowTokens: 30_000, kvStableReachTokens: 8_000, + targetChunkTokens: 50, + }), + } as never], + }); + const cm = framework.getAgent('assistant')!.getContextManager(); + let shards = 0; + cm.onMessage((event) => { + if (event.type !== 'add' || !(event.message as { bodyGroupId?: string }).bodyGroupId) return; + if (++shards === throwAt) throw new Error('listener failed'); + }); + module.interjections = [{ participant: 'Antra', content: text(long) }]; + const error = console.error; + console.error = () => {}; + try { + framework.pushEvent({ type: 'external-message', source: 'test', content: 'go' } as unknown as ProcessEvent); + await framework.runUntilIdle(); + const landed = cm.getAllMessages().filter((m) => (m as { bodyGroupId?: string }).bodyGroupId); + assert.equal(landed.length, throwAt, `${throwAt} shards landed before the throw stopped the body`); + const injected = ofLong(membrane.lastStream!.receivedToolResultOptions[0]?.injectedMessages ?? []); + // The next turn's request is compiled from the store. + membrane.pushResponse(createMockResponse(text('Okay.'))); + framework.pushEvent({ type: 'external-message', source: 'test', content: 'go' } as unknown as ProcessEvent); + await framework.runUntilIdle(); + const next = ofLong(membrane.calls.at(-1)!.messages); + assert.equal(injected.length, 1, `throw at shard ${throwAt}: one message injected`); + assert.equal(next.length, 1, `throw at shard ${throwAt}: one message in the next request`); + assert.deepEqual(injected[0]!.content, next[0]!.content, `throw at shard ${throwAt}: the boundary carried what the next compile renders`); + } finally { + console.error = error; + await framework.stop(); + } + } + }); + it("a write that throws after it's stored, re-added from the queue, is weighed for its agent's open turn as a stored write is", async () => { const framework = await createFramework(); throwOnceAfterStoring(framework, 'from beta'); From e813e15587a9dc978f348fbc5d09d104f5d9421c Mon Sep 17 00:00:00 2001 From: Duncan-2410 Date: Thu, 15 Oct 2026 23:05:57 -0400 Subject: [PATCH 30/34] test(deferral): the shard-join test covers an image between two texts The test of what a tool boundary injects for shards that landed before a throw (127e13b) had only text bodies, so a change to where context-manager's join puts a body's non-text blocks changed neither side and couldn't fail it. A third case gives a short text, an image, then the long text, and throws at the second shard. It passes against the join as 0.14.1 has it, and fails against a join that renders the text first or keeps each block where it sat, so the copy of the join in asCompiled can't drift from it unnoticed. From Ansel-2428's read. Co-Authored-By: Claude Opus 5.5 --- test/deferred-window-only.test.ts | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/test/deferred-window-only.test.ts b/test/deferred-window-only.test.ts index dd01c8d48..a3d4673bb 100644 --- a/test/deferred-window-only.test.ts +++ b/test/deferred-window-only.test.ts @@ -559,11 +559,20 @@ describe('mid-turn deliveries keep the window what the stream carried', () => { it('at a tool boundary, the shards that landed before a throw are carried as the next compile renders them', async () => { // Probe by Ansel-2428: the autobiographical strategy shards a long // message on ingest and joins a body's consecutive shards when it - // compiles, so the boundary must carry what landed in that shape. + // compiles, so the boundary must carry what landed in that shape. The + // body with an image between two texts fails here if the join changes + // where it puts a non-text block, so the copy of the join can't drift. const long = Array.from({ length: 60 }, (_, i) => `Sentence ${i} of a long message from Antra.`).join(' '); const ofLong = (messages: Array<{ content: unknown }>) => messages.filter((m) => JSON.stringify(m.content).includes('Sentence 0 ')); - for (const throwAt of [2, 3]) { + const PNG = 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg=='; + const image = { type: 'image', source: { type: 'base64', data: PNG, mediaType: 'image/png' } } as ContentBlock; + const cases: Array<{ name: string; content: ContentBlock[]; throwAt: number }> = [ + { name: 'text, throw at shard 2', content: text(long), throwAt: 2 }, + { name: 'text, throw at shard 3', content: text(long), throwAt: 3 }, + { name: 'a short text, an image, then the long text, throw at shard 2', content: [...text('Intro. '), image, ...text(long)], throwAt: 2 }, + ]; + for (const [i, { name, content, throwAt }] of cases.entries()) { membrane = new MockMembrane(); membrane.pushResponse(createMockResponse([ { type: 'text', text: 'Moving.' }, @@ -571,7 +580,7 @@ describe('mid-turn deliveries keep the window what the stream carried', () => { ] as ContentBlock[], 'tool_use')); membrane.pushResponse(createMockResponse(text('Done.'))); const framework = await createFramework({ - storePath: join(tempDir, `shards-${throwAt}.chronicle`), + storePath: join(tempDir, `shards-${i}.chronicle`), membrane: membrane.asMembrane(), agents: [{ name: 'assistant', model: 'test-model', systemPrompt: 'You are a robot pilot.', @@ -587,23 +596,23 @@ describe('mid-turn deliveries keep the window what the stream carried', () => { if (event.type !== 'add' || !(event.message as { bodyGroupId?: string }).bodyGroupId) return; if (++shards === throwAt) throw new Error('listener failed'); }); - module.interjections = [{ participant: 'Antra', content: text(long) }]; + module.interjections = [{ participant: 'Antra', content }]; const error = console.error; console.error = () => {}; try { framework.pushEvent({ type: 'external-message', source: 'test', content: 'go' } as unknown as ProcessEvent); await framework.runUntilIdle(); const landed = cm.getAllMessages().filter((m) => (m as { bodyGroupId?: string }).bodyGroupId); - assert.equal(landed.length, throwAt, `${throwAt} shards landed before the throw stopped the body`); + assert.equal(landed.length, throwAt, `${name}: ${throwAt} shards landed before the throw stopped the body`); const injected = ofLong(membrane.lastStream!.receivedToolResultOptions[0]?.injectedMessages ?? []); // The next turn's request is compiled from the store. membrane.pushResponse(createMockResponse(text('Okay.'))); framework.pushEvent({ type: 'external-message', source: 'test', content: 'go' } as unknown as ProcessEvent); await framework.runUntilIdle(); const next = ofLong(membrane.calls.at(-1)!.messages); - assert.equal(injected.length, 1, `throw at shard ${throwAt}: one message injected`); - assert.equal(next.length, 1, `throw at shard ${throwAt}: one message in the next request`); - assert.deepEqual(injected[0]!.content, next[0]!.content, `throw at shard ${throwAt}: the boundary carried what the next compile renders`); + assert.equal(injected.length, 1, `${name}: one message injected`); + assert.equal(next.length, 1, `${name}: one message in the next request`); + assert.deepEqual(injected[0]!.content, next[0]!.content, `${name}: the boundary carried what the next compile renders`); } finally { console.error = error; await framework.stop(); From 2a25b761e6b3c1e0b40d96daa4c9c0db234b7b34 Mon Sep 17 00:00:00 2001 From: Duncan-2410 Date: Thu, 15 Oct 2026 23:36:44 -0400 Subject: [PATCH 31/34] docs(changelog): the fragment says what membrane's main declares, not what its releases will Two bullets named membrane's ToolContext.supportsInjectedMessages as true on its native path, and false on its XML prefill path, "in releases after 0.5.87". The field is on membrane's main, and no release carries it yet, so the fragment now says what membrane's main sets, rather than what membrane's later releases will hold, which is membrane's to say. Co-Authored-By: Claude Opus 5.5 --- .../mid-turn-window-only-held.fixed.md | 29 +++++++++---------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/changelog.d/mid-turn-window-only-held.fixed.md b/changelog.d/mid-turn-window-only-held.fixed.md index af83c14d6..6f897a684 100644 --- a/changelog.d/mid-turn-window-only-held.fixed.md +++ b/changelog.d/mid-turn-window-only-held.fixed.md @@ -16,21 +16,20 @@ refused. - A message injected into the live stream at a tool boundary now counts as read for RFC-006 coalescing when the stream says it carries injected - messages (membrane's `ToolContext.supportsInjectedMessages`, true on its - native path in releases after 0.5.87): the consumed watermark moves when - messages are injected, not only at a compile. Before, an injected message - sat above the watermark of the turn's compile, so a coalesced replacement - for its subject could remove a message the agent had heard and answered. - Within a branch the watermark only advances: a compile that started before - such a mark doesn't roll it back when it finishes. A stream that doesn't - say, as on membrane 0.5.87, still gets injected messages, and they stay - unread, as before: 0.5.87's XML tool path doesn't carry them, and warns - that they'll come next turn instead. -- A stream that says it can't carry injected messages (membrane's XML - prefill path, in releases after 0.5.87) gets none: everything deferred - for the agent during the turn waits for the turn's end, after its last - round, as a held message does, instead of being stored between rounds the - stream never showed it. + messages (membrane's `ToolContext.supportsInjectedMessages`, which + membrane's main sets true on its native path): the consumed watermark moves + when messages are injected, not only at a compile. Before, an injected + message sat above the watermark of the turn's compile, so a coalesced + replacement for its subject could remove a message the agent had heard and + answered. Within a branch the watermark only advances: a compile that + started before such a mark doesn't roll it back when it finishes. A stream + that doesn't say, as on membrane 0.5.87, still gets injected messages, and + they stay unread, as before: 0.5.87's XML tool path doesn't carry them, and + warns that they'll come next turn instead. +- A stream that says it can't carry injected messages (as membrane's main says + of its XML prefill path) gets none: everything deferred for the agent during + the turn waits for the turn's end, after its last round, as a held message + does, instead of being stored between rounds the stream never showed it. - A deferred write that a flush defers again does so whatever its content, under its id and in its place in the order writes arrived. That happens while its target's own turn, stream or tool cycle is alive, or the host is From 2d5d41e4cbd75474753bdd69919f6c0611bcc129 Mon Sep 17 00:00:00 2001 From: Duncan-2410 Date: Sat, 17 Oct 2026 11:15:39 -0400 Subject: [PATCH 32/34] test(deferred-window-only): throw after the store through the strategy, and pin landed shards to context-manager 0.14 From context-manager 0.15, a host's onMessage listener can no longer make a write throw: the manager logs the listener's throw and the write stands. A sharded write that fails partway now removes what it wrote. Five of these tests made their throw through such a listener. Against 0.15, four failed, and the fifth, the re-added write weighed for its agent's open turn, passed only because its throw no longer happened. - throwOnceAfterStoring throws through the strategy's onNewMessage instead: once, synchronously, for the record with the given text. The manager's own store listener calls it as each record is written, on 0.14 and 0.15 alike, so on both the write throws after its message is stored. The default strategy in these tests is PassthroughStrategy, whose onNewMessage does nothing, so skipping it for that one record changes nothing else. The single-message tests at a tool boundary, at the turn end and on a re-add now run the same path on both versions. - The two tests of carrying shards that landed before a throw are skipped when the installed context-manager is 0.15 or later, since no shards land before a throw there. The skip says so. The version comes from the installed package's package.json, which its exports don't expose, so a helper walks up from the entry point. The tests run on 0.14.x, which the dependency's range admits. Co-Authored-By: Claude Opus 5.5 --- test/deferred-window-only.test.ts | 68 +++++++++++++++++++++++-------- 1 file changed, 52 insertions(+), 16 deletions(-) diff --git a/test/deferred-window-only.test.ts b/test/deferred-window-only.test.ts index a3d4673bb..2f020d125 100644 --- a/test/deferred-window-only.test.ts +++ b/test/deferred-window-only.test.ts @@ -22,8 +22,9 @@ import { describe, it, beforeEach, afterEach } from 'node:test'; import assert from 'node:assert/strict'; import { tmpdir } from 'node:os'; -import { mkdtempSync, rmSync } from 'node:fs'; -import { join } from 'node:path'; +import { existsSync, mkdtempSync, readFileSync, rmSync } from 'node:fs'; +import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; import type { Module, ModuleContext, @@ -44,6 +45,33 @@ interface Interjection { content: ContentBlock[]; } +/** The installed context-manager's version. Its package's exports don't + * include package.json, so walk up from its entry point to it. */ +function contextManagerVersion(): string { + let dir = dirname(fileURLToPath(import.meta.resolve('@animalabs/context-manager'))); + for (;;) { + const file = join(dir, 'package.json'); + if (existsSync(file)) { + const pkg = JSON.parse(readFileSync(file, 'utf8')) as { name?: string; version?: string }; + if (pkg.name === '@animalabs/context-manager' && pkg.version) return pkg.version; + } + const up = dirname(dir); + if (up === dir) throw new Error("can't find @animalabs/context-manager's package.json"); + dir = up; + } +} + +/** Set, with the reason, when shards can't land before a throw: from + * context-manager 0.15, a sharded write that fails partway removes what it + * wrote. The tests of carrying landed shards pin 0.14.x. */ +const NO_SHARDS_LAND_BEFORE_A_THROW = (() => { + const version = contextManagerVersion(); + const [major, minor] = version.split('.').map(Number); + return major! > 0 || minor! >= 15 + ? `context-manager ${version} removes what a sharded write that fails partway wrote, so no shards land before a throw` + : false; +})(); + /** Tools that deliver messages while they run, the way a module or a channel * would while the agent is busy: `move` continues the turn, `halt` ends it. */ class InterjectingModule implements Module { @@ -468,16 +496,24 @@ describe('mid-turn deliveries keep the window what the stream carried', () => { await framework.stop(); }); - /** A store listener that throws once, after the record with this text is - * written: the write throws though its message is stored. */ + /** The strategy's onNewMessage throws once, synchronously, for the record + * with this text. The manager's own store listener calls it as each record + * is written, so the write throws though its message is stored. A host's + * onMessage listener can't do this on context-manager 0.15, which logs its + * throw and lets the write stand. */ function throwOnceAfterStoring(framework: AgentFramework, textOf: string): void { + const strategy = (framework.getAgent('assistant')!.getContextManager() as unknown as { + strategy: { onNewMessage?: (message: { content: ContentBlock[] }, ctx: unknown) => Promise }; + }).strategy; + const onNewMessage = strategy.onNewMessage; let thrown = false; - framework.getAgent('assistant')!.getContextManager().onMessage((event) => { - if (thrown || event.type !== 'add') return; - if (!event.message.content.some((b) => (b as { text?: string }).text === textOf)) return; - thrown = true; - throw new Error('listener failed'); - }); + strategy.onNewMessage = function (this: unknown, message, ctx) { + if (!thrown && message.content.some((b) => (b as { text?: string }).text === textOf)) { + thrown = true; + throw new Error('strategy failed'); + } + return onNewMessage ? onNewMessage.call(this, message, ctx) : Promise.resolve(); + }; } it('at a tool boundary, a write that throws after its message is stored counts as stored: injected, and not stored again', async () => { @@ -507,7 +543,7 @@ describe('mid-turn deliveries keep the window what the stream carried', () => { 'assistant: Done.', ], 'stored once, where the stream carried it'); assert.ok(errors.some((e) => e.includes("threw after it was stored; what was stored counts as written and isn't stored again") - && e.includes('listener failed')), errors.join('\n')); + && e.includes('strategy failed')), errors.join('\n')); await framework.stop(); }); @@ -521,7 +557,7 @@ describe('mid-turn deliveries keep the window what the stream carried', () => { : null; } - it("at a tool boundary, a sharded body that throws after its first shard: what landed is carried and counts as written, and the rest isn't stored", async () => { + it("at a tool boundary, a sharded body that throws after its first shard: what landed is carried and counts as written, and the rest isn't stored", { skip: NO_SHARDS_LAND_BEFORE_A_THROW }, async () => { membrane.pushResponse(createMockResponse([ { type: 'text', text: 'Moving.' }, { type: 'tool_use', id: 'c1', name: 'robot--move', input: {} }, @@ -529,7 +565,7 @@ describe('mid-turn deliveries keep the window what the stream carried', () => { membrane.pushResponse(createMockResponse(text('Done.'))); const framework = await createFramework(); shardOn(framework, 'part one'); - // The listener throws on the first shard's add, which stops the append + // The strategy throws on the first shard's add, which stops the append // loop before the second shard. throwOnceAfterStoring(framework, 'part one'); module.interjections = [ @@ -552,11 +588,11 @@ describe('mid-turn deliveries keep the window what the stream carried', () => { ], 'the first shard alone, stored once'); assert.equal(internalsOf(framework).deferredMessages.length, 0, 'nothing went back into the queue'); assert.ok(errors.some((e) => e.includes("threw after 1 shard of its body was stored; what was stored counts as written and isn't stored again") - && e.includes('listener failed')), errors.join('\n')); + && e.includes('strategy failed')), errors.join('\n')); await framework.stop(); }); - it('at a tool boundary, the shards that landed before a throw are carried as the next compile renders them', async () => { + it('at a tool boundary, the shards that landed before a throw are carried as the next compile renders them', { skip: NO_SHARDS_LAND_BEFORE_A_THROW }, async () => { // Probe by Ansel-2428: the autobiographical strategy shards a long // message on ingest and joins a body's consecutive shards when it // compiles, so the boundary must carry what landed in that shape. The @@ -672,7 +708,7 @@ describe('mid-turn deliveries keep the window what the stream carried', () => { assert.equal(internalsOf(framework).deferredMessages.length, 0, 'nothing went back into the queue'); assert.ok(errors.some((e) => e.includes("turn-end flush: deferred write") && e.includes("threw after it was stored; what was stored counts as written and isn't stored again") - && e.includes('listener failed')), errors.join('\n')); + && e.includes('strategy failed')), errors.join('\n')); await framework.stop(); // stop() lands whatever is still queued const reopened = await createFramework(); From caedfd2a557c51904d982106cd5264be3db97a20 Mon Sep 17 00:00:00 2001 From: Duncan-2410 Date: Sat, 17 Oct 2026 11:37:34 -0400 Subject: [PATCH 33/34] test(deferred-window-only): the re-add test checks the line its throw logs The test of a write that throws after it's stored, re-added from the queue, silenced console.error, so it passed whether or not its throw reached the write, as it did against context-manager 0.15 before the throw moved to the strategy. It now collects the lines, as its siblings do, and checks the one writeDrainedInOrder logs for the copy it found, so it fails on its own if the throw stops reaching the write. Co-Authored-By: Claude Opus 5.5 --- test/deferred-window-only.test.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/test/deferred-window-only.test.ts b/test/deferred-window-only.test.ts index 2f020d125..ebc0cf890 100644 --- a/test/deferred-window-only.test.ts +++ b/test/deferred-window-only.test.ts @@ -668,8 +668,9 @@ describe('mid-turn deliveries keep the window what the stream carried', () => { // addMessage notes it as one of the turn's arrivals. const open = { engaged: new Set(['chan-beta']), arrivals: [] as Array<{ messageId: string; channelId: string }> }; internals.openLogical.set('assistant', open); + const errors: string[] = []; const error = console.error; - console.error = () => {}; + console.error = (...args: unknown[]) => { errors.push(args.map(String).join(' ')); }; let written: unknown[]; try { written = internals.readdDrained( @@ -677,6 +678,9 @@ describe('mid-turn deliveries keep the window what the stream carried', () => { 'test flush', ); } finally { console.error = error; } + assert.ok(errors.some((e) => e.includes('test flush: deferred write') + && e.includes("threw after it was stored; what was stored counts as written and isn't stored again") + && e.includes('strategy failed')), errors.join('\n')); assert.equal(written.length, 1, 'it counts as written'); const copies = framework.getAgent('assistant')!.getContextManager().getAllMessages() .filter((m) => (m.metadata as { deferredWriteId?: string } | undefined)?.deferredWriteId === 'w1'); From ba0bc2981d8b712c16b36036e053a8da60fa887a Mon Sep 17 00:00:00 2001 From: Duncan-2410 Date: Sat, 17 Oct 2026 13:09:59 -0400 Subject: [PATCH 34/34] fix(deferral): note a write that threw after storing in the order it landed readdDrained took the notes addMessage misses for a write that threw after storing in a loop after the whole drain, so a later write that landed was noted first. An open turn's arrivals then read B, A, and when the turn resumed, remainingArrival re-pinned its locus to the earlier write's channel rather than the later one's. writeDrainedInOrder now hands such an entry to an onLanded callback as soon as its stored copy is found, before the writes after it, and readdDrained takes its notes there. DrainedWrite's afterThrow, which only that loop read, goes. Test: two writes re-added from the queue to an open turn engaged with both channels, the earlier throwing after it's stored. The arrivals read alpha, then beta, and the turn's remaining arrival is beta's channel. Co-Authored-By: Claude Opus 5.5 --- src/framework.ts | 30 ++++++++++++++++------------ test/deferred-window-only.test.ts | 33 +++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 13 deletions(-) diff --git a/src/framework.ts b/src/framework.ts index 836fdfa61..30420ed6a 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -491,14 +491,13 @@ const bySeq = (a: { seq: number }, b: { seq: number }): number => a.seq - b.seq; /** A drained entry as writeDrainedInOrder wrote it: the id its store gave it * (empty when addMessage deferred it again; a sharded body's first shard) - * and what the store holds of it. `afterThrow` marks an entry whose write - * threw after storing it, found in the store and counted as written: its - * content is what landed, as later compiles render it, which for a sharded - * body can be its first shards alone. */ + * and what the store holds of it. For an entry whose write threw after + * storing it, found in the store and counted as written, the content is + * what landed, as later compiles render it, which for a sharded body can be + * its first shards alone. */ interface DrainedWrite { id: MessageId; content: ContentBlock[]; - afterThrow?: true; } /** What later compiles render from the store for the copies of one write @@ -15831,12 +15830,15 @@ export class AgentFramework { * * Returns what was written (stored, or deferred again by addMessage, whose * id is then empty), in queue order: for one target, a prefix of its - * entries. + * entries. `onLanded` hears an entry whose write threw after storing it, + * with the id it landed under, as soon as it's found: before the writes + * after it, so whatever it records keeps their order. */ private writeDrainedInOrder( entries: DeferredWrite[], where: string, write: (msg: DeferredWrite) => MessageId, + onLanded?: (msg: DeferredWrite, id: MessageId) => void, ): Map { const ordered = [...entries].sort(bySeq); // An entry's target as addMessage resolves it, keyed by name for the cut. @@ -15854,8 +15856,9 @@ export class AgentFramework { } catch (error) { const landed = agent && before !== undefined ? this.storedCopiesOf(agent, before, msg.id) : []; if (landed.length > 0) { - written.set(msg, { id: landed[0]!.id, content: asCompiled(landed), afterThrow: true }); + written.set(msg, { id: landed[0]!.id, content: asCompiled(landed) }); this.logDeferredWriteError(msg, where, error, { count: landed.length, sharded: landed[0]!.bodyGroupId !== undefined }); + onLanded?.(msg, landed[0]!.id); continue; } cut.add(target); @@ -15925,7 +15928,9 @@ export class AgentFramework { * it again, in queue order and cut per target (writeDrainedInOrder). With * `bypassDeferral`, each is stored past its target's deferral (stop()). A * write that threw after storing missed the notes addMessage takes after a - * store, so they're taken here for what landed. */ + * store, so they're taken for what landed as soon as it's found, before + * the writes after it: an open turn's arrivals keep the order the writes + * landed in, and its newest stays the newest (remainingArrival). */ private readdDrained(entries: DeferredWrite[], where: string, opts?: { bypassDeferral?: boolean }): DeferredWrite[] { const bypassFor = (msg: DeferredWrite): string | undefined => opts?.bypassDeferral ? this.deliveryAgent(msg.forAgent)?.name : undefined; @@ -15934,11 +15939,10 @@ export class AgentFramework { fromQueue: { id: msg.id, seq: msg.seq }, ...(msg.forAgent ? { forAgent: msg.forAgent } : {}), ...(bypassFor(msg) ? { bypassDeferralFor: bypassFor(msg) } : {}), - })); - for (const [msg, w] of written) { - const agent = w.afterThrow ? this.deliveryAgent(msg.forAgent) : undefined; - if (agent) this.noteStored(agent, msg.metadata, w.id, bypassFor(msg)); - } + }), (msg, id) => { + const agent = this.deliveryAgent(msg.forAgent); + if (agent) this.noteStored(agent, msg.metadata, id, bypassFor(msg)); + }); return [...written.keys()]; } diff --git a/test/deferred-window-only.test.ts b/test/deferred-window-only.test.ts index ebc0cf890..44145ca24 100644 --- a/test/deferred-window-only.test.ts +++ b/test/deferred-window-only.test.ts @@ -689,6 +689,39 @@ describe('mid-turn deliveries keep the window what the stream carried', () => { await framework.stop(); }); + it("writes re-added from the queue are noted for an open turn in the order they landed, one that throws after it's stored among them", async () => { + const framework = await createFramework(); + throwOnceAfterStoring(framework, 'from alpha'); + const agent = framework.getAgent('assistant')!; + const internals = framework as unknown as { + openLogical: Map; + readdDrained(entries: Array<{ id: string; seq: number; participant: string; content: ContentBlock[]; metadata?: Record }>, where: string): unknown[]; + remainingArrival(agent: unknown, routing: unknown): string | undefined; + }; + // An open turn engaged with both channels, between its streams: the + // earlier write throws after it's stored, and the later one lands. + const open = { engaged: new Set(['chan-alpha', 'chan-beta']), arrivals: [] as Array<{ messageId: string; channelId: string }> }; + internals.openLogical.set('assistant', open); + const errors: string[] = []; + const error = console.error; + console.error = (...args: unknown[]) => { errors.push(args.map(String).join(' ')); }; + try { + internals.readdDrained([ + { id: 'w1', seq: 1, participant: 'Antra', content: text('from alpha'), metadata: { channelId: 'chan-alpha' } }, + { id: 'w2', seq: 2, participant: 'Antra', content: text('from beta'), metadata: { channelId: 'chan-beta' } }, + ], 'test flush'); + } finally { console.error = error; } + assert.equal(errors.filter((e) => e.includes('threw after it was stored')).length, 1, errors.join('\n')); + const idOf = (writeId: string) => agent.getContextManager().getAllMessages() + .find((m) => (m.metadata as { deferredWriteId?: string } | undefined)?.deferredWriteId === writeId)!.id; + assert.deepEqual(open.arrivals, [ + { messageId: idOf('w1'), channelId: 'chan-alpha' }, + { messageId: idOf('w2'), channelId: 'chan-beta' }, + ]); + assert.equal(internals.remainingArrival(agent, open), 'chan-beta', "the turn's newest arrival is the later write's"); + await framework.stop(); + }); + it('at the turn end, a write that throws after its message is stored is not queued again', async () => { membrane.pushResponse(createMockResponse([ { type: 'text', text: 'Moving.' },