diff --git a/changelog.d/fix-188-route-failure-owner.fixed.md b/changelog.d/fix-188-route-failure-owner.fixed.md new file mode 100644 index 000000000..4c15514d3 --- /dev/null +++ b/changelog.d/fix-188-route-failure-owner.fixed.md @@ -0,0 +1,9 @@ +- Notices about an agent's own action now reach the window that agent reads, deferred against that agent's own turn, stream and tool cycle (#188). Exceptions: a failed reply its speaker will never read, and a channel an ended agent's send opened, are told to the primary by name; any other notice for an agent no longer registered goes nowhere, logged. + - The notices: a failed reply (`[discord-send-failed]`), bounced prose (beside its `{{unsent}}` clipboard), the notices about the agent's drafts (a draft held at once, an `{{unsent}}` that names nothing or a draft it can't send, a resend whose outcome couldn't be recorded, and `>>skip_reply {{unsent}}`'s), a background script's wake, and a channel opened by the agent's own send. A channel opened by subscription policy, with no agent behind it, is still announced to the primary. + - A conversation fork's or an ephemeral run's notices land in its own window instead of the primary's. In connectome-host this moves a subagent's notices from the primary, where the default path put them, to the subagent. + - A resident's notices are addressed to it too, in the slot the residents share, and wait on its own turn rather than the primary's. Whether it may land there while another resident's turn is alive is the question of #291. + - When a fork or an ephemeral run is released, each failed-reply notice it never read is passed to the primary, once, by name, in the words of the speaker's own notice: "A reply by `` (N chars) could not be delivered to … (…). Nothing was posted. `` has ended.", or, for a delivery that wasn't confirmed, "Delivery of a reply by `` (N chars) to … was not confirmed (…). It may or may not have been posted, so sending it again could post it twice. `` has ended.", with the connector's detail when there is one. Never read means still queued for it, or stored in its window above its consumed watermark: after its last compile, and after the last tool boundary whose stream declared it carries the messages injected there. After a branch switch, only what both branches share up to the point they part counts as read. The same goes for a delivery that fails after its speaker is released, and for such a notice restored at boot whose agent is gone; a notice still queued goes on as that same queue entry, so a crash before its ack passes it on once. That notice's metadata carries `speaker`, and the delivery's `outcome` and `detail` as the speaker's notice had them; every failed-reply notice's metadata now carries `textLen`. With no primary registered, no one is told, and it's logged. A host stopped while quiesced, before a fork's or run's window release, queues the primary's notice of each failed reply that window holds unread, for resume or boot. + - A channel opened by an agent's own send after that agent has ended (an ephemeral run's final reply is routed after the run settles) is announced to the primary, by name: "Now open: … — opened by ``'s reply; `` has ended.", with the `channel_close` line. Its metadata carries `speaker`. With no primary registered, no one is told, and it's logged. + - Writes still queued for a fork or an ephemeral run when it is released land in its own window, which Chronicle keeps, with one acknowledgement once a failed reply among them has gone on to the primary. A fork or run that ends while the host is quiesced is deregistered at once, and only its window's release waits for resume, which lands them then; no other flush takes them meanwhile. A write addressed to it meanwhile is treated as for any agent no longer registered. + - The fallback reaper no longer disposes a closing fork whose turn is still alive before its stream has registered. + - Any other notice or background-script wake made for an agent that is no longer registered is dropped, logged once, with no trace and no wake. diff --git a/src/framework.ts b/src/framework.ts index 394ccd81e..7f358a9c1 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -2191,6 +2191,9 @@ export class AgentFramework { * ephemeral agents have isolated slots and are their own only readers. */ private readonly sharedSlotAgents = new Set(); + /** The windows of forks and runs deregistered while the host was quiesced, + * by agent name; resume() releases them (releaseWindowNowOrAfterResume). */ + private readonly releasesAfterResume = new Map(); /** The coalescing commit in flight, if any (see commitCoalescingDurable). */ private coalescingCommit: Promise | null = null; private inferenceRouter: InferenceRouter | null = null; @@ -3312,8 +3315,8 @@ export class AgentFramework { // queued with what follows it for its target, and a memory-only queue // still loses it at exit. An entry for a target no longer registered (a // disposed fork) is left: dropped with a memory-only queue at exit, and - // logged and dropped from a persisted one by the boot or resume flush - // that meets it. + // from a persisted one, by the boot or resume flush that meets it, passed + // to the primary if it's a failed reply, and otherwise dropped, logged. // // A registered agent's stream that outlasted its wait may still write, // so that agent's entries stay queued, said once for it: no entry lands @@ -3323,6 +3326,15 @@ export class AgentFramework { // turn-end flush takes them, as at any turn's end, and a later stop() // takes what that leaves. Until then a persisted queue keeps them for // boot, and a memory-only one loses them at exit. + // A window release the quiesce postponed (releasesAfterResume) never runs + // once the host stops, so its unread pass runs here: each failed reply + // stored unread in that window goes to the primary now, deferred while + // quiesced like any write, so it waits in the queue with the rest. What + // was queued for that agent stays queued, as a gone agent's. + for (const agent of this.releasesAfterResume.values()) { + for (const metadata of this.unreadFailedReplies(agent)) this.passFailedReplyToPrimary(agent.name, metadata); + } + this.releasesAfterResume.clear(); if (this.deferredMessages.length > 0) { const registered = [...this.agents.keys()]; const entries = registered @@ -5589,10 +5601,15 @@ export class AgentFramework { this.resumeInFlight = true; try { - // Flush context writes deferred by the quiesce window (module events, - // api message.send) — with no turn alive there is no other flush - // point. Per-message try/catch and per-message durable ack (see - // flushDeferredWrites); nothing here may reach the plane reopen below. + // The windows of forks and runs deregistered during the window are + // released first, each landing its queued writes and passing on its + // unread failed replies, as at any release. Then flush context writes + // deferred by the quiesce window + // (module events, api message.send) — with no turn alive there is no + // other flush point. Per-message try/catch and per-message durable ack + // (see flushDeferredWrites); nothing here may reach the plane reopen + // below. + this.runReleasesAfterResume(); await this.flushDeferredWrites('resume'); this.persistHostMode(null); this.hostModeRecordMayBeQuiesced = false; @@ -5673,12 +5690,28 @@ export class AgentFramework { flush.sort(bySeq); this.deferredMessages = keep; this.handOffDeferredWrites(flush); - const stored = this.readdDrained(flush, `host-mode ${label}`).length; + // A target no longer registered (a fork or a run gone before a crash + // that left this write to boot recovery) has no window to land in. A + // failed reply it never read becomes the primary's by-name notice, under + // the entry's own id and place in the queue (queuedFailedReplyForPrimary), + // and is written with the rest; any other write is dropped, logged, by + // addMessage. + let passed = 0; + const batch = flush.flatMap((msg) => { + if (msg.forAgent === undefined || this.agents.has(msg.forAgent)) return [msg]; + if ((msg.metadata as { kind?: unknown } | undefined)?.kind !== 'discord-send-failed') return [msg]; + const forPrimary = this.queuedFailedReplyForPrimary(msg.forAgent, msg); + if (!forPrimary) return []; + passed++; + return [forPrimary]; + }); + const stored = this.readdDrained(batch, `host-mode ${label}`).length; // Ack = sync the chronicle FIRST, then rewrite the durable queue without // the batch. Never the other way round. this.ackDeferredWrites(); console.error( `[host-mode] ${label}: flushed ${stored}/${flush.length} deferred context write(s)` + + (passed > 0 ? `, ${passed} failed repl${passed === 1 ? 'y' : 'ies'} of a released agent passed to the primary` : '') + (keep.length > 0 ? `, ${keep.length} kept for a still-alive turn` : ''), ); } @@ -7371,17 +7404,12 @@ export class AgentFramework { if (this.ephemeralRuns.get(agent.name) === run) this.ephemeralRuns.delete(agent.name); if (this.agents.get(agent.name) === agent) { // What is still queued for the run, such as a message held for its - // turn's end, goes to its store now, past its deferral: once the run - // is deregistered, a flush finds no agent to take it and drops it. - // The run's namespace stays in the store for investigation, so the - // messages stay with it. A quiesced host defers them again instead, - // as stop() does, and with the run gone, the flush that meets them - // later drops them. - const queued = this.drainDeferredFor(agent.name); - if (queued.length > 0) { - this.readdDrained(queued, `end of ephemeral run ${agent.name}`, { bypassDeferral: !this.quiesced }); - this.ackDeferredWrites(); - } + // turn's end, is left to its window's release below + // (releaseWindowNowOrAfterResume), as for a disposed fork: it lands + // there in the run's store, which keeps the run's namespace for + // investigation, now or, while quiesced, at resume. A failed reply + // among it goes on to the primary as the same queue entry, so no ack + // comes between the entry and the primary's notice. // Exact-generation disposal owns these name-keyed entries. Remove them // before deregistration; the late driveStream finally will see the // generation mismatch and leave any future owner untouched. @@ -7394,6 +7422,7 @@ export class AgentFramework { // deregistered, so disposal must release EventGate liveness itself. this.eventGate?.onInferenceEnded(agent.name); this.agents.delete(agent.name); + this.releaseWindowNowOrAfterResume(agent); } this.toolImageLedgers.delete(agent.name); // Spawn-and-dispose bookkeeping (main, d453165/fee96a7): without this, @@ -11210,6 +11239,7 @@ export class AgentFramework { const channelId = this.conversationAgentHomes.get(agentName); const agent = this.agents.get(agentName); this.agents.delete(agentName); + if (agent) this.releaseWindowNowOrAfterResume(agent); this.turnRouting.delete(agentName); this.proseSpeechChains.delete(agentName); this.openLogical.delete(agentName); @@ -11309,9 +11339,12 @@ export class AgentFramework { // Fallback reaper: a closing fork whose closure inference never ran // (request dropped as stale, inference policy veto) would otherwise stay - // registered forever. No active stream + no pending request = done. + // registered forever. No turn, no active stream and no pending request = + // done. A turn holds its token from dequeue, through its hooks and + // compile, before its stream registers, so a fork mid-compile is busy. for (const agentName of [...this.closingConversationAgents]) { if ( + !this.activeTurnTokens.has(agentName) && !this.activeStreams.has(agentName) && !this.pendingRequests.some((r) => r.agentName === agentName) ) { @@ -12804,12 +12837,13 @@ export class AgentFramework { textLen: held.reduce((n, d) => n + d.text.length, 0), }); if (opts.notice === 'now') { + const target = this.noticeTargetFor(agent.name, 'held-draft'); try { - this.addMessage( + if (target) this.addMessage( 'user', [{ type: 'text', text: this.draftsHeldNotice(agent.name, held, reason) }], { system: true, kind: 'prose-drafts', draftOwner: agent.name, draftIds: held.map((d) => d.id) } as MessageMetadata, - { forAgent: agent.name }, + target, ); } catch (err) { console.error('[drafts] held-draft notice failed:', err); @@ -13707,22 +13741,44 @@ export class AgentFramework { .map((c) => (c.label && c.label !== c.channelId ? `${c.label} (${c.channelId})` : c.channelId)) .join(', '); const plural = channels.length > 1; - const causeText = - cause === 'subscription-policy' - ? 'your subscription policy just admitted ' + (plural ? 'them' : 'it') - : 'your own message into the closed channel engaged it'; - const text = - `[channels] Now open: ${shown} — ${causeText}. What an open channel delivers to you depends on its server. ` + + const optOut = ' What an open channel delivers to you depends on its server. ' + `If you don't want ${plural ? 'one of them' : 'it'}, use channel_close — ` + 'your choice sticks; neither policy nor a later send will silently reopen it.'; + // An open caused by an agent's own send is that agent's to hear about, in + // the window it reads; a policy admission with no agent behind it goes to + // the primary. An agent that has ended (a run whose final reply is routed + // after its release) can close nothing, and no channel opens without + // someone being told who can close it: the primary, the one left, is told + // by name, as for a failed reply. The default path would fall back to the + // first registered agent when there is no primary. + let text: string; + let metadata = { system: true, kind: 'channel-notice' } as MessageMetadata; + let target: { forAgent: string } | undefined; + let told: string; + if (agentName && !this.agents.has(agentName)) { + if (!this.primaryAgentName || !this.agents.has(this.primaryAgentName)) { + console.error(`[channel] ${agentName} has ended, and there is no primary to tell that ${shown} opened`); + return; + } + text = `[channels] Now open: ${shown} — opened by ${agentName}'s reply; ${agentName} has ended.` + optOut; + metadata = { system: true, kind: 'channel-notice', speaker: agentName } as MessageMetadata; + told = `the primary, by name (${agentName} has ended)`; + } else { + const causeText = + cause === 'subscription-policy' + ? 'your subscription policy just admitted ' + (plural ? 'them' : 'it') + : 'your own message into the closed channel engaged it'; + text = `[channels] Now open: ${shown} — ${causeText}.` + optOut; + if (agentName) target = { forAgent: agentName }; + told = agentName ?? 'primary'; + } try { // Through framework.addMessage for mid-turn safety (deferral + // hear-while-acting injection) — a delivery-forced open happens inside // a live turn, and a direct context-manager append there would store - // the notice out of order. Targets the primary agent; connectome-host - // runs trunk-only, so this is the delivering agent in practice. - this.addMessage('user', [{ type: 'text', text }], { system: true, kind: 'channel-notice' }); - console.error(`[channel] auto-open notice (${cause}) -> ${agentName ?? 'primary'}: ${shown}`); + // the notice out of order. + this.addMessage('user', [{ type: 'text', text }], metadata, target); + console.error(`[channel] auto-open notice (${cause}) -> ${told}: ${shown}`); } catch (err) { console.error('recordChannelAutoOpenNotice failed:', err); } @@ -13851,12 +13907,13 @@ export class AgentFramework { if (own) { this.bounceProse(agent, own, reason, undefined, prefix, round); } else { + const target = this.noticeTargetFor(agent.name, 'empty-{{unsent}}'); try { - this.addMessage( + if (target) this.addMessage( 'user', [{ type: 'text', text: `[prose-routing] ${reason}, so nothing was sent. drafts(action: "list") shows your open drafts.` }], { system: true, kind: 'prose-bounce' }, - { forAgent: agent.name }, + target, ); } catch (err) { console.error('[drafts] empty-unsent notice failed:', err); @@ -13901,13 +13958,14 @@ export class AgentFramework { : ' Your message could not be held as a draft; its words remain in your history.'; const text = `[prose-routing] {{unsent}} is draft ${draft.id}, ${why.startsWith('which') ? why : `and ${why}`}. ` + `Nothing was sent; ${next}.${kept}`; + const target = this.noticeTargetFor(agent.name, 'refused-{{unsent}}'); try { - this.addMessage( + if (target) this.addMessage( 'user', [{ type: 'text', text }], // It names the refused draft as well as any copy it held. { system: true, kind: 'prose-bounce', draftOwner: agent.name, draftIds: held ? [draft.id, held.id] : [draft.id] } as MessageMetadata, - { forAgent: agent.name }, + target, ); } catch (err) { console.error('[drafts] refused-unsent notice failed:', err); @@ -13965,12 +14023,13 @@ export class AgentFramework { const text = `[prose-routing] {{unsent}} was draft ${unsent.id}; its send came back ${outcome.status}, but recording that ` + `failed, so the draft reads as ${this.draftStateNow(agent.name, unsent.id)} now. drafts(action: "read") shows it.`; + const target = this.noticeTargetFor(agent.name, 'unrecorded-outcome'); try { - this.addMessage( + if (target) this.addMessage( 'user', [{ type: 'text', text }], { system: true, kind: 'prose-bounce', draftOwner: agent.name, draftIds: [unsent.id] } as MessageMetadata, - { forAgent: agent.name }, + target, ); } catch (noticeErr) { console.error('[drafts] unrecorded-outcome notice failed:', noticeErr); @@ -14108,8 +14167,10 @@ export class AgentFramework { const draft = this.proseDrafts.latestBounce(agent.name); if (!draft) return; const tell = (text: string, what: string): void => { + const target = this.noticeTargetFor(agent.name, what); + if (!target) return; try { - this.addMessage('user', [{ type: 'text', text }], { system: true, kind: 'prose-bounce' }, { forAgent: agent.name }); + this.addMessage('user', [{ type: 'text', text }], { system: true, kind: 'prose-bounce' }, target); } catch (err) { console.error(`[drafts] ${what} notice failed:`, err); } @@ -14311,9 +14372,12 @@ export class AgentFramework { const notice = `[prose-routing] Your text (${text.length} chars) may not have been delivered — ${reason}. ` + "It isn't held as a draft, since a resend could post it twice; the words remain in your history."; + const target = this.noticeTargetFor(name, 'unknown-send'); try { - const id = this.addMessage('user', [{ type: 'text', text: notice }], { system: true, kind: 'prose-bounce' } as MessageMetadata, { forAgent: name }); - this.emitTrace({ type: 'message:added', messageId: id ?? 'deferred', source: 'prose-bounce' }); + if (target) { + const id = this.addMessage('user', [{ type: 'text', text: notice }], { system: true, kind: 'prose-bounce' } as MessageMetadata, target); + this.emitTrace({ type: 'message:added', messageId: id ?? 'deferred', source: 'prose-bounce' }); + } } catch (err) { console.error('bounceProse: failed to record the unknown-send notice:', err); } @@ -14370,31 +14434,34 @@ export class AgentFramework { const notice = `[prose-routing] Your text (${text.length} chars) ${delivered} — ${reason}.${cand} ${kept}${risky}` + (prefix === '>>' ? ' The prose_help tool shows the full syntax.' : ''); - try { - // Through framework.addMessage, NOT the context manager directly: while - // the turn is still streaming this defers the notice to the next tool - // boundary, where it is BOTH stored in order and injected into the live - // stream (hear-while-acting) — the model sees the error and can correct - // the prefix within the same turn. A direct append here would land the - // notice before the turn's assistant blocks (cache bust, invisible - // until the next compile). Addressed to the bouncing resident alone. - const id = this.addMessage( - 'user', - [{ type: 'text', text: notice }], - { system: true, kind: 'prose-bounce', ...(draft ? { draftOwner: name, draftIds: [draft.id] } : {}) } as MessageMetadata, - { forAgent: name }, - ); - if (id) this.emitTrace({ type: 'message:added', messageId: id, source: 'prose-bounce' }); - else this.emitTrace({ type: 'message:added', messageId: 'deferred', source: 'prose-bounce' }); - } catch (err) { - console.error('bounceProse: failed to record bounce notice:', err); + const target = this.noticeTargetFor(name, 'prose-bounce'); + if (target) { + try { + // Through framework.addMessage, NOT the context manager directly: while + // the turn is still streaming this defers the notice to the next tool + // boundary, where it is BOTH stored in order and injected into the live + // stream (hear-while-acting) — the model sees the error and can correct + // the prefix within the same turn. A direct append here would land the + // notice before the turn's assistant blocks (cache bust, invisible + // until the next compile). Addressed to the bouncing agent (noticeTargetFor). + const id = this.addMessage( + 'user', + [{ type: 'text', text: notice }], + { system: true, kind: 'prose-bounce', ...(draft ? { draftOwner: name, draftIds: [draft.id] } : {}) } as MessageMetadata, + target, + ); + if (id) this.emitTrace({ type: 'message:added', messageId: id, source: 'prose-bounce' }); + else this.emitTrace({ type: 'message:added', messageId: 'deferred', source: 'prose-bounce' }); + } catch (err) { + console.error('bounceProse: failed to record bounce notice:', err); + } } const capped = streak > AgentFramework.PROSE_BOUNCE_WAKE_CAP; console.error( `[prose] ${name}: BOUNCED ${text.length} chars — ${reason}` + (capped ? ' (bounce-wake cap reached; notice appended without wake)' : ''), ); - if (!capped) { + if (!capped && target) { // Wake so the resend can happen immediately — an unsent DM reply must // not sit invisible until the next unrelated event. this.pendingRequests.push({ @@ -17003,7 +17070,9 @@ export class AgentFramework { // this flush, and another agent's pending tools say nothing about this // window. addMessage judges each entry by the same rule, so the drained // prefix is stored in order, or goes back into the queue whole under - // its ids when a successor turn has already taken the agent. + // its ids when a successor turn has already taken the agent. A fork or + // run deregistered while the host is quiesced leaves its queue to its + // window's release at resume (drainDeferredFor). if (frameReachedTerminal && this.deferredMessages.length > 0 && !this.pendingAssistantBlocks.has(agent.name)) { this.readdDrained(this.drainDeferredFor(agent.name), 'turn-end flush'); this.ackDeferredWrites(); @@ -17744,12 +17813,14 @@ export class AgentFramework { * later if that ever needs revisiting. */ private injectScriptWake(record: BackgroundScriptRecord, envelope: string): void { + const target = this.noticeTargetFor(record.agentName, `background-script ${record.id} wake`); + if (!target) return; try { const messageId = this.addMessage('user', [{ type: 'text', text: envelope }], { source: 'background-script', scriptId: record.id, tags: ['script:wake'], - }); + }, target); this.pendingRequests.push({ agentName: record.agentName, reason: 'script:wake', @@ -17758,7 +17829,7 @@ export class AgentFramework { }); this.emitTrace({ type: 'message:added', - messageId, + messageId: messageId || 'deferred', source: `background-script:${record.id}`, }); } catch (err) { @@ -18960,6 +19031,11 @@ export class AgentFramework { */ private drainDeferredFor(agentName: string, stopAt?: (msg: DeferredWrite) => boolean): DeferredWrite[] { if (this.deferredMessages.length === 0) return []; + // A fork or run deregistered while the host is quiesced leaves its queue + // to its window's release at resume (runReleasesAfterResume, which clears + // the name first): drained by any other flush, its entries would meet no + // registered agent, be dropped, and be acked. + if (this.releasesAfterResume.has(agentName)) return []; const mine: typeof this.deferredMessages = []; const rest: typeof this.deferredMessages = []; for (const msg of this.deferredMessages) { @@ -19149,6 +19225,267 @@ export class AgentFramework { return [...written.keys()]; } + /** + * Where a notice about an agent's own action belongs: its failed reply, its + * bounced prose (beside its `{{unsent}}` clipboard), its background script's + * wake, the notices about its drafts. Every registered agent's notice is + * addressed to that agent, so it lands in the window that agent reads and + * waits on that agent's own turn, stream and tool cycle: a conversation + * fork or an ephemeral run reads only its isolated slot, and a resident's + * notice would otherwise wait on the primary's turn rather than its own. + * Residents share one message slot, so a resident's notice can land there + * while another resident's turn is alive; agent-framework #291 covers that. + * + * null when the agent is no longer registered (a fork or a run already + * released): there is no window to write, so the caller writes nothing, + * traces nothing and requests no wake. The drop is logged here, once. A + * failed reply is passed on by name instead (passFailedReplyToPrimary). The + * notice of a channel an agent's own send opened follows the same rule in + * recordChannelAutoOpenNotice, which passes it on by name too. + */ + private noticeTargetFor(agentName: string, notice: string): { forAgent: string } | null { + if (this.agents.has(agentName)) return { forAgent: agentName }; + console.error(`[notice] ${agentName} is no longer registered, so its ${notice} notice is dropped`); + return null; + } + + /** + * A conversation fork or an ephemeral run has been deregistered. A release + * never runs while the host is quiesced: it waits for resume() + * (releaseWindowNowOrAfterResume), since a write could land mid-surgery. + * + * The agent never reads its window again, so every failed-reply notice it + * never read is passed to the primary, by name, once: each one stored in + * its window above its consumed watermark (flushed there at its turn's end, + * or stored while it was idle) and not heard at a tool boundary, and each + * one still queued for it. Writes still + * queued would wait for a boundary it never reaches, and the flush that + * finds it gone would drop them, so they land in its own window now, which + * Chronicle keeps for investigation. Its stream is over or cancelled, so it + * compiles nothing more. One path still stores after these writes: a + * `complete` that arrived while the agent still waited for tools awaits + * the spill of its last tool results, and stores them once that resolves, + * even if a watchdog has ended the run meanwhile. A tool boundary stores + * nothing for a round whose turn was stopped while its results were + * prepared. + */ + private releaseWindow(agent: Agent): void { + for (const metadata of this.unreadFailedReplies(agent)) this.passFailedReplyToPrimary(agent.name, metadata); + const queued = this.drainDeferredFor(agent.name); + if (queued.length === 0) return; + const cm = agent.getContextManager(); + const forPrimary: DeferredWrite[] = []; + for (const msg of queued) { + try { + cm.addMessage(msg.participant, msg.content, withDeferredWriteId(msg.metadata, msg.id)); + } catch (err) { + console.error( + `[deferred-flush] ${agent.name}: failed to store a queued write before release ` + + `(participant=${msg.participant}):`, + err, + ); + } + // A failed reply, now in a window no one reads: the entry itself goes + // on to the primary as its by-name notice, under its own id and seq, so + // a crash before the ack below replays it once, not twice. + if ((msg.metadata as { kind?: unknown } | undefined)?.kind !== 'discord-send-failed') continue; + const passed = this.queuedFailedReplyForPrimary(agent.name, msg); + if (passed) forPrimary.push(passed); + } + if (forPrimary.length > 0) this.readdDrained(forPrimary, `release of ${agent.name}`); + this.ackDeferredWrites(); + } + + /** + * The failed-reply notices stored in `agent`'s window that it never read: + * those above the lower of its consumed watermark and the point where the + * watermark's branch and the current one part (branchesPartAt), or every + * one, when it never compiled. The watermark moves at a compile, and at a + * tool boundary whose + * stream declares it carries the messages injected there + * (markContextConsumed), so a notice the live stream heard is under it, and + * one a boundary stored without sending (a turn a tool ended, a budget + * restart, a stream that may drop injections) is above it. + */ + private unreadFailedReplies(agent: Agent): MessageMetadata[] { + try { + const cm = agent.getContextManager(); + const watermark = agent.getConsumedWatermark(); + const floor = watermark + ? Math.min(watermark.sequence, this.branchesPartAt(watermark.branch, cm.currentBranch().name)) + : Number.NEGATIVE_INFINITY; + return cm.getAllMessages() + .filter((m) => { + const metadata = m.metadata as { kind?: unknown } | undefined; + return m.sequence > floor && metadata?.kind === 'discord-send-failed'; + }) + .map((m) => m.metadata as MessageMetadata); + } catch (err) { + // A release must complete whatever its window's read meets. + console.error(`[route-failure] ${agent.name}: could not read its window for unread failed replies:`, err); + return []; + } + } + + /** + * The last sequence two branches' histories share: Infinity for one branch, + * and for two, the point where they part. A branch's records are numbered + * on from its branch point (Chronicle gives each record its branch's head + * plus one), so up to that point both hold the same records under the same + * numbers, and past it each branch's numbers are its own. -Infinity when + * the store knows no shared history for them, as for an empty branch. + */ + private branchesPartAt(a: string, b: string): number { + if (a === b) return Number.POSITIVE_INFINITY; + const branches = this.store.listBranches(); + const byName = new Map(branches.map((branch) => [branch.name, branch])); + const byId = new Map(branches.map((branch) => [branch.id, branch])); + // Each branch `a` descends from, with the last sequence `a` shares with it. + const sharedWithA = new Map(); + let through = Number.POSITIVE_INFINITY; + for (let x = byName.get(a); x; x = x.parentId != null ? byId.get(x.parentId) : undefined) { + sharedWithA.set(x.id, through); + if (x.branchPoint == null) break; + through = Math.min(through, x.branchPoint); + } + through = Number.POSITIVE_INFINITY; + for (let y = byName.get(b); y; y = y.parentId != null ? byId.get(y.parentId) : undefined) { + const shared = sharedWithA.get(y.id); + if (shared !== undefined) return Math.min(shared, through); + if (y.branchPoint == null) break; + through = Math.min(through, y.branchPoint); + } + return Number.NEGATIVE_INFINITY; + } + + /** + * Release a deregistered fork's or run's window (releaseWindow) now, or, + * while the host is quiesced, when it resumes. A release lands the agent's + * queued writes in its window, and while quiesced a write could land + * mid-surgery; dropping them instead would lose them from both the window + * and the recovery queue. Only the window waits: the agent is already + * deregistered, so its stream can't dispatch anything more and resume + * doesn't weigh its context, and a write addressed to it meanwhile is + * treated as for any agent no longer registered (dropped, or passed to the + * primary if it's a failed reply). + */ + private releaseWindowNowOrAfterResume(agent: Agent): void { + if (this.quiesced) this.releasesAfterResume.set(agent.name, agent); + else this.releaseWindow(agent); + } + + /** resume(): the windows whose release the quiesce window postponed, each on its own. */ + private runReleasesAfterResume(): void { + const agents = [...this.releasesAfterResume.values()]; + this.releasesAfterResume.clear(); + for (const agent of agents) { + try { + this.releaseWindow(agent); + } catch (err) { + console.error(`[host-mode] resume: the release of ${agent.name}'s window, postponed by the quiesce window, failed:`, err); + } + } + } + + /** + * A fork or an ephemeral run ended before reading that its reply's delivery + * to a channel failed: it was released with the notice stored in its window + * unread, or its delivery failed after release. (A notice still queued for + * it, at release or restored at boot, goes on as the queue entry itself: + * queuedFailedReplyForPrimary.) Tell the primary instead, naming the + * speaker, so an agent who can still act learns of it and can tell the + * human. It gives the reply's length, not its words, which stay in the + * speaker's window. Its wording follows the speaker's own notice + * (onRouteFailure): a delivery that wasn't confirmed may have been posted, + * and neither text says no one saw the reply, since only this channel's + * routing failed. Only for a delivery to a channel; a reply that had no + * channel to go to (no locus) had no human waiting for it. Never throws: + * it runs inside a release, which must complete whatever this write meets. + */ + private passFailedReplyToPrimary(speaker: string, metadata: MessageMetadata | undefined): void { + const notice = this.failedReplyNoticeForPrimary(speaker, metadata); + if (!notice) return; + try { + this.addMessage('user', notice.content, notice.metadata); + } catch (err) { + console.error(`[route-failure] failed to tell the primary that ${speaker}'s reply to ${String(notice.metadata.channelId)} failed:`, err); + } + } + + /** + * A queued failed-reply entry for `speaker`, who is gone, as the primary's + * by-name notice (passFailedReplyToPrimary's words): the same entry, under + * its own id and seq, now addressed to the primary. Written through the + * drain, it lands stamped with that id, or waits in the queue under it, so + * boot's exactly-once holds for it: a crash before the original's ack + * replays one entry, never the original and a separate notice. null when no + * one is told (logged): a reply that had no channel to reach, or no primary. + */ + private queuedFailedReplyForPrimary(speaker: string, msg: DeferredWrite): DeferredWrite | null { + const notice = this.failedReplyNoticeForPrimary(speaker, msg.metadata); + if (!notice) return null; + return { id: msg.id, seq: msg.seq, participant: 'user', content: notice.content, metadata: notice.metadata }; + } + + /** The primary's by-name notice of `speaker`'s failed reply, or null when no + * one is told, which is logged here (see passFailedReplyToPrimary). */ + private failedReplyNoticeForPrimary( + speaker: string, + metadata: MessageMetadata | undefined, + ): { content: ContentBlock[]; metadata: MessageMetadata } | null { + const failure = metadata as { + kind?: unknown; channelId?: unknown; reason?: unknown; textLen?: unknown; outcome?: unknown; detail?: unknown; + } | undefined; + if (failure?.kind !== 'discord-send-failed') return null; + if (typeof failure.channelId !== 'string' || failure.channelId === '') { + console.error(`[route-failure] ${speaker} has ended, and its reply had no channel to reach: no one else is told`); + return null; + } + // The default path falls back to the first registered agent when there + // is no primary, which could be the speaker being released. + if (!this.primaryAgentName || !this.agents.has(this.primaryAgentName)) { + console.error(`[route-failure] ${speaker} has ended, and there is no primary to tell that its reply to ${failure.channelId} failed`); + return null; + } + // A notice queued by an earlier build may carry none of these; say only + // what it has. + const textLen = typeof failure.textLen === 'number' ? failure.textLen : undefined; + const reason = typeof failure.reason === 'string' && failure.reason !== '' ? failure.reason : undefined; + const outcome = failure.outcome === 'failed' || failure.outcome === 'unknown' ? failure.outcome : undefined; + const detail = typeof failure.detail === 'string' ? failure.detail : undefined; + const size = textLen !== undefined ? ` (${textLen} chars)` : ''; + const why = reason !== undefined ? ` (${reason})` : ''; + const where = this.channelForNotice(failure.channelId); + const text = outcome === 'unknown' + ? `[discord-send-failed] Delivery of a reply by ${speaker}${size} to ${where} was not confirmed${why}. ` + + `It may or may not have been posted, so sending it again could post it twice. ${speaker} has ended.` + : `[discord-send-failed] A reply by ${speaker}${size} could not be delivered to ${where}${why}. ` + + `Nothing was posted. ${speaker} has ended.`; + return { + content: [{ type: 'text', text: text + (detail !== undefined ? ` Connector detail: ${detail}` : '') }], + metadata: { + system: true, kind: 'discord-send-failed', channelId: failure.channelId, speaker, + ...(reason !== undefined ? { reason } : {}), + ...(textLen !== undefined ? { textLen } : {}), + ...(outcome !== undefined ? { outcome } : {}), + ...(detail !== undefined ? { detail } : {}), + } as MessageMetadata, + }; + } + + /** + * A channel as a notice names it: the label when the registry has one, since + * a bare snowflake is unresolvable for the agent (the 2026-07-21 incident + * read as "a stale artifact", not a live failure). + */ + private channelForNotice(channelId: string | null): string { + if (!channelId) return 'the channel'; + const label = this.channelRegistry?.getDescriptor(channelId)?.label; + return label && label !== channelId + ? `${label.startsWith('#') ? label : `#${label}`} (${channelId})` + : channelId; + } + private editMessage(id: MessageId, content: ContentBlock[]): void { const agent = this.primaryAgentName ? this.agents.get(this.primaryAgentName) @@ -19940,21 +20277,21 @@ export class AgentFramework { // addMessage() alone does not request inference, so this never wakes // her (matching the `discord-send-failed-skip` gate intent: context // yes, wake no). - onRouteFailure: ({ channelId, reason, textLen, outcome, detail }) => { + onRouteFailure: ({ conversationId, channelId, reason, textLen, outcome, detail }) => { try { - // Render a human-readable channel name when we can — a bare - // snowflake in the marker is unresolvable for the agent (the - // 2026-07-21 incident read as "a stale artifact", not a live - // failure). The marker is `system: true`, so it is never - // conversational and never influences routing. - const label = channelId - ? this.channelRegistry?.getDescriptor(channelId)?.label - : undefined; - const where = channelId - ? label && label !== channelId - ? `${label.startsWith('#') ? label : `#${label}`} (${channelId})` - : channelId - : 'the channel'; + // The marker is `system: true`, so it is never conversational and + // never influences routing. The channel is named by its label when + // the registry has one (channelForNotice). + const where = this.channelForNotice(channelId); + const metadata: MessageMetadata = { + system: true, kind: 'discord-send-failed', channelId: channelId ?? '', reason, textLen, + ...(outcome ? { outcome } : {}), ...(detail !== undefined ? { detail } : {}), + }; + if (!this.agents.has(conversationId)) { + // Released before its delivery failed: no window left to read it. + this.passFailedReplyToPrimary(conversationId, metadata); + return; + } // No-locus failures (headless / WebUI turns with no home or // trigger channel) are not Discord failures: "[discord-send- // failed] could not be delivered to the channel" sent the agent @@ -19971,10 +20308,12 @@ export class AgentFramework { ? `[discord-send-failed] Delivery of your previous reply (${textLen} chars) to ${where} was not confirmed (${reason}). It may or may not have been posted, so sending it again could post it twice. It is saved in your archive.` : `[discord-send-failed] Your previous reply (${textLen} chars) could not be delivered to ${where} (${reason}). Nothing was posted; it is saved in your archive.`) : `[send-undeliverable] Your previous reply (${textLen} chars) had no channel to go to — ${reason}. This is a routing/configuration situation, not a channel failure. This route delivered it nowhere; if another surface showed it (a reply thread, the console), that copy stands. It is saved in your archive.`; + // A registered speaker's notice is addressed to it (noticeTargetFor). this.addMessage( 'user', [{ type: 'text', text: text + (detail !== undefined ? ` Connector detail: ${detail}` : '') }], - { system: true, kind: 'discord-send-failed', channelId: channelId ?? '', reason, ...(outcome ? { outcome } : {}), ...(detail !== undefined ? { detail } : {}) }, + metadata, + { forAgent: conversationId }, ); } catch (err) { console.error('onRouteFailure: failed to record send-failure marker:', err); diff --git a/test/puppet-tool-call.test.ts b/test/puppet-tool-call.test.ts index 412110b6c..26510b7c2 100644 --- a/test/puppet-tool-call.test.ts +++ b/test/puppet-tool-call.test.ts @@ -54,6 +54,9 @@ function puppetHarness(opts?: { fw.deferredSeq = 0; fw.deferredScanFrom = new Map(); fw.pendingAssistantBlocks = new Map(); + // Window releases a quiesce postponed, which every by-name flush checks + // (drainDeferredFor): none here, as the constructor starts it. + fw.releasesAfterResume = new Map(); // A puppet call is an operation stop() waits for. fw.operatorOperations = new Set(); fw.primaryAgentName = 'princess'; diff --git a/test/route-failure-targeting.test.ts b/test/route-failure-targeting.test.ts new file mode 100644 index 000000000..444bdc09a --- /dev/null +++ b/test/route-failure-targeting.test.ts @@ -0,0 +1,1260 @@ +import { describe, it, beforeEach, afterEach } from 'node:test'; +import assert from 'node:assert/strict'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { AgentFramework } from '../src/index.js'; +import { ContextManager, PassthroughStrategy } from '@animalabs/context-manager'; +import { CapabilityGrant } from '../src/mcpl/capability-grant.js'; +import type { McplServerRegistry } from '../src/mcpl/server-registry.js'; +import type { ProcessEvent, Module, ModuleContext, ToolDefinition, ToolCall, ToolResult, EventResponse } from '../src/index.js'; +import type { ContentBlock } from '@animalabs/membrane'; +import type { ChannelRegistry } from '../src/mcpl/channel-registry.js'; +import { HeldStream, MockMembrane, createMockResponse } from './helpers/mock-membrane.js'; + +function internals(framework: AgentFramework) { + return framework as unknown as { + initializeMcpl(servers: []): Promise; + channelRegistry: ChannelRegistry; + activeTurnTokens: Map; + deferredMessages: Array<{ forAgent?: string; metadata?: Record }>; + pendingRequests: unknown[]; + pendingAssistantBlocks: Map; + mcplServerRegistry: McplServerRegistry; + lastConversationSweep: number; + sweepExpiredConversations(): void; + quiesced: boolean; + deferredWritesPersisted: boolean; + unackedDeferredWrites: unknown[]; + flushDeferredWrites(label: string): Promise; + landedDeferredWriteIds(ids: Set, scanFrom: Map): Set; + injectScriptWake(record: { id: string; agentName: string }, envelope: string): void; + closingConversationAgents: Set; + activeStreams: Map>; + recordChannelAutoOpenNotice(agentName: string | undefined, channels: Array<{ channelId: string; label?: string }>, cause: 'subscription-policy' | 'opened-by-delivery' | 'opened-by-reply'): void; + addMessage(participant: string, content: ContentBlock[], metadata?: Record, opts?: { forAgent?: string }): string; + }; +} + +/** A HeldStream that says when the framework has taken its events: how many it + * yielded, and when it was left, by a consumer's break or at its end. */ +class WatchedHeldStream extends HeldStream { + yielded = 0; + private finish!: () => void; + readonly finished = new Promise((resolve) => { this.finish = resolve; }); + override async *[Symbol.asyncIterator](): ReturnType { + try { + for await (const event of super[Symbol.asyncIterator]()) { + this.yielded++; + yield event; + } + } finally { + this.finish(); + } + } +} + +/** A tool whose call makes `speaker`'s reply fail to reach a channel while its + * turn is alive, and a tool that does nothing; either can quiesce the host + * from inside the turn. */ +class SenderModule implements Module { + readonly name = 'sender'; + framework: AgentFramework | null = null; + speaker = 'worker'; + quiesceOnCall = false; + endTurnOnPost = false; + async start(_ctx: ModuleContext): Promise {} + async stop(): Promise {} + getTools(): ToolDefinition[] { + return [ + { name: 'post', description: 'Post a reply', inputSchema: { type: 'object', properties: {} } }, + { name: 'noop', description: 'Do nothing', inputSchema: { type: 'object', properties: {} } }, + ]; + } + async handleToolCall(call: ToolCall): Promise { + if (call.name.endsWith('post')) { + await internals(this.framework!).channelRegistry.routeSpeech(this.speaker, 'mid-turn reply', 'missing-channel'); + } + if (this.quiesceOnCall) internals(this.framework!).quiesced = true; + if (this.endTurnOnPost && call.name.endsWith('post')) return { success: true, data: { ok: true }, endTurn: true }; + return { success: true, data: { ok: true } }; + } + async onProcess(): Promise { return {}; } +} + +const SEND_FAILED = '[discord-send-failed]'; +const BOUNCED = 'was not delivered —'; +const count = (messages: Array<{ content: unknown }>, needle: string) => + messages.filter((m) => JSON.stringify(m.content).includes(needle)).length; + +describe('notices about an agent\'s own action', () => { + let tempDir: string; + let framework: AgentFramework; + let membrane: MockMembrane; + + beforeEach(async () => { + tempDir = mkdtempSync(join(tmpdir(), 'route-failure-targeting-')); + membrane = new MockMembrane(); + framework = await AgentFramework.create({ + storePath: join(tempDir, 'test.chronicle'), + membrane: membrane.asMembrane(), + agents: [ + { name: 'primary', model: 'test-model', systemPrompt: 'Primary.', proseRouting: 'locus' }, + { name: 'secondary', model: 'test-model', systemPrompt: 'Secondary.' }, + ], + modules: [], + conversations: { templateAgent: 'primary' }, + }); + // Install the real framework callback and registry without a live connector. + await internals(framework).initializeMcpl([]); + }); + + afterEach(async () => { + internals(framework).activeTurnTokens.clear(); + internals(framework).pendingAssistantBlocks.clear(); + internals(framework).quiesced = false; + await framework.stop(); + rmSync(tempDir, { recursive: true, force: true }); + }); + + /** A real conversation fork: an incoming DM binds one, and its reply fails (no such channel). */ + async function spawnFork(reply = 'Reply from the fork.') { + membrane.pushResponse(createMockResponse([{ type: 'text', text: reply }])); + framework.pushEvent({ + type: 'mcpl:channel-incoming', + serverId: 'srv', + channelId: 'missing-channel', + messageId: 'incoming-1', + author: { id: 'user-1', name: 'User' }, + content: [{ type: 'text', text: 'Hello fork.' }], + timestamp: new Date().toISOString(), + metadata: { channel_type: 'im' }, + triggerInference: true, + } as unknown as ProcessEvent); + await framework.runUntilIdle(); + const binding = framework.getConversationRouter()!.getBinding('missing-channel'); + assert.ok(binding, 'the incoming DM must spawn a real conversation fork'); + return framework.getAgent(binding.agentName)!; + } + + async function runEphemeral(proseRouting: 'locus' | 'explicit', speech: string) { + const created = await framework.createEphemeralAgent({ + name: 'worker', model: 'test-model', systemPrompt: 'Do the task.', allowedTools: 'all', proseRouting, + }); + created.contextManager.addMessage('user', [{ type: 'text', text: 'Run once.' }]); + membrane.pushResponse(createMockResponse([{ type: 'text', text: speech }])); + const run = framework.runEphemeralToCompletion(created.agent, created.contextManager); + framework.start(); + await run; + await framework.runUntilIdle(); + // The run is released; read what it left in Chronicle. + const archive = await ContextManager.open({ + store: framework.getStore(), namespace: 'subagent/worker', isolate: true, strategy: new PassthroughStrategy(), + }); + return (await archive.compile()).messages; + } + + describe('residents share one message slot', () => { + it('addresses a secondary resident\'s notice to it, in the slot every resident reads', async (t) => { + const primary = t.mock.method(framework.getAgent('primary')!.getContextManager(), 'addMessage'); + const secondary = t.mock.method(framework.getAgent('secondary')!.getContextManager(), 'addMessage'); + const result = await internals(framework).channelRegistry.routeSpeech('secondary', 'hello', 'missing-channel'); + + assert.equal(result, null); + assert.equal(primary.mock.callCount(), 0); + assert.equal(secondary.mock.callCount(), 1, 'written through the speaker\'s own window'); + const [participant, content, metadata] = secondary.mock.calls[0]!.arguments; + assert.equal(participant, 'user'); + assert.match(JSON.stringify(content), /\[discord-send-failed\] Your previous reply \(5 chars\) could not be delivered to missing-channel/); + assert.equal(metadata?.system, true); + assert.equal(metadata?.kind, 'discord-send-failed'); + assert.equal(metadata?.channelId, 'missing-channel'); + assert.match(String(metadata?.reason), /no registered channel/); + assert.equal(metadata?.textLen, 5); + t.mock.restoreAll(); + const { messages } = await framework.getAgent('secondary')!.getContextManager().compile(); + assert.equal(count(messages, SEND_FAILED), 1, 'the speaking resident reads it'); + assert.equal(internals(framework).pendingRequests.length, 0, 'a failure notice must not request inference'); + assert.equal(membrane.calls.length, 0); + }); + + it('keeps primary failures in the primary context manager', async (t) => { + const primary = t.mock.method(framework.getAgent('primary')!.getContextManager(), 'addMessage'); + const secondary = t.mock.method(framework.getAgent('secondary')!.getContextManager(), 'addMessage'); + + await internals(framework).channelRegistry.routeSpeech('primary', 'hello', null); + + assert.equal(primary.mock.callCount(), 1); + assert.equal(secondary.mock.callCount(), 0); + assert.equal(primary.mock.calls[0]!.arguments[2]?.kind, 'discord-send-failed'); + assert.match(JSON.stringify(primary.mock.calls[0]!.arguments[1]), /\[send-undeliverable\] Your previous reply \(5 chars\) had no channel to go to/); + assert.equal(internals(framework).pendingRequests.length, 0); + }); + + it('waits for the speaking resident\'s own live turn', async () => { + const state = internals(framework); + state.activeTurnTokens.set('secondary', 1); + + await state.channelRegistry.routeSpeech('secondary', 'hello', null); + + assert.equal(state.deferredMessages.length, 1, 'the speaker is mid-turn'); + assert.equal(state.deferredMessages[0]!.forAgent, 'secondary'); + const { messages } = await framework.getAgent('secondary')!.getContextManager().compile(); + assert.equal(count(messages, SEND_FAILED), 0); + }); + + for (const speaker of ['secondary', 'primary'] as const) { + it(`waits for ${speaker}'s own tool cycle before writing its notice`, async () => { + const state = internals(framework); + state.pendingAssistantBlocks.set(speaker, []); + + await state.channelRegistry.routeSpeech(speaker, 'hello', null); + + assert.equal(state.deferredMessages.length, 1, 'a write now would split its tool_use from its result'); + assert.equal(state.deferredMessages[0]!.forAgent, speaker); + const { messages } = await framework.getAgent(speaker)!.getContextManager().compile(); + assert.equal(count(messages, SEND_FAILED), 0); + }); + } + }); + + describe('a fork or an ephemeral run reads only its own window', () => { + it('delivers a failed fork reply notice to the fork after its turn, without waking it again', async () => { + const reply = 'Reply from the fork.'; + const fork = await spawnFork(reply); + const { messages: forkMessages } = await fork.getContextManager().compile(); + const { messages: primaryMessages } = await framework.getAgent('primary')!.getContextManager().compile(); + const notices = forkMessages.filter((m) => JSON.stringify(m.content).includes(SEND_FAILED)); + assert.equal(notices.length, 1, 'the fork must see its own failure notice exactly once'); + assert.equal(count(primaryMessages, SEND_FAILED), 0); + const replyIndex = forkMessages.findIndex((m) => m.participant === fork.name && JSON.stringify(m.content).includes(reply)); + assert.ok(replyIndex >= 0, 'the attempted reply is still archived'); + assert.ok(forkMessages.indexOf(notices[0]!) > replyIndex, 'failure notice follows the attempted reply'); + assert.equal(internals(framework).deferredMessages.length, 0, 'the completed turn flushes its notice'); + assert.equal(internals(framework).pendingRequests.length, 0); + assert.equal(membrane.calls.length, 1, 'failure notice is context only, not another wake'); + }); + + it('defers a fork\'s notice against the fork\'s own live turn', async (t) => { + const fork = await spawnFork(); + const state = internals(framework); + const primary = t.mock.method(framework.getAgent('primary')!.getContextManager(), 'addMessage'); + const forkCm = t.mock.method(fork.getContextManager(), 'addMessage'); + state.activeTurnTokens.set(fork.name, 99); + + await state.channelRegistry.routeSpeech(fork.name, 'hello', null); + + assert.equal(primary.mock.callCount(), 0); + assert.equal(forkCm.mock.callCount(), 0, 'the notice waits for the fork\'s safe boundary'); + assert.equal(state.deferredMessages.length, 1); + assert.equal(state.deferredMessages[0]!.forAgent, fork.name); + assert.equal(state.deferredMessages[0]!.metadata?.kind, 'discord-send-failed'); + assert.equal(state.pendingRequests.length, 0); + }); + + it('tells the primary, by name, when the speaker ended before its delivery failed', async (t) => { + const primary = t.mock.method(framework.getAgent('primary')!.getContextManager(), 'addMessage'); + const secondary = t.mock.method(framework.getAgent('secondary')!.getContextManager(), 'addMessage'); + + await internals(framework).channelRegistry.routeSpeech('released-conversation', 'hello', 'missing-channel'); + + assert.equal(secondary.mock.callCount(), 0); + assert.equal(primary.mock.callCount(), 1); + const [, content, metadata] = primary.mock.calls[0]!.arguments; + assert.match(JSON.stringify(content), + /\[discord-send-failed\] A reply by released-conversation \(5 chars\) could not be delivered to missing-channel \(no registered channel[^)]*\)\. Nothing was posted\. released-conversation has ended\./); + assert.equal(metadata?.speaker, 'released-conversation'); + assert.equal(metadata?.textLen, 5); + assert.equal(internals(framework).pendingRequests.length, 0, 'context only: the primary is not woken'); + }); + + it('tells the primary, by name, in the speaker\'s notice\'s words when an ended speaker\'s delivery was not confirmed', async (t) => { + const primary = t.mock.method(framework.getAgent('primary')!.getContextManager(), 'addMessage'); + const registry = internals(framework).channelRegistry as unknown as { + onRouteFailure(info: { conversationId: string; channelId: string | null; reason: string; textLen: number; outcome?: 'failed' | 'unknown'; detail?: string }): void; + }; + + registry.onRouteFailure({ + conversationId: 'released-conversation', channelId: 'missing-channel', reason: 'connection lost', textLen: 5, + outcome: 'unknown', detail: 'status 503', + }); + + assert.equal(primary.mock.callCount(), 1); + const [, content, metadata] = primary.mock.calls[0]!.arguments; + assert.equal((content as Array<{ text: string }>)[0]!.text, + '[discord-send-failed] Delivery of a reply by released-conversation (5 chars) to missing-channel was not confirmed (connection lost). ' + + 'It may or may not have been posted, so sending it again could post it twice. released-conversation has ended. Connector detail: status 503'); + assert.equal(metadata?.speaker, 'released-conversation'); + assert.equal(metadata?.outcome, 'unknown'); + assert.equal(metadata?.detail, 'status 503'); + assert.equal(internals(framework).pendingRequests.length, 0, 'context only: the primary is not woken'); + }); + + it('tells no one, logged, when an ended speaker\'s reply had no channel to reach', async (t) => { + const primary = t.mock.method(framework.getAgent('primary')!.getContextManager(), 'addMessage'); + const errors: string[] = []; + t.mock.method(console, 'error', (...args: unknown[]) => { errors.push(args.map(String).join(' ')); }); + + await internals(framework).channelRegistry.routeSpeech('released-conversation', 'hello', null); + + assert.equal(primary.mock.callCount(), 0); + assert.equal(internals(framework).deferredMessages.length, 0); + assert.ok(errors.some((e) => e.includes('[route-failure] released-conversation has ended, and its reply had no channel to reach'))); + }); + + it('tells no one, logged, when an ephemeral run\'s final reply had no channel to reach, before or after its release', async (t) => { + const errors: string[] = []; + t.mock.method(console, 'error', (...args: unknown[]) => { errors.push(args.map(String).join(' ')); }); + const speech = 'Worker speech with no channel to go to.'; + + const archived = await runEphemeral('locus', speech); + + // The run settles before its final reply is routed, so its release and + // the reply's failure can come in either order: the notice lands in its + // window and is read at its release, or the failure finds it gone. Either + // way, a reply that had no channel to reach is passed on to no one. + assert.ok(archived.some((m) => m.participant === 'worker' && JSON.stringify(m.content).includes(speech)), 'the run\'s reply is archived'); + const { messages } = await framework.getAgent('primary')!.getContextManager().compile(); + assert.equal(count(messages, 'Your previous reply'), 0, 'the primary did not write that reply'); + assert.equal(count(messages, 'A reply by worker'), 0); + assert.ok(!errors.some((e) => /not a registered agent/.test(e))); + assert.ok(errors.some((e) => e.includes('[route-failure] worker has ended, and its reply had no channel to reach'))); + assert.equal(internals(framework).deferredMessages.length, 0); + }); + + it('passes an ephemeral run\'s failed delivery to a channel on to the primary, by name', async () => { + (internals(framework).channelRegistry as unknown as { defaultPublishChannel: string | null }).defaultPublishChannel = 'missing-channel'; + const speech = 'Worker speech for a channel that is not registered.'; + + await runEphemeral('locus', speech); + + // Whether the delivery fails before the run's release (its window's + // notice is passed on at release) or after it (the failure finds it + // gone), the primary is told once, by name. + const { messages } = await framework.getAgent('primary')!.getContextManager().compile(); + const named = messages.filter((m) => JSON.stringify(m.content).includes(SEND_FAILED)); + assert.equal(named.length, 1); + assert.match(JSON.stringify(named[0]!.content), /A reply by worker \(\d+ chars\) could not be delivered to missing-channel .*worker has ended/); + assert.equal(internals(framework).deferredMessages.length, 0); + }); + + it('still releases an ephemeral run when the primary cannot be told, logged', async (t) => { + (internals(framework).channelRegistry as unknown as { defaultPublishChannel: string | null }).defaultPublishChannel = 'missing-channel'; + const primaryCm = framework.getAgent('primary')!.getContextManager(); + const store = primaryCm.addMessage.bind(primaryCm); + t.mock.method(primaryCm, 'addMessage', (...args: Parameters) => { + if (JSON.stringify(args[1]).includes('A reply by worker')) throw new Error('store refused the write'); + return store(...args); + }); + const errors: string[] = []; + t.mock.method(console, 'error', (...args: unknown[]) => { errors.push(args.map(String).join(' ')); }); + + await runEphemeral('locus', 'Worker speech for a channel that is not registered.'); + + assert.equal(framework.getAgent('worker'), null, 'the run is released'); + assert.ok(errors.some((e) => e.includes('[route-failure] failed to tell the primary that worker\'s reply to missing-channel failed'))); + }); + + it('tells no one, logged, when there is no primary to tell', async (t) => { + const dir = mkdtempSync(join(tmpdir(), 'route-failure-no-primary-')); + const bare = await AgentFramework.create({ + storePath: join(dir, 'test.chronicle'), membrane: membrane.asMembrane(), agents: [], modules: [], + }); + try { + await internals(bare).initializeMcpl([]); + (internals(bare).channelRegistry as unknown as { defaultPublishChannel: string | null }).defaultPublishChannel = 'missing-channel'; + const errors: string[] = []; + t.mock.method(console, 'error', (...args: unknown[]) => { errors.push(args.map(String).join(' ')); }); + const created = await bare.createEphemeralAgent({ + name: 'worker', model: 'test-model', systemPrompt: 'Do the task.', allowedTools: 'all', proseRouting: 'locus', + }); + created.contextManager.addMessage('user', [{ type: 'text', text: 'Run once.' }]); + membrane.pushResponse(createMockResponse([{ type: 'text', text: 'Worker speech with nobody to tell.' }])); + const run = bare.runEphemeralToCompletion(created.agent, created.contextManager); + bare.start(); + await run; + await bare.runUntilIdle(); + + const archive = await ContextManager.open({ + store: bare.getStore(), namespace: 'subagent/worker', isolate: true, strategy: new PassthroughStrategy(), + }); + const { messages } = await archive.compile(); + assert.equal(count(messages, 'A reply by worker'), 0, 'the run is not told about itself'); + assert.ok(errors.some((e) => e.includes('[route-failure] worker has ended, and there is no primary to tell that its reply to missing-channel failed'))); + } finally { + await bare.stop(); + rmSync(dir, { recursive: true, force: true }); + } + }); + + it('keeps an ephemeral run\'s bounced prose notice to its own window, or drops it, logged, once the run is released', async (t) => { + const errors: string[] = []; + t.mock.method(console, 'error', (...args: unknown[]) => { errors.push(args.map(String).join(' ')); }); + + const archived = await runEphemeral('explicit', 'Worker prose with no destination.'); + + // The run settles before its final prose is routed, so the bounce can + // come before or after its release: its notice lands in its own window, + // or, once it's gone, is dropped, logged, and passed to no one. + const dropped = errors.some((e) => e.includes('[notice] worker is no longer registered, so its prose-bounce notice is dropped')); + assert.equal(count(archived, BOUNCED) + (dropped ? 1 : 0), 1, 'the notice lands in its own window or is dropped, once'); + const { messages } = await framework.getAgent('primary')!.getContextManager().compile(); + assert.equal(count(messages, BOUNCED), 0, 'the primary wrote no such text, and is not told of it'); + assert.ok(!errors.some((e) => e.includes('[route-failure]')), 'a bounce is not a failed reply'); + assert.equal(internals(framework).deferredMessages.length, 0); + }); + + it('drops, logged once, a draft notice for an agent no longer registered: its drafts are held, with no write or trace', async (t) => { + const state = internals(framework) as unknown as ReturnType & { + agents: Map; + routingOf(name: string): unknown; + holdProseDrafts(agent: unknown, texts: Array<{ text: string }>, reason: string, options: unknown): Array<{ id: string }>; + bounceProse(agent: unknown, text: string, reason: string, candidates: undefined, prefix: '>>', round: unknown, after?: { outcome?: 'unknown' }): void; + takeUnsent(agent: unknown, body: string, round: unknown, prefix: '>>'): unknown; + dismissLatestBounce(agent: unknown): void; + draftsSending: Set; + draftsInFlight: Set; + proseDrafts: { get(agent: string, id: string): unknown; recordOutcome(...args: unknown[]): void }; + deliverWithUnsent(agent: unknown, body: string, channelId: string, unsent: unknown): Promise; + }; + const agent = framework.getAgent('secondary')!; + const writes = [ + t.mock.method(framework.getAgent('primary')!.getContextManager(), 'addMessage'), + t.mock.method(agent.getContextManager(), 'addMessage'), + ]; + const errors: string[] = []; + t.mock.method(console, 'error', (...args: unknown[]) => { errors.push(args.map(String).join(' ')); }); + const traced: string[] = []; + const off = framework.onTrace((event) => { if (event.type === 'message:added') traced.push(JSON.stringify(event)); }); + state.agents.delete('secondary'); + let woken = -1; + try { + const round = { routing: state.routingOf('secondary'), notice: 'later' }; + // {{unsent}} with nothing held, and no words of its own. + assert.deepEqual(state.takeUnsent(agent, '{{unsent}}', round, '>>'), { refused: true }); + const held = state.holdProseDrafts(agent, [{ text: 'words held at once' }], 'bounced', { routing: state.routingOf('secondary'), notice: 'now' }); + assert.equal(held.length, 1, 'the draft is held as before'); + // >>skip_reply {{unsent}}, and {{unsent}}, while that draft is being sent. + state.draftsSending.add(`secondary\u0000${held[0]!.id}`); + state.draftsInFlight.add(`secondary\u0000${held[0]!.id}`); + state.dismissLatestBounce(agent); + assert.equal((state.takeUnsent(agent, '{{unsent}}', round, '>>') as { refused?: unknown }).refused, true); + // {{unsent}} whose send came back, but whose outcome could not be recorded. + t.mock.method(state.channelRegistry, 'resolveDestination', () => ({ destination: { serverId: 'srv', channelId: 'unrecorded-channel' } })); + t.mock.method(state.channelRegistry, 'deliverSpeech', async () => ({ status: 'delivered', at: Date.now() })); + t.mock.method(state.proseDrafts, 'recordOutcome', () => { throw new Error('journal refused'); }); + await state.deliverWithUnsent(agent, 'words resent', 'unrecorded-channel', state.proseDrafts.get('secondary', held[0]!.id)); + state.bounceProse(agent, 'words whose send threw', 'connection lost', undefined, '>>', round, { outcome: 'unknown' }); + // Prose with no destination: a bounce, whose notice would wake its speaker. + state.bounceProse(agent, 'words with no destination', 'no destination', undefined, '>>', round); + // Counted before the loop runs: running, it takes what is pending. + woken = (state.pendingRequests as Array<{ agentName?: string }>).filter((r) => r.agentName === 'secondary').length; + await framework.runUntilIdle(); + } finally { + state.agents.set('secondary', agent); + off(); + } + assert.equal(writes.reduce((n, w) => n + w.mock.callCount(), 0), 0, 'nothing is written to any window'); + assert.equal(state.deferredMessages.length, 0); + assert.deepEqual(traced, [], 'nothing is traced as added'); + assert.ok(errors.includes('[notice] secondary is no longer registered, so its held-draft notice is dropped')); + assert.ok(errors.includes('[notice] secondary is no longer registered, so its unknown-send notice is dropped')); + assert.ok(errors.includes('[notice] secondary is no longer registered, so its prose-bounce notice is dropped')); + assert.ok(errors.includes('[notice] secondary is no longer registered, so its empty-{{unsent}} notice is dropped')); + assert.ok(errors.includes('[notice] secondary is no longer registered, so its in-flight-dismiss notice is dropped')); + assert.ok(errors.includes('[notice] secondary is no longer registered, so its refused-{{unsent}} notice is dropped')); + assert.ok(errors.includes('[notice] secondary is no longer registered, so its unrecorded-outcome notice is dropped')); + assert.equal(woken, 0, 'no wake is requested for it'); + assert.ok(!errors.some((e) => /not a registered agent/.test(e)), 'not dropped by addMessage instead'); + }); + + it('drops, logged once, a background-script wake for an agent no longer registered: no write, trace or wake', async (t) => { + const primary = t.mock.method(framework.getAgent('primary')!.getContextManager(), 'addMessage'); + const errors: string[] = []; + t.mock.method(console, 'error', (...args: unknown[]) => { errors.push(args.map(String).join(' ')); }); + const traces: string[] = []; + framework.onTrace((event) => { if (event.type === 'message:added') traces.push(JSON.stringify(event)); }); + + internals(framework).injectScriptWake({ id: 's-2', agentName: 'released-run' }, '[background script s-2] Your background script DIED.'); + + assert.equal(primary.mock.callCount(), 0, 'the primary started no such script'); + assert.deepEqual(errors, ['[notice] released-run is no longer registered, so its background-script s-2 wake notice is dropped']); + assert.deepEqual(traces, [], 'nothing was queued or stored'); + assert.equal(internals(framework).deferredMessages.length, 0); + assert.equal(internals(framework).pendingRequests.length, 0, 'no wake for an agent that is gone'); + }); + + it('wakes a fork\'s background script into the fork\'s own window', async () => { + const fork = await spawnFork(); + const envelope = '[background script s-1] Your background script DIED 0m after start.'; + + internals(framework).injectScriptWake({ id: 's-1', agentName: fork.name }, envelope); + + const { messages: forkMessages } = await fork.getContextManager().compile(); + const { messages: primaryMessages } = await framework.getAgent('primary')!.getContextManager().compile(); + assert.equal(count(forkMessages, envelope), 1); + assert.equal(count(primaryMessages, envelope), 0); + }); + + /** Run the fallback reaper over a closing fork, as a minute's sweep would. */ + function reap(forkName: string) { + const state = internals(framework); + state.closingConversationAgents.add(forkName); + state.lastConversationSweep = 0; + state.sweepExpiredConversations(); + } + + it('tells the primary, once, of a failure notice a fork stored but never read, when the fork is released', async () => { + const reply = 'Reply from the fork.'; + const fork = await spawnFork(reply); + // The notice landed in the fork's window after its turn: no compile has read it. + + reap(fork.name); + + assert.equal(framework.getAgent(fork.name), null, 'released with no closure turn'); + const { messages } = await framework.getAgent('primary')!.getContextManager().compile(); + const told = messages.filter((m) => JSON.stringify(m.content).includes(SEND_FAILED)); + assert.equal(told.length, 1); + assert.match(JSON.stringify(told[0]!.content), + new RegExp(`A reply by ${fork.name} \\(${reply.length} chars\\) could not be delivered to missing-channel \\(no registered channel[^)]*\\)\\. Nothing was posted\\. ${fork.name} has ended\\.`)); + }); + + it('passes on only what a fork never read: a notice its later turn compiled stays its own', async () => { + const fork = await spawnFork('Reply from the fork.'); + const second = 'A second reply, which fails as well.'; + membrane.pushResponse(createMockResponse([{ type: 'text', text: second }])); + framework.pushEvent({ + type: 'mcpl:channel-incoming', + serverId: 'srv', + channelId: 'missing-channel', + messageId: 'incoming-2', + author: { id: 'user-1', name: 'User' }, + content: [{ type: 'text', text: 'Hello again.' }], + timestamp: new Date().toISOString(), + metadata: { channel_type: 'im' }, + triggerInference: true, + } as unknown as ProcessEvent); + await framework.runUntilIdle(); + assert.equal(membrane.calls.length, 2, 'the fork\'s second turn compiled its first notice'); + + reap(fork.name); + + const { messages } = await framework.getAgent('primary')!.getContextManager().compile(); + const told = messages.filter((m) => JSON.stringify(m.content).includes(SEND_FAILED)); + assert.equal(told.length, 1); + assert.match(JSON.stringify(told[0]!.content), new RegExp(`A reply by ${fork.name} \\(${second.length} chars\\)`)); + }); + + it('does not reap a closing fork whose turn holds its token before its stream registers', async () => { + const fork = await spawnFork(); + const state = internals(framework); + state.activeTurnTokens.set(fork.name, 7); + + reap(fork.name); + assert.equal(framework.getAgent(fork.name), fork, 'a turn mid-compile is busy'); + + state.activeTurnTokens.delete(fork.name); + reap(fork.name); + assert.equal(framework.getAgent(fork.name), null); + }); + + it('lands a queued ordinary write in a released fork\'s window, with its durable id, and acks it', async () => { + const fork = await spawnFork(); + const state = internals(framework); + state.deferredWritesPersisted = true; + state.activeTurnTokens.set(fork.name, 3); + state.addMessage('user', [{ type: 'text', text: 'queued for the fork' }], { kind: 'test-note' }, { forAgent: fork.name }); + assert.equal(state.deferredMessages.length, 1); + state.activeTurnTokens.delete(fork.name); + + reap(fork.name); + + assert.equal(framework.getAgent(fork.name), null); + assert.equal(state.deferredMessages.length, 0); + assert.equal(state.unackedDeferredWrites.length, 0, 'acked out of the recovery queue'); + const archive = await ContextManager.open({ + store: framework.getStore(), namespace: 'conversations/' + fork.name, isolate: true, strategy: new PassthroughStrategy(), + }); + const stored = archive.getAllMessages().find((m) => JSON.stringify(m.content).includes('queued for the fork')); + assert.ok(stored, 'it landed in the fork\'s own window'); + assert.equal(typeof (stored.metadata as { deferredWriteId?: unknown } | undefined)?.deferredWriteId, 'string'); + }); + + for (const [what, hold] of [ + ['its own tool cycle', (state: ReturnType, fork: string) => { state.pendingAssistantBlocks.set(fork, []); }], + ['its own stream', (state: ReturnType, fork: string) => { state.activeStreams.set(fork, new Promise(() => {})); }], + ] as const) { + it(`defers a fork's notice while only ${what} is alive`, async (t) => { + const fork = await spawnFork(); + const state = internals(framework); + const forkCm = t.mock.method(fork.getContextManager(), 'addMessage'); + hold(state, fork.name); + try { + await state.channelRegistry.routeSpeech(fork.name, 'hello', null); + + assert.equal(forkCm.mock.callCount(), 0); + assert.equal(state.deferredMessages.length, 1); + assert.equal(state.deferredMessages[0]!.forAgent, fork.name); + } finally { + // A stream that never settles would hold stop() in afterEach. + state.activeStreams.delete(fork.name); + } + }); + } + + it('lands a fork\'s notice while only another agent\'s tool cycle is pending', async (t) => { + const fork = await spawnFork(); + const state = internals(framework); + const forkCm = t.mock.method(fork.getContextManager(), 'addMessage'); + state.pendingAssistantBlocks.set('primary', []); + + await state.channelRegistry.routeSpeech(fork.name, 'hello', null); + + assert.equal(forkCm.mock.callCount(), 1, 'the fork\'s window has no cycle in flight'); + assert.equal(state.deferredMessages.length, 0); + }); + + it('tells the agent whose send opened a channel, in its own window; a policy open tells the primary', async (t) => { + const fork = await spawnFork(); + const state = internals(framework); + const primary = t.mock.method(framework.getAgent('primary')!.getContextManager(), 'addMessage'); + const forkCm = t.mock.method(fork.getContextManager(), 'addMessage'); + + state.recordChannelAutoOpenNotice(fork.name, [{ channelId: 'c-1', label: '#one' }], 'opened-by-reply'); + assert.equal(forkCm.mock.callCount(), 1); + assert.equal(primary.mock.callCount(), 0); + assert.match(JSON.stringify(forkCm.mock.calls[0]!.arguments[1]), /Now open: #one \(c-1\) — your own message into the closed channel engaged it/); + + state.recordChannelAutoOpenNotice(undefined, [{ channelId: 'c-2' }], 'subscription-policy'); + assert.equal(primary.mock.callCount(), 1, 'no agent caused a policy open'); + assert.equal(forkCm.mock.callCount(), 1); + }); + + it('tells the primary, by name, of a channel an ended agent\'s send opened, and the primary\'s channel_close closes it', async (t) => { + const state = internals(framework); + const channels = (state.channelRegistry as unknown as { channels: Map }).channels; + // The channel the ended run's reply opened, on a server that lets it close. + channels.set('srv:opened-1', { serverId: 'srv', descriptor: { id: 'opened-1', type: 'srv', label: '#opened' }, open: true } as never); + const closed: unknown[] = []; + const server = { + grant: new CapabilityGrant(new Set(['channels.publish', 'channels.lifecycle']), []), + sendChannelsTyping() {}, + async sendChannelsClose(params: unknown) { closed.push(params); }, + }; + t.mock.method(state.mcplServerRegistry, 'getServer', () => server as never); + + state.recordChannelAutoOpenNotice('released-run', [{ channelId: 'opened-1', label: '#opened' }], 'opened-by-delivery'); + + const told = framework.getAgent('primary')!.getContextManager().getAllMessages() + .filter((m) => JSON.stringify(m.content).includes('Now open: #opened (opened-1)')); + assert.equal(told.length, 1, 'the primary is told once'); + assert.match(JSON.stringify(told[0]!.content), /opened by released-run's reply; released-run has ended\. .*use channel_close/); + assert.equal((told[0]!.metadata as { speaker?: unknown } | undefined)?.speaker, 'released-run'); + assert.equal(state.deferredMessages.length, 0); + + // The remedy it names is the primary's to use. + (framework as unknown as { dispatchChannelToolCall(agent: string, call: { id: string; name: string; input: Record }): void }) + .dispatchChannelToolCall('primary', { id: 'close-1', name: 'channel_close', input: { channelId: 'opened-1' } }); + for (let i = 0; i < 50 && channels.get('srv:opened-1')!.open; i++) await new Promise((resolve) => setImmediate(resolve)); + assert.deepEqual(closed, [{ channelId: 'opened-1' }]); + assert.equal(channels.get('srv:opened-1')!.open, false, 'the channel the reply opened is closed'); + }); + + it('tells no one, logged, of a channel an ended agent\'s send opened when there is no primary', async (t) => { + const errors: string[] = []; + t.mock.method(console, 'error', (...args: unknown[]) => { errors.push(args.map(String).join(' ')); }); + const primary = t.mock.method(framework.getAgent('primary')!.getContextManager(), 'addMessage'); + const state = internals(framework) as unknown as ReturnType & { primaryAgentName: string | null }; + const name = state.primaryAgentName; + state.primaryAgentName = null; + try { + state.recordChannelAutoOpenNotice('released-run', [{ channelId: 'opened-2' }], 'opened-by-delivery'); + } finally { + state.primaryAgentName = name; + } + assert.equal(primary.mock.callCount(), 0); + assert.equal(state.deferredMessages.length, 0); + assert.deepEqual(errors, ['[channel] released-run has ended, and there is no primary to tell that opened-2 opened']); + }); + + it('passes a restored failure notice whose agent is gone to the primary, by name, and drops its other writes, logged', async (t) => { + const state = internals(framework); + const errors: string[] = []; + t.mock.method(console, 'error', (...args: unknown[]) => { errors.push(args.map(String).join(' ')); }); + (state.deferredMessages as unknown as Array>).push( + { + id: 'w-1', seq: 1, participant: 'user', forAgent: 'conversation-gone', + content: [{ type: 'text', text: '[discord-send-failed] Your previous reply (12 chars) could not be delivered.' }], + metadata: { system: true, kind: 'discord-send-failed', channelId: 'missing-channel', reason: 'no registered channel', textLen: 12 }, + }, + { + // Queued by an earlier build: no textLen, and no reason either. + id: 'w-2', seq: 2, participant: 'user', forAgent: 'conversation-older', + content: [{ type: 'text', text: '[discord-send-failed] Your previous reply could not be delivered.' }], + metadata: { system: true, kind: 'discord-send-failed', channelId: 'missing-channel' }, + }, + { + id: 'w-3', seq: 3, participant: 'user', forAgent: 'conversation-gone', + content: [{ type: 'text', text: 'an ordinary write' }], + }, + ); + + await state.flushDeferredWrites('boot-recovery'); + + const { messages } = await framework.getAgent('primary')!.getContextManager().compile(); + const told = messages.filter((m) => JSON.stringify(m.content).includes('has ended')).map((m) => JSON.stringify(m.content)); + assert.equal(told.length, 2); + assert.match(told[0]!, /A reply by conversation-gone \(12 chars\) could not be delivered to missing-channel \(no registered channel\)\. Nothing was posted\. conversation-gone has ended\./); + assert.match(told[1]!, /A reply by conversation-older could not be delivered to missing-channel\. Nothing was posted\. conversation-older has ended\./); + assert.ok(!told.some((t) => t.includes('undefined'))); + // Each is the entry itself, under its own id: boot's landed scan finds + // it, so a crash before the ack replays neither. + const stamped = framework.getAgent('primary')!.getContextManager().getAllMessages() + .filter((m) => JSON.stringify(m.content).includes('has ended')) + .map((m) => (m.metadata as { deferredWriteId?: unknown } | undefined)?.deferredWriteId); + assert.deepEqual(stamped, ['w-1', 'w-2']); + assert.deepEqual([...state.landedDeferredWriteIds(new Set(['w-1', 'w-2']), new Map())].sort(), ['w-1', 'w-2']); + assert.equal(count(messages, 'an ordinary write'), 0); + assert.ok(errors.some((e) => /"conversation-gone" is not a registered agent — message dropped \(participant=user\)/.test(e))); + assert.equal(state.deferredMessages.length, 0); + }); + + it('passes a restored failure notice on as the same queue entry while the primary is mid-turn: it waits under its own id', async () => { + const state = internals(framework); + (state.deferredMessages as unknown as Array>).push({ + id: 'w-9', seq: 9, participant: 'user', forAgent: 'conversation-gone', + content: [{ type: 'text', text: '[discord-send-failed] Your previous reply (12 chars) could not be delivered.' }], + metadata: { system: true, kind: 'discord-send-failed', channelId: 'missing-channel', reason: 'no registered channel', textLen: 12 }, + }); + state.activeTurnTokens.set('primary', 1); + + await state.flushDeferredWrites('resume'); + + assert.equal(state.deferredMessages.length, 1, 'one entry, not the original and a separate notice'); + const entry = state.deferredMessages[0] as unknown as { id: string; forAgent?: string; content: unknown; metadata?: Record }; + assert.equal(entry.id, 'w-9'); + assert.equal(entry.forAgent, undefined, 'now the primary\'s'); + assert.equal(entry.metadata?.speaker, 'conversation-gone'); + assert.match(JSON.stringify(entry.content), /A reply by conversation-gone \(12 chars\) could not be delivered/); + }); + + it('counts as read, across branches, only what an agent\'s window shares with the branch it read up to the point they part', async () => { + const state = internals(framework) as unknown as { unreadFailedReplies(agent: unknown): Array<{ tag?: unknown }> }; + const agent = framework.getAgent('secondary')!; + const cm = agent.getContextManager(); + const store = framework.getStore(); + const notice = (tag: string) => cm.getMessage(cm.addMessage('user', [{ type: 'text', text: `[discord-send-failed] ${tag}` }], + { system: true, kind: 'discord-send-failed', channelId: 'missing-channel', tag } as never))!; + const tags = () => state.unreadFailedReplies(agent).map((m) => m.tag); + + const read = notice('read'); + agent.markContextConsumed(); + // A branch made at the head shares everything the agent read. + store.createBranch('maintained', store.currentBranch().name); + await framework.switchBranch('maintained'); + notice('after the switch'); + assert.deepEqual(tags(), ['after the switch'], 'what it read stays read; what came after is unread'); + + // A branch made before what it read: past that point, nothing on it was + // read, though its numbers repeat the ones the agent read up to. + store.createBranchAt('rewound', 'maintained', read.sequence - 1); + await framework.switchBranch('rewound'); + const again = notice('on the rewound branch'); + assert.ok(again.sequence <= agent.getConsumedWatermark()!.sequence, 'numbered within what the watermark covers on its own branch'); + assert.deepEqual(tags(), ['on the rewound branch']); + }); + + it('counts as unread a notice stored on a branch after the agent read past the point where it parts from another', async () => { + const state = internals(framework) as unknown as { unreadFailedReplies(agent: unknown): Array<{ tag?: unknown }> }; + const agent = framework.getAgent('secondary')!; + const cm = agent.getContextManager(); + const store = framework.getStore(); + const notice = (tag: string) => cm.getMessage(cm.addMessage('user', [{ type: 'text', text: `[discord-send-failed] ${tag}` }], + { system: true, kind: 'discord-send-failed', channelId: 'missing-channel', tag } as never))!; + const tags = () => state.unreadFailedReplies(agent).map((m) => m.tag); + + notice('before the branch'); + const home = store.currentBranch().name; + store.createBranch('side', home); + await framework.switchBranch('side'); + notice('on the side branch'); + notice('also on the side branch'); + agent.markContextConsumed(); + // Back on the branch the side branch was made from: its numbers past the + // branch point are its own, and the agent read none of them. + await framework.switchBranch(home); + const late = notice('stored at home after the agent read on the side branch'); + assert.ok(late.sequence <= agent.getConsumedWatermark()!.sequence, 'numbered within what the watermark covers on the side branch'); + assert.deepEqual(tags(), ['stored at home after the agent read on the side branch']); + }); + + it('counts as unread what the current branch has from a branch between it and the one the agent read on', async () => { + const state = internals(framework) as unknown as { unreadFailedReplies(agent: unknown): Array<{ tag?: unknown }> }; + const agent = framework.getAgent('secondary')!; + const cm = agent.getContextManager(); + const store = framework.getStore(); + const notice = (tag: string) => cm.getMessage(cm.addMessage('user', [{ type: 'text', text: `[discord-send-failed] ${tag}` }], + { system: true, kind: 'discord-send-failed', channelId: 'missing-channel', tag } as never))!; + const tags = () => state.unreadFailedReplies(agent).map((m) => m.tag); + + notice('read at home'); + const home = store.currentBranch().name; + store.createBranch('between', home); + await framework.switchBranch('between'); + notice('on the branch between'); + notice('also on the branch between'); + store.createBranch('current', 'between'); + await framework.switchBranch(home); + for (const n of [1, 2, 3]) notice(`later at home ${n}`); + agent.markContextConsumed(); + // The current branch parts from home where the branch between left it, + // below its own branch point, so the agent read none of what it has + // from the branch between, though their numbers are within what the + // watermark covers at home. + await framework.switchBranch('current'); + assert.deepEqual(tags(), ['on the branch between', 'also on the branch between']); + }); + + it('counts as unread, across a branch made after the agent read, what was stored after its watermark', async () => { + const state = internals(framework) as unknown as { unreadFailedReplies(agent: unknown): Array<{ tag?: unknown }> }; + const agent = framework.getAgent('secondary')!; + const cm = agent.getContextManager(); + const store = framework.getStore(); + const notice = (tag: string) => cm.getMessage(cm.addMessage('user', [{ type: 'text', text: `[discord-send-failed] ${tag}` }], + { system: true, kind: 'discord-send-failed', channelId: 'missing-channel', tag } as never))!; + const tags = () => state.unreadFailedReplies(agent).map((m) => m.tag); + + notice('read'); + agent.markContextConsumed(); + notice('stored after the watermark'); + const home = store.currentBranch().name; + store.createBranch('later', home); + await framework.switchBranch('later'); + assert.deepEqual(tags(), ['stored after the watermark']); + }); + + for (const condition of ['ordinary', 'other-tool-cycle', 'quiesced', 'write-fails'] as const) { + it('handles a failed TTL-closure reply notice at disposal: ' + condition, async (t) => { + const state = internals(framework); + const channelId = 'closure-channel'; + let failPublishing = false; + const publisher = { + grant: new CapabilityGrant(new Set(['channels.publish']), []), + sendChannelsTyping() {}, + async sendChannelsPublish() { + if (failPublishing && condition === 'quiesced') state.quiesced = true; + return { delivered: !failPublishing }; + }, + }; + t.mock.method(state.mcplServerRegistry, 'getServer', () => publisher as never); + (state.channelRegistry as unknown as { channels: Map }).channels.set('srv:' + channelId, { + serverId: 'srv', descriptor: { id: channelId, type: 'srv', label: 'closure test' }, open: true, + }); + membrane.pushResponse(createMockResponse([{ type: 'text', text: 'Initial reply.' }])); + framework.pushEvent({ + type: 'mcpl:channel-incoming', serverId: 'srv', channelId, messageId: 'closure-incoming', + author: { id: 'user', name: 'User' }, + content: [{ type: 'text', text: 'Please help.' }], + timestamp: new Date().toISOString(), + metadata: { mentioned: true }, + triggerInference: true, + } as unknown as ProcessEvent); + await framework.runUntilIdle(); + const binding = framework.getConversationRouter()!.getBinding(channelId)!; + assert.ok(binding); + const fork = framework.getAgent(binding.agentName)!; + + failPublishing = true; + const finalReply = 'Final reply that the server cannot deliver.'; + membrane.pushResponse(createMockResponse([{ type: 'text', text: finalReply }])); + if (condition === 'other-tool-cycle') state.pendingAssistantBlocks.set('secondary', []); + // A host with a recovery queue: what lands at release is acked out of it. + if (condition === 'ordinary') state.deferredWritesPersisted = true; + if (condition === 'write-fails') { + const cm = fork.getContextManager(); + const store = cm.addMessage.bind(cm); + t.mock.method(cm, 'addMessage', (...args: Parameters) => { + if (JSON.stringify(args[1]).includes(SEND_FAILED)) throw new Error('store refused the write'); + return store(...args); + }); + } + const errors: string[] = []; + t.mock.method(console, 'error', (...args: unknown[]) => { errors.push(args.map(String).join(' ')); }); + binding.lastActivity = 0; + state.lastConversationSweep = 0; + state.sweepExpiredConversations(); + await framework.runUntilIdle(); + state.pendingAssistantBlocks.clear(); + + if (condition === 'quiesced') { + // The fork is disposed when its closure turn ends, quiesced or not. + // Only its window's release waits for resume, since a write could + // land mid-surgery: its notice is still queued, and the primary + // hasn't been told. + assert.equal(framework.getAgent(fork.name), null, 'disposed at once'); + assert.equal(state.deferredMessages.length, 1); + assert.equal(state.deferredMessages[0]!.forAgent, fork.name); + const waiting = await framework.getAgent('primary')!.getContextManager().compile(); + assert.equal(count(waiting.messages, SEND_FAILED), 0); + await framework.resume({ force: true }); + } + assert.equal(framework.getAgent(fork.name), null, 'the fork is disposed when its closure turn ends'); + // other-tool-cycle: a write waits only for its own target's tool + // cycle, so neither the fork's notice nor the primary's waits for the + // secondary's. + assert.equal(state.deferredMessages.length, 0, 'nothing is left queued'); + assert.equal(state.unackedDeferredWrites.length, 0, 'nor waiting in the recovery queue'); + // Open a fresh manager over the retained fork namespace, rather than + // relying only on the disposed Agent's in-memory context manager. + const archive = await ContextManager.open({ + store: framework.getStore(), namespace: 'conversations/' + fork.name, + isolate: true, strategy: new PassthroughStrategy(), + }); + const { messages } = await archive.compile(); + const replyIndex = messages.findIndex((m) => m.participant === fork.name && JSON.stringify(m.content).includes(finalReply)); + assert.ok(replyIndex >= 0, 'the failed closure reply remains archived'); + if (condition === 'write-fails') { + // A write the store refuses is logged; disposal still completes. + assert.equal(count(messages, SEND_FAILED), 0); + assert.ok(errors.some((e) => e.includes(`[deferred-flush] ${fork.name}: failed to store a queued write before release`))); + } else { + const noticeIndex = messages.findIndex((m) => JSON.stringify(m.content).includes(SEND_FAILED)); + assert.ok(noticeIndex > replyIndex, 'the archived failure notice follows the closure reply'); + assert.equal(count(messages, SEND_FAILED), 1); + assert.match(JSON.stringify(messages[noticeIndex]!.content), /Your previous reply.*delivered:false/); + // Every write of a drained entry carries its durable id (drainDeferredFor). + const stored = archive.getAllMessages().find((m) => JSON.stringify(m.content).includes(SEND_FAILED)); + const id = (stored?.metadata as { deferredWriteId?: unknown } | undefined)?.deferredWriteId; + assert.equal(typeof id, 'string'); + if (condition === 'quiesced') { + // Still queued for the fork at its release, the notice goes on to + // the primary as that entry, under the same id. (Otherwise it was + // stored in the fork's window at its turn's end, and is passed on + // from there, as a new write.) + const passed = framework.getAgent('primary')!.getContextManager().getAllMessages() + .find((m) => JSON.stringify(m.content).includes(`A reply by ${fork.name}`)); + assert.equal((passed?.metadata as { deferredWriteId?: unknown } | undefined)?.deferredWriteId, id); + } + } + // The fork never reads its notice again, so the primary is told, by name. + const primary = await framework.getAgent('primary')!.getContextManager().compile(); + const told = primary.messages.filter((m) => JSON.stringify(m.content).includes(SEND_FAILED)); + assert.equal(told.length, 1); + assert.match(JSON.stringify(told[0]!.content), + new RegExp(`A reply by ${fork.name} \\(${finalReply.length} chars\\) could not be delivered to #closure test \\(closure-channel\\) \\(.*delivered:false.*\\)\\. Nothing was posted\\. ${fork.name} has ended\\.`)); + assert.equal(membrane.calls.length, 2, 'only the initial and closure turns ran'); + assert.equal(state.pendingRequests.length, 0, 'a failed closure reply does not wake another turn'); + }); + } + }); + + describe('a run\'s notices at its tool boundaries, and its release', () => { + let runDir: string; + let runFramework: AgentFramework; + let runMembrane: MockMembrane; + let sender: SenderModule; + + beforeEach(async () => { + runDir = mkdtempSync(join(tmpdir(), 'route-failure-runs-')); + runMembrane = new MockMembrane(); + sender = new SenderModule(); + runFramework = await AgentFramework.create({ + storePath: join(runDir, 'test.chronicle'), + membrane: runMembrane.asMembrane(), + agents: [{ name: 'primary', model: 'test-model', systemPrompt: 'Primary.', proseRouting: 'locus' }], + modules: [sender], + }); + sender.framework = runFramework; + await internals(runFramework).initializeMcpl([]); + }); + + afterEach(async () => { + internals(runFramework).quiesced = false; + await runFramework.stop(); + rmSync(runDir, { recursive: true, force: true }); + }); + + /** Run 'worker' once through these rounds; every round but the last ends on its tool call. */ + async function run(rounds: ContentBlock[][]) { + const created = await runFramework.createEphemeralAgent({ + name: 'worker', model: 'test-model', systemPrompt: 'Do the task.', allowedTools: 'all', proseRouting: 'explicit', + }); + created.contextManager.addMessage('user', [{ type: 'text', text: 'Run once.' }]); + rounds.forEach((blocks, i) => runMembrane.pushResponse(createMockResponse(blocks, i < rounds.length - 1 ? 'tool_use' : undefined))); + const done = runFramework.runEphemeralToCompletion(created.agent, created.contextManager); + runFramework.start(); + await done; + await runFramework.runUntilIdle(); + } + + async function archived() { + const archive = await ContextManager.open({ + store: runFramework.getStore(), namespace: 'subagent/worker', isolate: true, strategy: new PassthroughStrategy(), + }); + return (await archive.compile()).messages; + } + + const injected = () => runMembrane.lastStream!.receivedToolResultOptions + .flatMap((o) => o?.injectedMessages ?? []) + .map((m) => JSON.stringify(m.content)); + + it('passes on nothing a run read live at a tool boundary, when its stream declares it carries injected messages', async () => { + runMembrane.supportsInjectedMessages = true; + await run([ + [{ type: 'tool_use', id: 'p1', name: 'sender--post', input: {} }], + [{ type: 'text', text: '>>skip_reply done' }], + ]); + + assert.ok(injected().some((c) => c.includes(SEND_FAILED)), 'the notice reached the run\'s live stream'); + assert.equal(count(await archived(), SEND_FAILED), 1, 'and its window'); + const { messages } = await runFramework.getAgent('primary')!.getContextManager().compile(); + assert.equal(count(messages, 'A reply by worker'), 0, 'it read it, so no one else is told'); + }); + + it('passes on, by name, a notice injected into a stream that doesn\'t declare it carries injected messages', async () => { + // Such a stream may be one that drops them (membrane's XML path), so + // the run can't be counted as having read it. + await run([ + [{ type: 'tool_use', id: 'p1', name: 'sender--post', input: {} }], + [{ type: 'text', text: '>>skip_reply done' }], + ]); + + assert.ok(injected().some((c) => c.includes(SEND_FAILED)), 'the notice was offered to the run\'s live stream'); + const { messages } = await runFramework.getAgent('primary')!.getContextManager().compile(); + assert.equal(count(messages, 'A reply by worker'), 1); + }); + + it('passes on, by name, a notice its boundary stored but never sent, when a tool ends the turn', async () => { + runMembrane.supportsInjectedMessages = true; + sender.endTurnOnPost = true; + await run([ + [{ type: 'tool_use', id: 'p1', name: 'sender--post', input: {} }], + // Never asked for: the post's result ends the turn. + [{ type: 'text', text: 'unused' }], + ]); + + assert.ok(!injected().some((c) => c.includes(SEND_FAILED)), 'the turn ended before the notice was sent'); + assert.equal(count(await archived(), SEND_FAILED), 1, 'it was stored in the run\'s window'); + const { messages } = await runFramework.getAgent('primary')!.getContextManager().compile(); + assert.equal(count(messages, 'A reply by worker'), 1, 'the run never read it, so the primary is told'); + }); + + it('injects a bounced prose notice at the next tool boundary, after the prose it bounced', async () => { + const prose = 'Worker prose with no destination.'; + await run([ + [{ type: 'text', text: prose }, { type: 'tool_use', id: 'n1', name: 'sender--noop', input: {} }], + [{ type: 'text', text: '>>skip_reply done' }], + ]); + + assert.ok(injected().some((c) => c.includes(BOUNCED)), 'the speaker hears it within the turn'); + const messages = await archived(); + const proseIndex = messages.findIndex((m) => m.participant === 'worker' && JSON.stringify(m.content).includes(prose)); + const noticeIndex = messages.findIndex((m) => JSON.stringify(m.content).includes(BOUNCED)); + assert.ok(proseIndex >= 0, 'the bounced prose is in its window'); + assert.ok(noticeIndex > proseIndex, 'the notice follows it there'); + assert.match(JSON.stringify(messages[noticeIndex]!.content), /\{\{unsent\}\}/, 'naming the clipboard that holds it'); + const { messages: primary } = await runFramework.getAgent('primary')!.getContextManager().compile(); + assert.equal(count(primary, BOUNCED), 0); + }); + + it('deregisters a run that ends while the host is quiesced at once, and releases its window at resume, landing what it left queued', async () => { + // A stream that can't carry injected messages leaves the post's failure + // notice queued for the run's turn end, and the run ends while the host + // is quiesced, so it is still queued for the window's release at resume. + runMembrane.supportsInjectedMessages = false; + sender.quiesceOnCall = true; + await run([ + [{ type: 'tool_use', id: 'p1', name: 'sender--post', input: {} }], + [{ type: 'text', text: '>>skip_reply done' }], + ]); + const state = internals(runFramework); + assert.equal(state.quiesced, true); + assert.equal(runFramework.getAgent('worker'), null, 'deregistered at once'); + assert.ok(state.deferredMessages.some((m) => m.forAgent === 'worker'), 'its failure notice is still queued'); + const waiting = await runFramework.getAgent('primary')!.getContextManager().compile(); + assert.equal(count(waiting.messages, 'A reply by worker'), 0, 'the primary isn\'t told before resume'); + + await runFramework.resume({ force: true }); + + assert.equal(state.deferredMessages.length, 0); + assert.equal(count(await archived(), SEND_FAILED), 1, 'what it left queued landed in its window'); + const { messages } = await runFramework.getAgent('primary')!.getContextManager().compile(); + assert.equal(count(messages, 'A reply by worker'), 1, 'and, still queued at the release, it goes on to the primary by name'); + }); + + it('passes a failed reply still queued when a run ends unquiesced on to the primary as the same queue entry', async () => { + let release!: () => void; + const released = new Promise((resolve) => { release = resolve; }); + runMembrane.streamYielding = (request) => { + runMembrane.calls.push(request); + return new HeldStream(released, []); + }; + const created = await runFramework.createEphemeralAgent({ + name: 'worker', model: 'test-model', systemPrompt: 'Do the task.', allowedTools: 'all', proseRouting: 'explicit', + }); + created.contextManager.addMessage('user', [{ type: 'text', text: 'Run once.' }]); + const state = internals(runFramework); + runFramework.start(); + const done = runFramework.runEphemeralToCompletion(created.agent, created.contextManager, { idleTimeoutMs: 300, idlePollMs: 50 }); + for (let i = 0; i < 400 && !state.activeStreams.has('worker'); i++) await new Promise((resolve) => setTimeout(resolve, 5)); + assert.ok(state.activeStreams.has('worker'), 'the run\'s stream started'); + // A failed reply queued for the run's turn end, which the run's stall + // then ends first. + (state.deferredMessages as unknown as Array>).push({ + id: 'w-late', seq: 1_000_000, participant: 'user', forAgent: 'worker', + content: [{ type: 'text', text: `${SEND_FAILED} Your previous reply (5 chars) could not be delivered to #x (missing-channel).` }], + metadata: { system: true, kind: 'discord-send-failed', channelId: 'missing-channel', reason: 'no registered channel', textLen: 5 }, + }); + await assert.rejects(done, /stalled/); + release(); + for (let i = 0; i < 20; i++) await new Promise((resolve) => setImmediate(resolve)); + + assert.equal(state.deferredMessages.length, 0); + const idOf = (m: { metadata?: unknown } | undefined) => (m?.metadata as { deferredWriteId?: unknown } | undefined)?.deferredWriteId; + const archive = await ContextManager.open({ + store: runFramework.getStore(), namespace: 'subagent/worker', isolate: true, strategy: new PassthroughStrategy(), + }); + const record = archive.getAllMessages().filter((m) => JSON.stringify(m.content).includes(SEND_FAILED)); + assert.equal(record.length, 1, 'it landed in the run\'s window'); + assert.equal(idOf(record[0]), 'w-late'); + const told = runFramework.getAgent('primary')!.getContextManager().getAllMessages() + .filter((m) => JSON.stringify(m.content).includes('A reply by worker')); + assert.equal(told.length, 1, 'the primary is told once'); + assert.equal(idOf(told[0]), 'w-late', 'as the same entry, under its id'); + }); + + it('leaves a run that ended during a surgery to its window\'s release: the surgery\'s flush doesn\'t drop its queue', async () => { + const created = await runFramework.createEphemeralAgent({ + name: 'worker', model: 'test-model', systemPrompt: 'Do the task.', allowedTools: 'all', proseRouting: 'explicit', + }); + created.contextManager.addMessage('user', [{ type: 'text', text: 'Run once.' }]); + const state = internals(runFramework) as unknown as ReturnType & { + reserveStoreForSurgery(verb: string, agentName: string): () => void; + }; + // Not started, so the run is registered and idle when the surgery begins. + const done = runFramework.runEphemeralToCompletion(created.agent, created.contextManager, { startupTimeoutMs: 150 }); + state.quiesced = true; + (state.deferredMessages as unknown as Array>).push({ + id: 'w-held', seq: 1_000_000, participant: 'user', forAgent: 'worker', + content: [{ type: 'text', text: 'Written for the run while the host was quiesced.' }], + }); + const releaseSurgery = state.reserveStoreForSurgery('test surgery', 'primary'); + await assert.rejects(done, /failed to start inference/); + assert.equal(runFramework.getAgent('worker'), null, 'the run ended during the surgery'); + + releaseSurgery(); + assert.deepEqual((state.deferredMessages as unknown as Array<{ id: string }>).map((m) => m.id), ['w-held'], 'its write waits for its window\'s release'); + + await runFramework.resume({ force: true }); + assert.equal(state.deferredMessages.length, 0); + assert.equal(count(await archived(), 'Written for the run while the host was quiesced.'), 1, 'and lands in its window at resume'); + }); + + it('passes a failed reply a run stored unread on to the primary\'s queue when the host stops before the release it waited for', async () => { + const created = await runFramework.createEphemeralAgent({ + name: 'worker', model: 'test-model', systemPrompt: 'Do the task.', allowedTools: 'all', proseRouting: 'explicit', + }); + created.contextManager.addMessage('user', [{ type: 'text', text: 'Run once.' }]); + // Stored in its window, and never read: the run never compiled. + created.contextManager.addMessage('user', + [{ type: 'text', text: `${SEND_FAILED} Your previous reply (5 chars) could not be delivered to #x (missing-channel).` }], + { system: true, kind: 'discord-send-failed', channelId: 'missing-channel', reason: 'no registered channel', textLen: 5 } as never); + const state = internals(runFramework) as unknown as ReturnType & { releasesAfterResume: Map }; + // Not started, so its startup watchdog ends it, while the host is quiesced. + const done = runFramework.runEphemeralToCompletion(created.agent, created.contextManager, { startupTimeoutMs: 150 }); + state.quiesced = true; + await assert.rejects(done, /failed to start inference/); + assert.ok(state.releasesAfterResume.has('worker'), 'its window\'s release waits for resume'); + + await runFramework.stop(); + + const passed = (state.deferredMessages as unknown as Array<{ forAgent?: string; content: ContentBlock[]; metadata?: Record }>) + .filter((m) => JSON.stringify(m.content).includes('A reply by worker')); + assert.equal(passed.length, 1, 'queued for the primary, for resume or boot'); + assert.equal(passed[0]!.forAgent, undefined); + assert.equal(passed[0]!.metadata?.speaker, 'worker'); + }); + + it('completes a release whose window can\'t be read for what it left unread: what was queued lands, logged', async (t) => { + const errors: string[] = []; + t.mock.method(console, 'error', (...args: unknown[]) => { errors.push(args.map(String).join(' ')); }); + const created = await runFramework.createEphemeralAgent({ + name: 'worker', model: 'test-model', systemPrompt: 'Do the task.', allowedTools: 'all', proseRouting: 'explicit', + }); + created.contextManager.addMessage('user', [{ type: 'text', text: 'Run once.' }]); + t.mock.method(created.agent, 'getConsumedWatermark', () => { throw new Error('watermark unreadable'); }); + const state = internals(runFramework); + (state.deferredMessages as unknown as Array>).push({ + id: 'w-q', seq: 1_000_000, participant: 'user', forAgent: 'worker', + content: [{ type: 'text', text: 'Queued for the run.' }], + }); + + await assert.rejects( + runFramework.runEphemeralToCompletion(created.agent, created.contextManager, { startupTimeoutMs: 150 }), + /failed to start inference/, + ); + + assert.equal(state.deferredMessages.length, 0); + assert.equal(count(await archived(), 'Queued for the run.'), 1, 'what was queued landed in its window'); + assert.ok(errors.some((e) => e.includes('[route-failure] worker: could not read its window for unread failed replies'))); + }); + + it('resumes past a run that ended while the host was quiesced: resume doesn\'t weigh its context', async (t) => { + sender.quiesceOnCall = true; + await run([ + [{ type: 'tool_use', id: 'n1', name: 'sender--noop', input: {} }], + [{ type: 'text', text: '>>skip_reply done' }], + ]); + const previewed: string[] = []; + // The run's context, were it weighed, would not fit; nothing else gives a verdict. + t.mock.method(runFramework as unknown as { previewAgentRuntimeSettings(name: string): unknown }, 'previewAgentRuntimeSettings', (name: string) => { + previewed.push(name); + return name === 'worker' + ? { available: true, effective: { fits: false, finalTokens: 2, budgetTokens: 1 } } + : { available: false, reason: 'no_preview_support' }; + }); + + await runFramework.resume(); + + assert.equal(internals(runFramework).quiesced, false, 'resumed'); + assert.ok(!previewed.includes('worker'), 'the ended run is not weighed'); + }); + + it('dispatches nothing an ended run\'s stream yields late while the host is quiesced', async (t) => { + let release!: () => void; + const released = new Promise((resolve) => { release = resolve; }); + let held: WatchedHeldStream | undefined; + runMembrane.streamYielding = (request) => { + runMembrane.calls.push(request); + // Yields nothing, cancel included, until released; then a tool call. + held = new WatchedHeldStream(released, [createMockResponse([{ type: 'tool_use', id: 'late-1', name: 'sender--noop', input: {} }], 'tool_use')]); + return held; + }; + const calls = t.mock.method(sender, 'handleToolCall'); + const created = await runFramework.createEphemeralAgent({ + name: 'worker', model: 'test-model', systemPrompt: 'Do the task.', allowedTools: 'all', proseRouting: 'explicit', + }); + created.contextManager.addMessage('user', [{ type: 'text', text: 'Run once.' }]); + const state = internals(runFramework); + runFramework.start(); + const done = runFramework.runEphemeralToCompletion(created.agent, created.contextManager, { idleTimeoutMs: 1000, idlePollMs: 50 }); + for (let i = 0; i < 400 && !state.activeStreams.has('worker'); i++) await new Promise((resolve) => setTimeout(resolve, 5)); + assert.ok(state.activeStreams.has('worker'), 'the run\'s stream started'); + state.quiesced = true; + await assert.rejects(done, /stalled/); + assert.equal(runFramework.getAgent('worker'), null, 'the ended run is deregistered at once, quiesced or not'); + assert.ok((state as unknown as { releasesAfterResume: Map }).releasesAfterResume.has('worker'), 'it ended while quiesced'); + + release(); + const left = await Promise.race([ + held!.finished.then(() => true), + new Promise((resolve) => setTimeout(() => resolve(false), 5_000)), + ]); + + assert.ok(left, 'its stream was left, before the 5 s bound'); + assert.equal(held!.yielded, 1, 'it yielded the late tool call first'); + assert.equal(calls.mock.callCount(), 0, 'its late tool call is not dispatched'); + await runFramework.resume({ force: true }); + }); + }); +});