From 88f34fe5a1960071710583e1e2a7441979a1cf96 Mon Sep 17 00:00:00 2001 From: Ada-1017 Date: Tue, 13 Oct 2026 09:36:31 -0400 Subject: [PATCH 01/30] chore(deps): context-manager ^0.14.0 Co-Authored-By: Claude Opus 5.5 --- changelog.d/cm-0.14.changed.md | 2 ++ package.json | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 changelog.d/cm-0.14.changed.md diff --git a/changelog.d/cm-0.14.changed.md b/changelog.d/cm-0.14.changed.md new file mode 100644 index 000000000..c3b5191f9 --- /dev/null +++ b/changelog.d/cm-0.14.changed.md @@ -0,0 +1,2 @@ +- Depend on `@animalabs/context-manager` ^0.14.0. A preview, or any compile made with `dryRun`, now leaves the context manager's estimate, its calibration and a pending kv-unified draft's marker indices as it found them. The context manager's state queries (`isReady`, `getPendingWork`, `getHotContextSettings`, `previewContext` and others) now throw while its strategy's initialization is pending or has failed, where they used to answer. A reposted sharded document now renders as two bodies, so a stored conversation holding one misses the prompt cache once after the upgrade. + - **Operators:** context-manager 0.14.0 stamps its persisted calibration multiplier with the pricing it was learned under. Two kinds of store can refuse every compile with `OverBudgetError` after the upgrade: one whose multiplier was last learned before context-manager 0.11.0, and one that holds XML-history carriers and had learned a multiplier below 1. Each needs its multiplier reset or re-stamped once, as the "Operators" entry in context-manager 0.14.0's changelog describes. diff --git a/package.json b/package.json index 4bf3dbcaa..5a51cc019 100644 --- a/package.json +++ b/package.json @@ -47,7 +47,7 @@ "license": "MIT", "dependencies": { "@animalabs/chronicle": "^0.4.0", - "@animalabs/context-manager": "^0.13.0", + "@animalabs/context-manager": "^0.14.0", "@animalabs/membrane": "^0.5.78", "chokidar": "^4.0.3", "discord.js": "^14.25.1", From eefc10cb112dbadf8efed642c073d5ede46ca188 Mon Sep 17 00:00:00 2001 From: Ada-1017 Date: Tue, 13 Oct 2026 11:25:37 -0400 Subject: [PATCH 02/30] fix(maintenance): an agent whose context manager refuses its readiness sits the pass out context-manager 0.14.0 refuses its strategy-state queries (isReady, getPendingWork and others) while its strategy's initialization is pending or has failed, where 0.13 answered. runQueuedMaintenance called isReady and getPendingWork for every agent outside any try, so one agent mid-switch (a surgery's awaited switchBranch or fork) or with a failed initialization failed the whole pass: every other agent's maintenance was skipped, and a framework-scope context-maintenance-failed alert fired. Its finally block and the maintenance snapshot threw the same way. contextReadiness() reads an agent's readiness or returns its refusal. A refusing agent sits the pass out, and the others run. The snapshot reports the refusal as refused; a pass record reports one met after the ticks as refusedAfter. A refusal inside the tick loop still lands in the agent's own catch (record.error and the agent-scope alert), as any tick failure. Tests: - a fenced agent (setStrategy with a strategy whose initialize throws) beside a healthy one: the healthy agent's pass runs to ready, the fenced agent is absent from the pass and refused in the snapshot, no alert; - a tick that fences its own manager: the pass record carries readyAfter false and refusedAfter, and no pass-scope alert fires. Co-Authored-By: Claude Opus 5.5 --- changelog.d/cm-0.14.changed.md | 1 + src/framework.ts | 44 +++++++++--- src/types/framework.ts | 11 +++ test/maintenance-timer.test.ts | 124 +++++++++++++++++++++++++++++++++ 4 files changed, 172 insertions(+), 8 deletions(-) diff --git a/changelog.d/cm-0.14.changed.md b/changelog.d/cm-0.14.changed.md index c3b5191f9..5ab1b792a 100644 --- a/changelog.d/cm-0.14.changed.md +++ b/changelog.d/cm-0.14.changed.md @@ -1,2 +1,3 @@ - Depend on `@animalabs/context-manager` ^0.14.0. A preview, or any compile made with `dryRun`, now leaves the context manager's estimate, its calibration and a pending kv-unified draft's marker indices as it found them. The context manager's state queries (`isReady`, `getPendingWork`, `getHotContextSettings`, `previewContext` and others) now throw while its strategy's initialization is pending or has failed, where they used to answer. A reposted sharded document now renders as two bodies, so a stored conversation holding one misses the prompt cache once after the upgrade. - **Operators:** context-manager 0.14.0 stamps its persisted calibration multiplier with the pricing it was learned under. Two kinds of store can refuse every compile with `OverBudgetError` after the upgrade: one whose multiplier was last learned before context-manager 0.11.0, and one that holds XML-history carriers and had learned a multiplier below 1. Each needs its multiplier reset or re-stamped once, as the "Operators" entry in context-manager 0.14.0's changelog describes. +- A context-maintenance pass now runs every other agent when one agent's context manager refuses to report its readiness, which context-manager 0.14.0 does while its strategy's initialization is pending (a branch switch in flight) or has failed. Without this, that one refusal would fail the whole pass. The maintenance snapshot reports the refusal on the agent as `refused`, and a pass's record reports one met after its ticks as `refusedAfter`. diff --git a/src/framework.ts b/src/framework.ts index a9890d076..abf87f18b 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -2515,8 +2515,13 @@ export class AgentFramework { const tools = this.compressionToolsForAgent(agent.name); cm.setToolDefinitions(tools); if (this.providerGateBlocked(agent.name)) return []; - if (cm.isReady()) return []; - const pending = cm.getPendingWork()?.description; + // The context manager refuses its state queries while its strategy's + // initialization is pending (a branch switch in flight) or has failed. + // Such an agent sits this pass out; the others still run, and the + // snapshot reports the refusal. + const state = this.contextReadiness(cm); + if ('refused' in state || state.ready) return []; + const pending = state.pending; const progress = this.contextProgress(cm); const record: ContextMaintenanceAgentRun = { agentName: agent.name, @@ -2568,10 +2573,15 @@ export class AgentFramework { }, ); } finally { - const pending = cm.getPendingWork()?.description; + const state = this.contextReadiness(cm); record.finishedAt = Date.now(); - record.readyAfter = cm.isReady(); - if (pending) record.pendingAfter = pending; + if ('refused' in state) { + record.readyAfter = false; + record.refusedAfter = state.refused; + } else { + record.readyAfter = state.ready; + if (state.pending) record.pendingAfter = state.pending; + } const progress = this.contextProgress(cm); if (progress) record.progressAfter = progress; } @@ -2585,6 +2595,23 @@ export class AgentFramework { } } + /** + * The context manager's readiness, or its refusal to say: it refuses its + * state queries while its strategy's initialization is pending or has + * failed, until a successful `switchBranch` or `setStrategy`. + */ + private contextReadiness( + cm: ContextManager, + ): { ready: boolean; pending?: string } | { refused: string } { + try { + const ready = cm.isReady(); + const pending = ready ? undefined : cm.getPendingWork()?.description; + return pending ? { ready, pending } : { ready }; + } catch (error) { + return { refused: error instanceof Error ? error.message : String(error) }; + } + } + private contextProgress(cm: ContextManager): Record | undefined { const strategy = cm.getStrategy() as { getProgressSnapshot?: () => unknown; @@ -3928,12 +3955,13 @@ export class AgentFramework { getContextMaintenanceSnapshot(): ContextMaintenanceSnapshot { const agents = [...this.agents.values()].map((agent) => { const cm = agent.getContextManager(); - const pending = cm.getPendingWork()?.description; + const state = this.contextReadiness(cm); const progress = this.contextProgress(cm); return { agentName: agent.name, - ready: cm.isReady(), - ...(pending ? { pending } : {}), + ...('refused' in state + ? { ready: false, refused: state.refused } + : { ready: state.ready, ...(state.pending ? { pending: state.pending } : {}) }), ...(progress ? { progress } : {}), }; }); diff --git a/src/types/framework.ts b/src/types/framework.ts index 9f5995c68..c43bded1c 100644 --- a/src/types/framework.ts +++ b/src/types/framework.ts @@ -302,6 +302,11 @@ export interface ContextMaintenanceAgentRun { pendingAfter?: string; progressBefore?: Record; progressAfter?: Record; + /** + * Set when the context manager refused to report its readiness after the + * pass: its strategy's initialization was pending or had failed. + */ + refusedAfter?: string; error?: string; } @@ -323,6 +328,12 @@ export interface ContextMaintenanceSnapshot { agentName: string; ready: boolean; pending?: string; + /** + * The context manager's refusal to report its readiness: its strategy's + * initialization is pending (a branch switch in flight) or has failed. + * Such an agent is skipped by maintenance passes until it answers again. + */ + refused?: string; progress?: Record; }>; } diff --git a/test/maintenance-timer.test.ts b/test/maintenance-timer.test.ts index 915abce97..8a73ed041 100644 --- a/test/maintenance-timer.test.ts +++ b/test/maintenance-timer.test.ts @@ -100,6 +100,51 @@ class FailingStrategy implements ContextStrategy { } } +/** A strategy whose initialization always fails. */ +class RefusingInitStrategy implements ContextStrategy { + readonly name = 'refusing-init-test'; + + async initialize(_ctx: StrategyContext): Promise { + throw new Error('strategy initialization failed for the test'); + } + + checkReadiness(): ReadinessState { + return { ready: false, description: 'not expected' }; + } + + async tick(_ctx: StrategyContext): Promise {} + + select( + _store: MessageStoreView, + _log: ContextLogView, + _budget: TokenBudget, + ): ContextEntry[] { + return []; + } +} + +/** A strategy whose tick fences its own manager with a failed initialization. */ +class FencingTickStrategy implements ContextStrategy { + readonly name = 'fencing-tick-test'; + manager?: { setStrategy(strategy: ContextStrategy): Promise }; + + checkReadiness(): ReadinessState { + return { ready: false, description: 'fencing tick queued' }; + } + + async tick(_ctx: StrategyContext): Promise { + await this.manager?.setStrategy(new RefusingInitStrategy()).catch(() => undefined); + } + + select( + _store: MessageStoreView, + _log: ContextLogView, + _budget: TokenBudget, + ): ContextEntry[] { + return []; + } +} + class ToolModule implements Module { readonly name = 'maintenance-tools'; @@ -337,4 +382,83 @@ describe('queued context maintenance timer', () => { rmSync(dir, { recursive: true, force: true }); } }); + + it('an agent whose context manager refuses sits the pass out, and the others still run', async () => { + const dir = mkdtempSync(join(tmpdir(), 'af-maintenance-refused-')); + const healthy = new QueuedStrategy(); + const framework = await AgentFramework.create({ + storePath: join(dir, 'store'), + membrane, + agents: [ + { name: 'fenced', model: 'test-model', systemPrompt: 'test', strategy: new QueuedStrategy() }, + { name: 'healthy', model: 'test-model', systemPrompt: 'test', strategy: healthy }, + ], + modules: [], + syncIntervalMs: 0, + maintenanceIntervalMs: 10, + }); + const alerts: Array<{ kind: string; agentName: string; data?: Record }> = []; + (framework as any).opsAlert = ( + kind: string, + agentName: string, + _message: string, + opts?: { data?: Record }, + ) => alerts.push({ kind, agentName, data: opts?.data }); + + // A failed strategy initialization fences the context manager: its state + // queries throw until a successful switchBranch or setStrategy. + const fenced = framework.getAgent('fenced')!.getContextManager(); + await assert.rejects(fenced.setStrategy(new RefusingInitStrategy()), /strategy initialization failed for the test/); + assert.throws(() => fenced.isReady(), /requires successful strategy initialization/); + + try { + framework.start(); + await waitFor(() => healthy.ticks >= 2); + await waitFor(() => framework.getContextMaintenanceSnapshot().history.length >= 1); + const snapshot = framework.getContextMaintenanceSnapshot(); + assert.deepEqual(snapshot.history[0].agents.map((run) => run.agentName), ['healthy']); + assert.equal(snapshot.history[0].agents[0].readyAfter, true); + const entry = snapshot.agents.find((agent) => agent.agentName === 'fenced'); + assert.equal(entry?.ready, false); + assert.match(entry?.refused ?? '', /requires successful strategy initialization/); + assert.deepEqual(alerts, []); + } finally { + await framework.stop(); + rmSync(dir, { recursive: true, force: true }); + } + }); + + it('a refusal that begins during the pass is recorded on the agent, not thrown out of the pass', async () => { + const dir = mkdtempSync(join(tmpdir(), 'af-maintenance-refused-after-')); + const strategy = new FencingTickStrategy(); + const framework = await AgentFramework.create({ + storePath: join(dir, 'store'), + membrane, + agents: [{ name: 'agent', model: 'test-model', systemPrompt: 'test', strategy }], + modules: [], + syncIntervalMs: 0, + maintenanceIntervalMs: 10, + }); + strategy.manager = framework.getAgent('agent')!.getContextManager(); + const alerts: Array<{ kind: string; agentName: string; data?: Record }> = []; + (framework as any).opsAlert = ( + kind: string, + agentName: string, + _message: string, + opts?: { data?: Record }, + ) => alerts.push({ kind, agentName, data: opts?.data }); + + try { + framework.start(); + await waitFor(() => framework.getContextMaintenanceSnapshot().history.length >= 1); + const run = framework.getContextMaintenanceSnapshot().history[0].agents[0]; + assert.equal(run.agentName, 'agent'); + assert.equal(run.readyAfter, false); + assert.match(run.refusedAfter ?? '', /requires successful strategy initialization/); + assert.equal(alerts.some((alert) => alert.data?.scope === 'pass'), false); + } finally { + await framework.stop(); + rmSync(dir, { recursive: true, force: true }); + } + }); }); From d475206f2ab8cd1b31f3f5e318ab6e98450badd4 Mon Sep 17 00:00:00 2001 From: Ada-1017 Date: Tue, 13 Oct 2026 11:39:19 -0400 Subject: [PATCH 03/30] fix(recovery): offline recovery reads a body as context-manager 0.14 does context-manager 0.14.0 reads a sharded message's body pairwise in store order: a message continues the body before it only when it carries the same bodyGroupId at a higher shardIndex. The chunker names a group by its text, so a document posted twice stores two copies under one group id, and 0.14's changelog asks hosts that group by bodyGroupId alone to read bodies through continuesBody or bodyBounds. The offline recovery validators grouped by bodyGroupId alone, so they refused two legitimate selections as splitting a body: an anchor at the end of one copy (the next message, the other copy's first shard, carries the same id), and a suppression of one copy that keeps the other. Both now use continuesBody: the anchor check asks whether the next message continues the anchor's body, and the removal check asks it of each adjacent pair across the scan. Tests (with two copies of one two-shard document): an anchor at the end of the first copy succeeds and discards the second; an anchor at the first copy's first shard is still refused; suppressing the second copy keeps the first. Co-Authored-By: Claude Opus 5.5 --- changelog.d/cm-0.14.changed.md | 1 + src/recovery/offline-branch.ts | 33 ++++---- test/offline-recovery-branch.test.ts | 122 +++++++++++++++++++++++++++ 3 files changed, 140 insertions(+), 16 deletions(-) diff --git a/changelog.d/cm-0.14.changed.md b/changelog.d/cm-0.14.changed.md index 5ab1b792a..0b650e13d 100644 --- a/changelog.d/cm-0.14.changed.md +++ b/changelog.d/cm-0.14.changed.md @@ -1,3 +1,4 @@ - Depend on `@animalabs/context-manager` ^0.14.0. A preview, or any compile made with `dryRun`, now leaves the context manager's estimate, its calibration and a pending kv-unified draft's marker indices as it found them. The context manager's state queries (`isReady`, `getPendingWork`, `getHotContextSettings`, `previewContext` and others) now throw while its strategy's initialization is pending or has failed, where they used to answer. A reposted sharded document now renders as two bodies, so a stored conversation holding one misses the prompt cache once after the upgrade. - **Operators:** context-manager 0.14.0 stamps its persisted calibration multiplier with the pricing it was learned under. Two kinds of store can refuse every compile with `OverBudgetError` after the upgrade: one whose multiplier was last learned before context-manager 0.11.0, and one that holds XML-history carriers and had learned a multiplier below 1. Each needs its multiplier reset or re-stamped once, as the "Operators" entry in context-manager 0.14.0's changelog describes. - A context-maintenance pass now runs every other agent when one agent's context manager refuses to report its readiness, which context-manager 0.14.0 does while its strategy's initialization is pending (a branch switch in flight) or has failed. Without this, that one refusal would fail the whole pass. The maintenance snapshot reports the refusal on the agent as `refused`, and a pass's record reports one met after its ticks as `refusedAfter`. +- Offline recovery (`createOfflineRecoveryBranch`) reads a sharded message's body as context-manager 0.14.0 does: a message continues the body before it only when it carries the same group id at a higher shard index. A document posted twice stores two copies under one group id, and an anchor at the end of one copy, or a suppression of one copy alone, is no longer refused as splitting the body. A selection inside one copy is still refused. diff --git a/src/recovery/offline-branch.ts b/src/recovery/offline-branch.ts index 6c2a202c8..04a0bed12 100644 --- a/src/recovery/offline-branch.ts +++ b/src/recovery/offline-branch.ts @@ -1,5 +1,5 @@ import { JsStore } from '@animalabs/chronicle'; -import { ContextManager } from '@animalabs/context-manager'; +import { ContextManager, continuesBody, type BodyShard } from '@animalabs/context-manager'; import { DEFAULT_DISCORD_AWARENESS_EMOJI, DiscordAwarenessOutbox, @@ -299,14 +299,15 @@ function validateAnchorBoundary( total: number, target: StoredWindowMessage, ): void { - const groupId = (target as StoredWindowMessage & { bodyGroupId?: string }).bodyGroupId; - if (!groupId || targetIndex + 1 >= total) return; + if (targetIndex + 1 >= total) return; const next = contextManager.getMessageWindow(targetIndex + 1, 1, { resolveBlobs: false, - }).messages[0] as (StoredWindowMessage & { bodyGroupId?: string }) | undefined; - if (next?.bodyGroupId === groupId) { + }).messages[0]; + // A body continues only into a later shard of the same ingestion: a + // reposted document shares its group id but starts a body of its own. + if (continuesBody(target as BodyShard, next as BodyShard | undefined)) { throw new Error( - `Recovery anchor ${String(target.id)} splits body group ${groupId}; choose its final shard.`, + `Recovery anchor ${String(target.id)} splits body group ${(target as BodyShard).bodyGroupId}; choose its final shard.`, ); } } @@ -321,7 +322,9 @@ function validateRemovalIntegrity( index >= interval.start && index <= interval.end); const toolUses = new Map(); const toolResults = new Map(); - const groupMembership = new Map(); + // The message before, in store order: a selection splits a body when a + // message continuing the one before it lands on the other side. + let previous: { message: BodyShard; removed: boolean } | null = null; for (let offset = 0; offset < total; offset += RECOVERY_SCAN_WINDOW) { const messages = contextManager.getMessageWindow( @@ -332,16 +335,14 @@ function validateRemovalIntegrity( for (let i = 0; i < messages.length; i++) { const index = offset + i; const isRemoved = removed(index); - const message = messages[i] as StoredWindowMessage & { bodyGroupId?: string }; - if (message.bodyGroupId) { - const prior = groupMembership.get(message.bodyGroupId); - if (prior !== undefined && prior !== isRemoved) { - throw new Error( - `Recovery selection splits body group ${message.bodyGroupId}; adjust the anchor or suppression boundary.`, - ); - } - groupMembership.set(message.bodyGroupId, isRemoved); + const message = messages[i]; + const shard = message as BodyShard; + if (previous && previous.removed !== isRemoved && continuesBody(previous.message, shard)) { + throw new Error( + `Recovery selection splits body group ${shard.bodyGroupId}; adjust the anchor or suppression boundary.`, + ); } + previous = { message: shard, removed: isRemoved }; for (const rawBlock of message.content) { const block = rawBlock as { type?: unknown; id?: unknown; toolUseId?: unknown }; if (block.type === 'tool_use' && typeof block.id === 'string') { diff --git a/test/offline-recovery-branch.test.ts b/test/offline-recovery-branch.test.ts index c514bc3d5..2991a8086 100644 --- a/test/offline-recovery-branch.test.ts +++ b/test/offline-recovery-branch.test.ts @@ -268,3 +268,125 @@ test('suppression rejects a range that splits a tool exchange', async () => { rmSync(dir, { recursive: true, force: true }); } }); + +// A document posted twice stores two runs of shards under one bodyGroupId +// (the chunker names a group by its text). Each run is a body of its own. +type ShardAppender = { + messageStore: { + append: ( + participant: string, + content: unknown[], + metadata?: unknown, + causedBy?: unknown, + extra?: { bodyGroupId: string; shardIndex: number }, + ) => { id: unknown }; + }; +}; + +function appendDocumentCopy(cm: ContextManager, discordMessageId: string): string[] { + return ['part one', 'part two'].map((text, shardIndex) => String( + (cm as unknown as ShardAppender).messageStore.append( + 'user', + [{ type: 'text', text }], + { serverId: 'discord', channelId: 'discord:g1:c1', messageId: discordMessageId }, + undefined, + { bodyGroupId: 'doc-group', shardIndex }, + ).id, + )); +} + +test('an anchor at the end of one copy of a reposted document does not split it', async () => { + const dir = mkdtempSync(join(tmpdir(), 'offline-recovery-repost-anchor-')); + const storePath = join(dir, 'agent.chronicle'); + try { + const store = JsStore.openOrCreate({ path: storePath }); + const cm = await ContextManager.open({ store, namespace: 'agents/cairn' }); + cm.addMessage('user', [{ type: 'text', text: 'before' }], { + serverId: 'discord', channelId: 'discord:g1:c1', messageId: 'm-before', + }); + const first = appendDocumentCopy(cm, 'm-doc-1'); + appendDocumentCopy(cm, 'm-doc-2'); + cm.close(); + store.close(); + + const result = await createOfflineRecoveryBranch({ + storePath, + agentName: 'cairn', + contextId: first[1], + branchName: 'recovery/cairn/repost-anchor', + }); + + assert.equal(result.messagesRemoved, 2); + assert.deepEqual(result.refs.map((ref) => ref.messageId), ['m-doc-2']); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); + +test('an anchor inside one copy of a document still splits its body', async () => { + const dir = mkdtempSync(join(tmpdir(), 'offline-recovery-split-anchor-')); + const storePath = join(dir, 'agent.chronicle'); + try { + const store = JsStore.openOrCreate({ path: storePath }); + const cm = await ContextManager.open({ store, namespace: 'agents/cairn' }); + cm.addMessage('user', [{ type: 'text', text: 'before' }], { + serverId: 'discord', channelId: 'discord:g1:c1', messageId: 'm-before', + }); + const first = appendDocumentCopy(cm, 'm-doc-1'); + appendDocumentCopy(cm, 'm-doc-2'); + cm.close(); + store.close(); + + await assert.rejects( + createOfflineRecoveryBranch({ + storePath, + agentName: 'cairn', + contextId: first[0], + branchName: 'recovery/cairn/split-anchor', + }), + /splits body group doc-group; choose its final shard/, + ); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); + +test('suppressing one copy of a reposted document keeps the other', async () => { + const dir = mkdtempSync(join(tmpdir(), 'offline-recovery-repost-suppress-')); + const storePath = join(dir, 'agent.chronicle'); + try { + const store = JsStore.openOrCreate({ path: storePath }); + const cm = await ContextManager.open({ store, namespace: 'agents/cairn' }); + cm.addMessage('user', [{ type: 'text', text: 'before' }], { + serverId: 'discord', channelId: 'discord:g1:c1', messageId: 'm-before', + }); + appendDocumentCopy(cm, 'm-doc-1'); + appendDocumentCopy(cm, 'm-doc-2'); + cm.addMessage('user', [{ type: 'text', text: 'current' }], { + serverId: 'discord', channelId: 'discord:g1:c1', messageId: 'm-current', + }); + cm.close(); + store.close(); + + const result = await createOfflineRecoveryBranch({ + storePath, + agentName: 'cairn', + messageId: 'm-current', + suppressMessageIds: ['m-doc-2'], + }); + + assert.equal(result.messagesRemoved, 0); + assert.equal(result.messagesSuppressed, 2); + assert.deepEqual(result.refs.map((ref) => ref.messageId), ['m-doc-2']); + const recoveredStore = JsStore.openOrCreate({ path: storePath }); + const recoveredCm = await ContextManager.open({ store: recoveredStore, namespace: 'agents/cairn' }); + assert.deepEqual( + recoveredCm.getAllMessages().map((message) => message.metadata?.messageId), + ['m-before', 'm-doc-1', 'm-doc-1', 'm-current'], + ); + recoveredCm.close(); + recoveredStore.close(); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); From 0ed307db75a366eaf27d1e84f8080d00129d52fb Mon Sep 17 00:00:00 2001 From: Ada-1017 Date: Tue, 13 Oct 2026 13:07:14 -0400 Subject: [PATCH 04/30] fix(maintenance, recovery): answer the internal review of the context-manager 0.14 adaptation Maintenance (src/framework.ts): - A failure to read an agent's readiness is that agent's failure, whatever threw: context-manager 0.14's initialization refusal, or a strategy's own refusal, such as the autobiographical one's for a branch it hasn't reloaded. The first round classified every throw as an initialization refusal and dropped the agent from the pass silently. Now the agent's run records the error and the agent gets its own context-maintenance-failed alert (scope agent; opsAlert's per-agent cooldown); the others run. - No pass starts while an operator surgery holds the store (surgeryHold), the tick loop starts no tick while it's set, and a failure met while it's set is logged without an alert: a surgery's switch makes the manager refuse, and that isn't a maintenance failure. - contextProgress is diagnostics: a strategy whose getProgressSnapshot throws (the autobiographical one does, in the same states as its readiness) reads as having no progress. Unguarded, it threw out of the pass's finally and out of getContextMaintenanceSnapshot. - The snapshot's agent field and the run record carry error; the first round's refused/refusedAfter are gone. resume(): one agent's preview read that throws (getRuntimeSettings and planRuntimeSettings reach getHotContextSettings, which context-manager 0.14 refuses while initialization is pending or failed) warns, naming the agent, and proceeds without a verdict for it, as for any preview that isn't available. Before, it threw out of resume() and left the host quiesced. Offline recovery: each suppressed interval is removed as a range. context-manager refuses to remove any message carrying a bodyGroupId alone, a one-shard body's too; removeRange refuses a bisection itself. Changelog: the Operators repair moves to a breaking fragment, scoped to who acts, with its migration and what's unchanged; resume and offline recovery go in a fixed fragment; the changed fragment names every way initialization is pending (switchBranch, fork, setStrategy, open). Tests: - maintenance-timer: a fenced agent fails alone, recorded and alerted, the other runs; a failure that begins during the pass is the agent's error and alert, not the pass's; a strategy whose readiness and progress both throw fails alone and the snapshot still answers; no pass starts while a surgery holds the store; a pass that ends unready records pendingAfter. - offline-recovery-branch: the anchor-repost test reopens the recovered store; a suppression that splits one copy is refused; a split across the 1,000-message scan window is refused; a one-shard copy suppressed beside its sibling. - host-quiesce: resume proceeds for an agent whose manager refuses. Co-Authored-By: Claude Opus 5.5 --- changelog.d/cm-0.14.breaking.md | 1 + changelog.d/cm-0.14.changed.md | 6 +- changelog.d/cm-0.14.fixed.md | 2 + src/framework.ts | 149 ++++++++++++++++--------- src/recovery/offline-branch.ts | 6 +- src/types/framework.ts | 13 +-- test/host-quiesce.test.ts | 22 ++++ test/maintenance-timer.test.ts | 161 +++++++++++++++++++++++++-- test/offline-recovery-branch.test.ts | 121 ++++++++++++++++++++ 9 files changed, 406 insertions(+), 75 deletions(-) create mode 100644 changelog.d/cm-0.14.breaking.md create mode 100644 changelog.d/cm-0.14.fixed.md diff --git a/changelog.d/cm-0.14.breaking.md b/changelog.d/cm-0.14.breaking.md new file mode 100644 index 000000000..61e76b840 --- /dev/null +++ b/changelog.d/cm-0.14.breaking.md @@ -0,0 +1 @@ +- **Operators of stores that learned an estimator calibration under context-manager 0.13 or earlier:** context-manager 0.14.0 stamps its persisted calibration multiplier with the pricing it was learned under. An unstamped multiplier is kept and re-stamped on upgrade, unless the store holds XML-history carriers, in which case it restarts from 1.0. Two kinds of store can then refuse every compile with `OverBudgetError` before inference: one whose multiplier was last learned before context-manager 0.11.0, and one that holds XML-history carriers and had learned a multiplier below 1. **Migration:** with the host stopped, reset the first kind's multiplier once, and re-stamp the second kind's checked value, as the **Operators** entry of context-manager 0.14.0's changelog describes. **Unchanged:** a store without XML-history carriers whose multiplier was learned on context-manager 0.11.0 or later keeps that multiplier, re-stamped. diff --git a/changelog.d/cm-0.14.changed.md b/changelog.d/cm-0.14.changed.md index 0b650e13d..5824ffd15 100644 --- a/changelog.d/cm-0.14.changed.md +++ b/changelog.d/cm-0.14.changed.md @@ -1,4 +1,2 @@ -- Depend on `@animalabs/context-manager` ^0.14.0. A preview, or any compile made with `dryRun`, now leaves the context manager's estimate, its calibration and a pending kv-unified draft's marker indices as it found them. The context manager's state queries (`isReady`, `getPendingWork`, `getHotContextSettings`, `previewContext` and others) now throw while its strategy's initialization is pending or has failed, where they used to answer. A reposted sharded document now renders as two bodies, so a stored conversation holding one misses the prompt cache once after the upgrade. - - **Operators:** context-manager 0.14.0 stamps its persisted calibration multiplier with the pricing it was learned under. Two kinds of store can refuse every compile with `OverBudgetError` after the upgrade: one whose multiplier was last learned before context-manager 0.11.0, and one that holds XML-history carriers and had learned a multiplier below 1. Each needs its multiplier reset or re-stamped once, as the "Operators" entry in context-manager 0.14.0's changelog describes. -- A context-maintenance pass now runs every other agent when one agent's context manager refuses to report its readiness, which context-manager 0.14.0 does while its strategy's initialization is pending (a branch switch in flight) or has failed. Without this, that one refusal would fail the whole pass. The maintenance snapshot reports the refusal on the agent as `refused`, and a pass's record reports one met after its ticks as `refusedAfter`. -- Offline recovery (`createOfflineRecoveryBranch`) reads a sharded message's body as context-manager 0.14.0 does: a message continues the body before it only when it carries the same group id at a higher shard index. A document posted twice stores two copies under one group id, and an anchor at the end of one copy, or a suppression of one copy alone, is no longer refused as splitting the body. A selection inside one copy is still refused. +- Depend on `@animalabs/context-manager` ^0.14.0. A preview, or any compile made with `dryRun`, now leaves the context manager's estimate, its calibration and a pending kv-unified draft's marker indices as it found them. The context manager refuses its state queries (`isReady`, `getPendingWork`, `getHotContextSettings`, `previewContext` and others), its compiles and its writes while its strategy's initialization is pending (inside `switchBranch`, `fork`, `setStrategy` or `ContextManager.open`) or has failed, where it used to answer; a successful `switchBranch` or `setStrategy`, or a restart, recovers it. A document posted twice is stored as two copies under one group id and is now read as two bodies everywhere, so a stored conversation holding one misses the prompt cache once after the upgrade. +- A context-maintenance pass no longer fails as a whole when one agent's readiness can't be read. context-manager 0.14.0 refuses its readiness queries while a strategy's initialization is pending or has failed, and a strategy can refuse on its own, as the autobiographical one does for a branch it hasn't reloaded. That agent's run now records the error, the agent gets its own `context-maintenance-failed` alert (`scope: 'agent'`), and the other agents run. Before, such a throw failed the pass for every agent, with one framework-scope alert. While an operator surgery holds the store, no pass starts and a running pass starts no new tick, and a failure met then is logged without an alert. `getContextMaintenanceSnapshot()` reports such an agent as `ready: false` with an `error`, and a strategy that can't give a progress snapshot reads as having none. diff --git a/changelog.d/cm-0.14.fixed.md b/changelog.d/cm-0.14.fixed.md new file mode 100644 index 000000000..042dfa4e9 --- /dev/null +++ b/changelog.d/cm-0.14.fixed.md @@ -0,0 +1,2 @@ +- `resume()` no longer throws when one agent's runtime settings can't be read, as when its context manager refuses while its strategy's initialization is pending or has failed. It warns, naming the agent and the error, and proceeds without a feasibility verdict for that agent, as it does for an agent whose strategy can't preview. +- Offline recovery (`createOfflineRecoveryBranch`) reads a sharded message's body as context-manager 0.14.0 does: a message continues the body before it only when it carries the same group id at a higher shard index. A document posted twice stores two copies under one group id, so an anchor at the end of one copy, or a suppression of one copy alone, is no longer refused. A selection inside one copy is still refused. An anchor at the end of a write that was cut short keeps that partial body and leaves out its retry, which context-manager 0.14.0 reads as a body of its own. Each suppressed interval is now removed as a range, so suppressing a one-shard copy no longer fails at removal. diff --git a/src/framework.ts b/src/framework.ts index abf87f18b..cda957516 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -2505,6 +2505,11 @@ export class AgentFramework { } private async runQueuedMaintenance(): Promise { + // A surgery holds the store while it switches or forks a branch, and a + // context manager mid-switch refuses its queries. Maintenance waits for + // the hold to end rather than report the surgery as a failure: no pass + // starts while it's up, and a running pass starts no new tick. + if (this.surgeryHold) return; const queued = [...this.agents.values()].flatMap((agent) => { try { agent.toolResultGuard.flushUnrecorded(); @@ -2515,27 +2520,26 @@ export class AgentFramework { const tools = this.compressionToolsForAgent(agent.name); cm.setToolDefinitions(tools); if (this.providerGateBlocked(agent.name)) return []; - // The context manager refuses its state queries while its strategy's - // initialization is pending (a branch switch in flight) or has failed. - // Such an agent sits this pass out; the others still run, and the - // snapshot reports the refusal. const state = this.contextReadiness(cm); - if ('refused' in state || state.ready) return []; - const pending = state.pending; + if ('ready' in state && state.ready) return []; const progress = this.contextProgress(cm); const record: ContextMaintenanceAgentRun = { agentName: agent.name, startedAt: Date.now(), ticks: 0, readyBefore: false, - ...(pending ? { pendingBefore: pending } : {}), + ...('pending' in state && state.pending ? { pendingBefore: state.pending } : {}), ...(progress ? { progressBefore: progress } : {}), }; - return [{ - agent, - cm, - record, - }]; + if ('error' in state) { + // Its readiness can't be read: that agent's failure, reported for it + // alone, while the others run. + record.error = state.error; + record.finishedAt = Date.now(); + this.reportMaintenanceFailure(agent.name, record); + return [{ agent, cm, record, failed: true }]; + } + return [{ agent, cm, record, failed: false }]; }); if (queued.length === 0) return; @@ -2547,11 +2551,11 @@ export class AgentFramework { this.currentMaintenanceRun = run; try { - await Promise.all(queued.map(async ({ agent, cm, record }) => { + await Promise.all(queued.filter((item) => !item.failed).map(async ({ agent, cm, record }) => { try { for ( let i = 0; - i < MAINTENANCE_TICKS_PER_PASS && this.running && !cm.isReady(); + i < MAINTENANCE_TICKS_PER_PASS && this.running && !this.surgeryHold && !cm.isReady(); i++ ) { await cm.tick(); @@ -2559,28 +2563,20 @@ export class AgentFramework { } } catch (error) { record.error = error instanceof Error ? error.message : String(error); - console.error(`[context-maintenance] agent=${agent.name} failed:`, record.error); - this.opsAlert( - 'context-maintenance-failed', - agent.name, - record.error, - { - data: { - scope: 'agent', - ...(record.pendingBefore ? { pending: record.pendingBefore } : {}), - ...(record.progressBefore ? { progress: record.progressBefore } : {}), - }, - }, - ); + this.reportMaintenanceFailure(agent.name, record); } finally { - const state = this.contextReadiness(cm); record.finishedAt = Date.now(); - if ('refused' in state) { - record.readyAfter = false; - record.refusedAfter = state.refused; - } else { - record.readyAfter = state.ready; - if (state.pending) record.pendingAfter = state.pending; + if (!this.surgeryHold) { + const state = this.contextReadiness(cm); + if ('error' in state) { + if (record.error === undefined) { + record.error = state.error; + this.reportMaintenanceFailure(agent.name, record); + } + } else { + record.readyAfter = state.ready; + if (state.pending) record.pendingAfter = state.pending; + } } const progress = this.contextProgress(cm); if (progress) record.progressAfter = progress; @@ -2596,31 +2592,66 @@ export class AgentFramework { } /** - * The context manager's readiness, or its refusal to say: it refuses its - * state queries while its strategy's initialization is pending or has - * failed, until a successful `switchBranch` or `setStrategy`. + * One agent's maintenance failure, recorded on its run (`record.error`): + * logged, and alerted for that agent unless a surgery holds the store, in + * which case the surgery is what ended it. + */ + private reportMaintenanceFailure(agentName: string, record: ContextMaintenanceAgentRun): void { + if (this.surgeryHold) { + console.error( + `[context-maintenance] agent=${agentName} stopped while a ${this.surgeryHold.verb} holds the store:`, + record.error, + ); + return; + } + console.error(`[context-maintenance] agent=${agentName} failed:`, record.error); + this.opsAlert( + 'context-maintenance-failed', + agentName, + record.error ?? 'unknown failure', + { + data: { + scope: 'agent', + ...(record.pendingBefore ? { pending: record.pendingBefore } : {}), + ...(record.progressBefore ? { progress: record.progressBefore } : {}), + }, + }, + ); + } + + /** + * An agent's context readiness, or the error that reading it threw. The + * context manager refuses while its strategy's initialization is pending + * or has failed (context-manager 0.14), and a strategy can refuse on its + * own, as the autobiographical one does after a store-level branch switch + * it hasn't reloaded for. Every such throw is that agent's failure. */ private contextReadiness( cm: ContextManager, - ): { ready: boolean; pending?: string } | { refused: string } { + ): { ready: boolean; pending?: string } | { error: string } { try { const ready = cm.isReady(); const pending = ready ? undefined : cm.getPendingWork()?.description; return pending ? { ready, pending } : { ready }; } catch (error) { - return { refused: error instanceof Error ? error.message : String(error) }; + return { error: error instanceof Error ? error.message : String(error) }; } } + /** The strategy's progress snapshot, for diagnostics: one it can't give reads as none. */ private contextProgress(cm: ContextManager): Record | undefined { - const strategy = cm.getStrategy() as { - getProgressSnapshot?: () => unknown; - }; - if (typeof strategy.getProgressSnapshot !== 'function') return undefined; - const progress = strategy.getProgressSnapshot(); - return progress && typeof progress === 'object' - ? progress as Record - : undefined; + try { + const strategy = cm.getStrategy() as { + getProgressSnapshot?: () => unknown; + }; + if (typeof strategy.getProgressSnapshot !== 'function') return undefined; + const progress = strategy.getProgressSnapshot(); + return progress && typeof progress === 'object' + ? progress as Record + : undefined; + } catch { + return undefined; + } } /** @@ -3558,11 +3589,23 @@ export class AgentFramework { } const failing: Array<{ agentName: string; preview: RuntimeSettingsPreview }> = []; for (const agentName of this.agents.keys()) { - let preview = this.previewAgentRuntimeSettings(agentName); - for (let attempt = 0; attempt < 8 && !preview.available && preview.reason === 'preview_in_flight'; attempt++) { - await new Promise((resolve) => setTimeout(resolve, 250)); - if (this.maintenancePass) await this.maintenancePass.catch(() => {}); + let preview: RuntimeSettingsPreview; + try { preview = this.previewAgentRuntimeSettings(agentName); + for (let attempt = 0; attempt < 8 && !preview.available && preview.reason === 'preview_in_flight'; attempt++) { + await new Promise((resolve) => setTimeout(resolve, 250)); + if (this.maintenancePass) await this.maintenancePass.catch(() => {}); + preview = this.previewAgentRuntimeSettings(agentName); + } + } catch (error) { + // The agent's settings can't be read (its context manager refuses while + // its strategy's initialization is pending or has failed): no verdict + // for it, as for any preview that isn't available. + console.warn( + `[host-mode] resume: feasibility preview unavailable for ${agentName} ` + + `(${error instanceof Error ? error.message : String(error)}) — proceeding without a verdict`, + ); + continue; } if (preview.available && preview.effective && !preview.effective.fits) { failing.push({ agentName, preview }); @@ -3959,8 +4002,8 @@ export class AgentFramework { const progress = this.contextProgress(cm); return { agentName: agent.name, - ...('refused' in state - ? { ready: false, refused: state.refused } + ...('error' in state + ? { ready: false, error: state.error } : { ready: state.ready, ...(state.pending ? { pending: state.pending } : {}) }), ...(progress ? { progress } : {}), }; diff --git a/src/recovery/offline-branch.ts b/src/recovery/offline-branch.ts index 04a0bed12..fbd315825 100644 --- a/src/recovery/offline-branch.ts +++ b/src/recovery/offline-branch.ts @@ -204,9 +204,11 @@ export async function createOfflineRecoveryBranch( await contextManager.switchBranch(createdBranch); // Work from newest to oldest so earlier redactions never perturb the // live positions used by later range endpoints. + // Each interval goes as a range, a single message included: removing one + // message alone is refused for any shard, a one-shard body's too, and + // the range removal refuses a bisected body itself. for (const interval of [...suppression.intervals].reverse()) { - if (interval.fromId === interval.toId) contextManager.removeMessage(interval.fromId); - else contextManager.removeMessages(interval.fromId, interval.toId); + contextManager.removeMessages(interval.fromId, interval.toId); } if (batch) outbox.activate(batch.id); } catch (error) { diff --git a/src/types/framework.ts b/src/types/framework.ts index c43bded1c..5725aee5d 100644 --- a/src/types/framework.ts +++ b/src/types/framework.ts @@ -303,10 +303,9 @@ export interface ContextMaintenanceAgentRun { progressBefore?: Record; progressAfter?: Record; /** - * Set when the context manager refused to report its readiness after the - * pass: its strategy's initialization was pending or had failed. + * The agent's failure in this pass: a tick that threw, or its readiness + * that couldn't be read, before the ticks or after them. */ - refusedAfter?: string; error?: string; } @@ -329,11 +328,11 @@ export interface ContextMaintenanceSnapshot { ready: boolean; pending?: string; /** - * The context manager's refusal to report its readiness: its strategy's - * initialization is pending (a branch switch in flight) or has failed. - * Such an agent is skipped by maintenance passes until it answers again. + * Why the agent's readiness can't be read: the context manager refuses + * while its strategy's initialization is pending or has failed, and a + * strategy can refuse on its own. `ready` is false then. */ - refused?: string; + error?: string; progress?: Record; }>; } diff --git a/test/host-quiesce.test.ts b/test/host-quiesce.test.ts index cae60190e..a4154e788 100644 --- a/test/host-quiesce.test.ts +++ b/test/host-quiesce.test.ts @@ -312,6 +312,28 @@ test('resume proceeds with a warn when the strategy cannot preview', async () => }); }); +test('resume proceeds without a verdict for an agent whose context manager refuses', async () => { + const membrane = new MockMembrane(); + await withFramework(membrane, async (framework) => { + await framework.quiesce(); + // A failed strategy initialization fences the context manager: its + // settings queries throw until a successful switchBranch or setStrategy. + const cm = framework.getAgent('agent')!.getContextManager(); + const failing = { + name: 'refusing-init-test', + async initialize() { throw new Error('strategy initialization failed for the test'); }, + checkReadiness: () => ({ ready: true }), + async tick() {}, + select: () => [], + } as unknown as ContextStrategy; + await assert.rejects(cm.setStrategy(failing), /strategy initialization failed for the test/); + assert.throws(() => cm.getHotContextSettings(), /requires successful strategy initialization/); + + const status = await framework.resume(); + assert.equal(status.quiesced, false); + }); +}); + test('maintenanceTick runs the real maintenance pass while quiesced', async () => { class QueuedStrategy implements ContextStrategy { readonly name = 'queued-test'; diff --git a/test/maintenance-timer.test.ts b/test/maintenance-timer.test.ts index 8a73ed041..15f78d3ee 100644 --- a/test/maintenance-timer.test.ts +++ b/test/maintenance-timer.test.ts @@ -123,6 +123,48 @@ class RefusingInitStrategy implements ContextStrategy { } } +/** A strategy that refuses on its own, as the autobiographical one does for a branch it hasn't reloaded. */ +class StaleStrategy implements ContextStrategy { + readonly name = 'stale-test'; + + checkReadiness(): ReadinessState { + throw new Error('StaleStrategy.checkReadiness requires reinitialization for the current branch generation'); + } + + getProgressSnapshot(): Record { + throw new Error('StaleStrategy.getProgressSnapshot requires reinitialization for the current branch generation'); + } + + async tick(_ctx: StrategyContext): Promise {} + + select( + _store: MessageStoreView, + _log: ContextLogView, + _budget: TokenBudget, + ): ContextEntry[] { + return []; + } +} + +/** A strategy with work pending that no tick finishes. */ +class NeverReadyStrategy implements ContextStrategy { + readonly name = 'never-ready-test'; + + checkReadiness(): ReadinessState { + return { ready: false, description: 'never ready' }; + } + + async tick(_ctx: StrategyContext): Promise {} + + select( + _store: MessageStoreView, + _log: ContextLogView, + _budget: TokenBudget, + ): ContextEntry[] { + return []; + } +} + /** A strategy whose tick fences its own manager with a failed initialization. */ class FencingTickStrategy implements ContextStrategy { readonly name = 'fencing-tick-test'; @@ -383,7 +425,7 @@ describe('queued context maintenance timer', () => { } }); - it('an agent whose context manager refuses sits the pass out, and the others still run', async () => { + it('an agent whose readiness can\'t be read fails alone: its run records the error and it gets its own alert', async () => { const dir = mkdtempSync(join(tmpdir(), 'af-maintenance-refused-')); const healthy = new QueuedStrategy(); const framework = await AgentFramework.create({ @@ -414,21 +456,27 @@ describe('queued context maintenance timer', () => { try { framework.start(); await waitFor(() => healthy.ticks >= 2); - await waitFor(() => framework.getContextMaintenanceSnapshot().history.length >= 1); + await waitFor(() => framework.getContextMaintenanceSnapshot().history.some( + (run) => run.agents.some((agent) => agent.agentName === 'healthy' && agent.readyAfter === true), + )); const snapshot = framework.getContextMaintenanceSnapshot(); - assert.deepEqual(snapshot.history[0].agents.map((run) => run.agentName), ['healthy']); - assert.equal(snapshot.history[0].agents[0].readyAfter, true); + const run = snapshot.history.find((r) => r.agents.some((agent) => agent.agentName === 'healthy'))!; + const fencedRun = run.agents.find((agent) => agent.agentName === 'fenced'); + assert.equal(fencedRun?.ticks, 0); + assert.match(fencedRun?.error ?? '', /requires successful strategy initialization/); const entry = snapshot.agents.find((agent) => agent.agentName === 'fenced'); assert.equal(entry?.ready, false); - assert.match(entry?.refused ?? '', /requires successful strategy initialization/); - assert.deepEqual(alerts, []); + assert.match(entry?.error ?? '', /requires successful strategy initialization/); + assert.ok(alerts.some((alert) => alert.agentName === 'fenced' && alert.data?.scope === 'agent')); + assert.equal(alerts.some((alert) => alert.data?.scope === 'pass'), false); + assert.equal(alerts.some((alert) => alert.agentName === 'healthy'), false); } finally { await framework.stop(); rmSync(dir, { recursive: true, force: true }); } }); - it('a refusal that begins during the pass is recorded on the agent, not thrown out of the pass', async () => { + it('a failure that begins during the pass is that agent\'s error and alert, not the pass\'s', async () => { const dir = mkdtempSync(join(tmpdir(), 'af-maintenance-refused-after-')); const strategy = new FencingTickStrategy(); const framework = await AgentFramework.create({ @@ -453,12 +501,107 @@ describe('queued context maintenance timer', () => { await waitFor(() => framework.getContextMaintenanceSnapshot().history.length >= 1); const run = framework.getContextMaintenanceSnapshot().history[0].agents[0]; assert.equal(run.agentName, 'agent'); - assert.equal(run.readyAfter, false); - assert.match(run.refusedAfter ?? '', /requires successful strategy initialization/); + assert.equal(run.ticks, 1); + assert.match(run.error ?? '', /requires successful strategy initialization/); + assert.equal(run.readyAfter, undefined); + assert.ok(alerts.some((alert) => alert.agentName === 'agent' && alert.kind === 'context-maintenance-failed' && alert.data?.scope === 'agent')); + assert.equal(alerts.some((alert) => alert.data?.scope === 'pass'), false); + } finally { + await framework.stop(); + rmSync(dir, { recursive: true, force: true }); + } + }); + + it('a strategy that can\'t report its readiness or its progress fails that agent alone, and the snapshot still answers', async () => { + const dir = mkdtempSync(join(tmpdir(), 'af-maintenance-stale-')); + const healthy = new QueuedStrategy(); + const framework = await AgentFramework.create({ + storePath: join(dir, 'store'), + membrane, + agents: [ + { name: 'stale', model: 'test-model', systemPrompt: 'test', strategy: new StaleStrategy() }, + { name: 'healthy', model: 'test-model', systemPrompt: 'test', strategy: healthy }, + ], + modules: [], + syncIntervalMs: 0, + maintenanceIntervalMs: 10, + }); + const alerts: Array<{ kind: string; agentName: string; data?: Record }> = []; + (framework as any).opsAlert = ( + kind: string, + agentName: string, + _message: string, + opts?: { data?: Record }, + ) => alerts.push({ kind, agentName, data: opts?.data }); + + try { + framework.start(); + await waitFor(() => healthy.ticks >= 2); + await waitFor(() => framework.getContextMaintenanceSnapshot().history.some( + (run) => run.agents.some((agent) => agent.agentName === 'healthy' && agent.readyAfter === true), + )); + const snapshot = framework.getContextMaintenanceSnapshot(); + const entry = snapshot.agents.find((agent) => agent.agentName === 'stale'); + assert.equal(entry?.ready, false); + assert.match(entry?.error ?? '', /requires reinitialization for the current branch generation/); + assert.equal(entry?.progress, undefined); + assert.ok(alerts.some((alert) => alert.agentName === 'stale' && alert.data?.scope === 'agent')); assert.equal(alerts.some((alert) => alert.data?.scope === 'pass'), false); } finally { await framework.stop(); rmSync(dir, { recursive: true, force: true }); } }); + + it('no pass starts while a surgery holds the store', async () => { + const dir = mkdtempSync(join(tmpdir(), 'af-maintenance-surgery-hold-')); + const strategy = new QueuedStrategy(); + const framework = await AgentFramework.create({ + storePath: join(dir, 'store'), + membrane, + agents: [{ name: 'agent', model: 'test-model', systemPrompt: 'test', strategy }], + modules: [], + syncIntervalMs: 0, + maintenanceIntervalMs: 10, + }); + const internals = framework as unknown as { surgeryHold: { verb: string; agentName: string; since: number } | null }; + internals.surgeryHold = { verb: 'rollback', agentName: 'agent', since: Date.now() }; + try { + framework.start(); + await new Promise((resolve) => setTimeout(resolve, 150)); + assert.equal(strategy.ticks, 0); + assert.equal(framework.getContextMaintenanceSnapshot().history.length, 0); + internals.surgeryHold = null; + await waitFor(() => strategy.ticks >= 2); + } finally { + internals.surgeryHold = null; + await framework.stop(); + rmSync(dir, { recursive: true, force: true }); + } + }); + + it('a pass that ends before its agent is ready records what is still pending', async () => { + const dir = mkdtempSync(join(tmpdir(), 'af-maintenance-pending-after-')); + const strategy = new NeverReadyStrategy(); + const framework = await AgentFramework.create({ + storePath: join(dir, 'store'), + membrane, + agents: [{ name: 'agent', model: 'test-model', systemPrompt: 'test', strategy }], + modules: [], + syncIntervalMs: 0, + maintenanceIntervalMs: 10, + }); + try { + framework.start(); + await waitFor(() => framework.getContextMaintenanceSnapshot().history.length >= 1); + const run = framework.getContextMaintenanceSnapshot().history.at(-1)!.agents[0]; + assert.equal(run.ticks, 8); + assert.equal(run.readyAfter, false); + assert.equal(run.pendingBefore, 'never ready'); + assert.equal(run.pendingAfter, 'never ready'); + } finally { + await framework.stop(); + rmSync(dir, { recursive: true, force: true }); + } + }); }); diff --git a/test/offline-recovery-branch.test.ts b/test/offline-recovery-branch.test.ts index 2991a8086..1f3e5bbf4 100644 --- a/test/offline-recovery-branch.test.ts +++ b/test/offline-recovery-branch.test.ts @@ -283,6 +283,19 @@ type ShardAppender = { }; }; +/** One body of `texts.length` shards, each stored with its own Discord message id. */ +function appendShards(cm: ContextManager, group: string, discordIds: string[]): string[] { + return discordIds.map((discordMessageId, shardIndex) => String( + (cm as unknown as ShardAppender).messageStore.append( + 'user', + [{ type: 'text', text: `shard ${shardIndex}` }], + { serverId: 'discord', channelId: 'discord:g1:c1', messageId: discordMessageId }, + undefined, + { bodyGroupId: group, shardIndex }, + ).id, + )); +} + function appendDocumentCopy(cm: ContextManager, discordMessageId: string): string[] { return ['part one', 'part two'].map((text, shardIndex) => String( (cm as unknown as ShardAppender).messageStore.append( @@ -318,6 +331,14 @@ test('an anchor at the end of one copy of a reposted document does not split it' assert.equal(result.messagesRemoved, 2); assert.deepEqual(result.refs.map((ref) => ref.messageId), ['m-doc-2']); + const recoveredStore = JsStore.openOrCreate({ path: storePath }); + const recoveredCm = await ContextManager.open({ store: recoveredStore, namespace: 'agents/cairn' }); + assert.deepEqual( + recoveredCm.getAllMessages().map((message) => message.metadata?.messageId), + ['m-before', 'm-doc-1', 'm-doc-1'], + ); + recoveredCm.close(); + recoveredStore.close(); } finally { rmSync(dir, { recursive: true, force: true }); } @@ -390,3 +411,103 @@ test('suppressing one copy of a reposted document keeps the other', async () => rmSync(dir, { recursive: true, force: true }); } }); + +test('a suppression that splits one copy of a document is refused', async () => { + const dir = mkdtempSync(join(tmpdir(), 'offline-recovery-split-suppress-')); + const storePath = join(dir, 'agent.chronicle'); + try { + const store = JsStore.openOrCreate({ path: storePath }); + const cm = await ContextManager.open({ store, namespace: 'agents/cairn' }); + cm.addMessage('user', [{ type: 'text', text: 'before' }], { + serverId: 'discord', channelId: 'discord:g1:c1', messageId: 'm-before', + }); + appendShards(cm, 'doc-group', ['m-s0', 'm-s1']); + appendShards(cm, 'doc-group', ['m-t0', 'm-t1']); + cm.addMessage('user', [{ type: 'text', text: 'current' }], { + serverId: 'discord', channelId: 'discord:g1:c1', messageId: 'm-current', + }); + cm.close(); + store.close(); + + await assert.rejects( + createOfflineRecoveryBranch({ + storePath, + agentName: 'cairn', + messageId: 'm-current', + suppressMessageIds: ['m-s1'], + }), + /Recovery selection splits body group doc-group/, + ); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); + +test('a split across a scan window is refused too', async () => { + const dir = mkdtempSync(join(tmpdir(), 'offline-recovery-split-seam-')); + const storePath = join(dir, 'agent.chronicle'); + try { + const store = JsStore.openOrCreate({ path: storePath }); + const cm = await ContextManager.open({ store, namespace: 'agents/cairn' }); + // 999 plain messages put the body's shards at positions 999 and 1000, + // on either side of the validator's 1,000-message scan window. + for (let i = 0; i < 999; i++) cm.addMessage('user', [{ type: 'text', text: `filler ${i}` }]); + appendShards(cm, 'seam-group', ['m-s0', 'm-s1']); + cm.addMessage('user', [{ type: 'text', text: 'current' }], { + serverId: 'discord', channelId: 'discord:g1:c1', messageId: 'm-current', + }); + assert.equal(cm.getMessageCount(), 1002); + cm.close(); + store.close(); + + await assert.rejects( + createOfflineRecoveryBranch({ + storePath, + agentName: 'cairn', + messageId: 'm-current', + suppressMessageIds: ['m-s1'], + }), + /Recovery selection splits body group seam-group/, + ); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); + +test('suppressing a one-shard copy of a reposted document keeps the other', async () => { + const dir = mkdtempSync(join(tmpdir(), 'offline-recovery-one-shard-')); + const storePath = join(dir, 'agent.chronicle'); + try { + const store = JsStore.openOrCreate({ path: storePath }); + const cm = await ContextManager.open({ store, namespace: 'agents/cairn' }); + cm.addMessage('user', [{ type: 'text', text: 'before' }], { + serverId: 'discord', channelId: 'discord:g1:c1', messageId: 'm-before', + }); + appendShards(cm, 'one-shard-group', ['m-doc-1']); + appendShards(cm, 'one-shard-group', ['m-doc-2']); + cm.addMessage('user', [{ type: 'text', text: 'current' }], { + serverId: 'discord', channelId: 'discord:g1:c1', messageId: 'm-current', + }); + cm.close(); + store.close(); + + const result = await createOfflineRecoveryBranch({ + storePath, + agentName: 'cairn', + messageId: 'm-current', + suppressMessageIds: ['m-doc-2'], + }); + + assert.equal(result.messagesSuppressed, 1); + const recoveredStore = JsStore.openOrCreate({ path: storePath }); + const recoveredCm = await ContextManager.open({ store: recoveredStore, namespace: 'agents/cairn' }); + assert.deepEqual( + recoveredCm.getAllMessages().map((message) => message.metadata?.messageId), + ['m-before', 'm-doc-1', 'm-current'], + ); + recoveredCm.close(); + recoveredStore.close(); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); From a600170425302bf94ffa1cd0e939eca202267e17 Mon Sep 17 00:00:00 2001 From: Ada-1017 Date: Tue, 13 Oct 2026 23:56:10 -0400 Subject: [PATCH 05/30] test(maintenance): a surgery that begins inside a pass Two tests for lines of the containment change that no test reached: - A pass starts no new tick once a surgery holds the store, and reads nothing after its ticks: the tick loop's surgeryHold guard, and the after-ticks readiness read skipped under the hold (readyAfter and pendingAfter stay unset). - A failure met while a surgery holds the store is recorded on the agent's run and logged naming the surgery, with no alert: reportMaintenanceFailure's hold branch. HookedTickStrategy is never ready and its tick runs a hook, so a test can begin the hold inside a pass. --- test/maintenance-timer.test.ts | 98 ++++++++++++++++++++++++++++++++++ 1 file changed, 98 insertions(+) diff --git a/test/maintenance-timer.test.ts b/test/maintenance-timer.test.ts index 15f78d3ee..2f3141db7 100644 --- a/test/maintenance-timer.test.ts +++ b/test/maintenance-timer.test.ts @@ -187,6 +187,30 @@ class FencingTickStrategy implements ContextStrategy { } } +/** A strategy with work pending whose tick runs a hook: a surgery can begin inside a pass. */ +class HookedTickStrategy implements ContextStrategy { + readonly name = 'hooked-tick-test'; + ticks = 0; + onTick?: () => void; + + checkReadiness(): ReadinessState { + return { ready: false, description: 'hooked tick queued' }; + } + + async tick(_ctx: StrategyContext): Promise { + this.ticks++; + this.onTick?.(); + } + + select( + _store: MessageStoreView, + _log: ContextLogView, + _budget: TokenBudget, + ): ContextEntry[] { + return []; + } +} + class ToolModule implements Module { readonly name = 'maintenance-tools'; @@ -580,6 +604,80 @@ describe('queued context maintenance timer', () => { } }); + it('a pass starts no new tick once a surgery holds the store, and reads nothing after its ticks', async () => { + const dir = mkdtempSync(join(tmpdir(), 'af-maintenance-surgery-mid-pass-')); + const strategy = new HookedTickStrategy(); + const framework = await AgentFramework.create({ + storePath: join(dir, 'store'), + membrane, + agents: [{ name: 'agent', model: 'test-model', systemPrompt: 'test', strategy }], + modules: [], + syncIntervalMs: 0, + maintenanceIntervalMs: 10, + }); + const internals = framework as unknown as { surgeryHold: { verb: string; agentName: string; since: number } | null }; + const alerts: string[] = []; + (framework as any).opsAlert = (kind: string) => alerts.push(kind); + strategy.onTick = () => { + internals.surgeryHold = { verb: 'rollback', agentName: 'agent', since: Date.now() }; + }; + try { + framework.start(); + await waitFor(() => framework.getContextMaintenanceSnapshot().history.length >= 1); + const run = framework.getContextMaintenanceSnapshot().history[0].agents[0]; + assert.equal(strategy.ticks, 1); + assert.equal(run.ticks, 1); + assert.equal(run.error, undefined); + assert.equal(run.readyAfter, undefined); + assert.equal(run.pendingAfter, undefined); + assert.deepEqual(alerts, []); + } finally { + internals.surgeryHold = null; + await framework.stop(); + rmSync(dir, { recursive: true, force: true }); + } + }); + + it('a failure met while a surgery holds the store is recorded and logged, without an alert', async () => { + const dir = mkdtempSync(join(tmpdir(), 'af-maintenance-surgery-failure-')); + const strategy = new HookedTickStrategy(); + const framework = await AgentFramework.create({ + storePath: join(dir, 'store'), + membrane, + agents: [{ name: 'agent', model: 'test-model', systemPrompt: 'test', strategy }], + modules: [], + syncIntervalMs: 0, + maintenanceIntervalMs: 10, + }); + const internals = framework as unknown as { surgeryHold: { verb: string; agentName: string; since: number } | null }; + const alerts: string[] = []; + (framework as any).opsAlert = (kind: string) => alerts.push(kind); + strategy.onTick = () => { + internals.surgeryHold = { verb: 'rollback', agentName: 'agent', since: Date.now() }; + throw new Error('the store switched branches under the tick'); + }; + const logged: string[] = []; + const consoleError = console.error; + console.error = (...args: unknown[]) => { logged.push(args.map(String).join(' ')); }; + try { + framework.start(); + await waitFor(() => framework.getContextMaintenanceSnapshot().history.length >= 1); + const run = framework.getContextMaintenanceSnapshot().history[0].agents[0]; + assert.equal(run.ticks, 0); + assert.equal(run.error, 'the store switched branches under the tick'); + assert.equal(run.readyAfter, undefined); + assert.deepEqual(alerts, []); + assert.ok(logged.some((line) => + line.includes('agent=agent stopped while a rollback holds the store') + && line.includes('the store switched branches under the tick'))); + } finally { + console.error = consoleError; + internals.surgeryHold = null; + await framework.stop(); + rmSync(dir, { recursive: true, force: true }); + } + }); + it('a pass that ends before its agent is ready records what is still pending', async () => { const dir = mkdtempSync(join(tmpdir(), 'af-maintenance-pending-after-')); const strategy = new NeverReadyStrategy(); From 22a82affba1d4f6b43ae41a39d7afc6048da4396 Mon Sep 17 00:00:00 2001 From: Ada-1017 Date: Wed, 14 Oct 2026 00:15:58 -0400 Subject: [PATCH 06/30] docs(changelog): name the alert a failed pass raised before The old pass-wide failure raised one alert with scope 'pass' (agent name 'framework'); "framework-scope" named neither field. --- changelog.d/cm-0.14.changed.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changelog.d/cm-0.14.changed.md b/changelog.d/cm-0.14.changed.md index 7f453d7c2..b84e557ff 100644 --- a/changelog.d/cm-0.14.changed.md +++ b/changelog.d/cm-0.14.changed.md @@ -1 +1 @@ -- A context-maintenance pass no longer fails as a whole when one agent's readiness can't be read. context-manager 0.14, which agent-framework has required since 0.21.1, refuses its readiness queries while a strategy's initialization is pending (inside `switchBranch`, `fork`, `setStrategy` or `ContextManager.open`) or has failed, and a strategy can refuse on its own, as the autobiographical one does for a branch it hasn't reloaded. That agent's run now records the error, the agent gets its own `context-maintenance-failed` alert (`scope: 'agent'`), and the other agents run. Before, such a throw failed the pass for every agent, with one framework-scope alert. While an operator surgery holds the store, no pass starts and a running pass starts no new tick, and a failure met then is logged without an alert. `getContextMaintenanceSnapshot()` reports such an agent as `ready: false` with an `error`, and a strategy that can't give a progress snapshot reads as having none. +- A context-maintenance pass no longer fails as a whole when one agent's readiness can't be read. context-manager 0.14, which agent-framework has required since 0.21.1, refuses its readiness queries while a strategy's initialization is pending (inside `switchBranch`, `fork`, `setStrategy` or `ContextManager.open`) or has failed, and a strategy can refuse on its own, as the autobiographical one does for a branch it hasn't reloaded. That agent's run now records the error, the agent gets its own `context-maintenance-failed` alert (`scope: 'agent'`), and the other agents run. Before, such a throw ended the pass for every agent, with one alert for the whole pass (`scope: 'pass'`). While an operator surgery holds the store, no pass starts and a running pass starts no new tick, and a failure met then is logged without an alert. `getContextMaintenanceSnapshot()` reports such an agent as `ready: false` with an `error`, and a strategy that can't give a progress snapshot reads as having none. From fb47fabeafff2f52eced570653a2bf66921d5094 Mon Sep 17 00:00:00 2001 From: Ada-1017 Date: Wed, 14 Oct 2026 00:32:21 -0400 Subject: [PATCH 07/30] fix(maintenance): maintenanceTick answers ran: false while a surgery holds the store No pass starts under the hold, so a tick reporting ran: true there claimed a pass that never ran. It now warns, naming the surgery, and answers ran: false, as it does for a framework that isn't running. The resume() refusal test now also checks the warning the changelog promises: it names the agent and the error. --- changelog.d/cm-0.14.changed.md | 2 +- src/framework.ts | 7 +++++ test/host-quiesce.test.ts | 56 ++++++++++++++++++++++++++++++++-- 3 files changed, 62 insertions(+), 3 deletions(-) diff --git a/changelog.d/cm-0.14.changed.md b/changelog.d/cm-0.14.changed.md index b84e557ff..d1dd33fcf 100644 --- a/changelog.d/cm-0.14.changed.md +++ b/changelog.d/cm-0.14.changed.md @@ -1 +1 @@ -- A context-maintenance pass no longer fails as a whole when one agent's readiness can't be read. context-manager 0.14, which agent-framework has required since 0.21.1, refuses its readiness queries while a strategy's initialization is pending (inside `switchBranch`, `fork`, `setStrategy` or `ContextManager.open`) or has failed, and a strategy can refuse on its own, as the autobiographical one does for a branch it hasn't reloaded. That agent's run now records the error, the agent gets its own `context-maintenance-failed` alert (`scope: 'agent'`), and the other agents run. Before, such a throw ended the pass for every agent, with one alert for the whole pass (`scope: 'pass'`). While an operator surgery holds the store, no pass starts and a running pass starts no new tick, and a failure met then is logged without an alert. `getContextMaintenanceSnapshot()` reports such an agent as `ready: false` with an `error`, and a strategy that can't give a progress snapshot reads as having none. +- A context-maintenance pass no longer fails as a whole when one agent's readiness can't be read. context-manager 0.14, which agent-framework has required since 0.21.1, refuses its readiness queries while a strategy's initialization is pending (inside `switchBranch`, `fork`, `setStrategy` or `ContextManager.open`) or has failed, and a strategy can refuse on its own, as the autobiographical one does for a branch it hasn't reloaded. That agent's run now records the error, the agent gets its own `context-maintenance-failed` alert (`scope: 'agent'`), and the other agents run. Before, such a throw ended the pass for every agent, with one alert for the whole pass (`scope: 'pass'`). While an operator surgery holds the store, no pass starts and a running pass starts no new tick, a failure met then is logged without an alert, and `maintenanceTick()` answers `ran: false`. `getContextMaintenanceSnapshot()` reports such an agent as `ready: false` with an `error`, and a strategy that can't give a progress snapshot reads as having none. diff --git a/src/framework.ts b/src/framework.ts index 31a6d578d..ace0c7382 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -4382,6 +4382,13 @@ export class AgentFramework { console.warn('[host-mode] maintenanceTick: framework is not running — no pass executed'); return { ...this.getContextMaintenanceSnapshot(), ran: false }; } + if (this.surgeryHold) { + // No pass starts while a surgery holds the store (runQueuedMaintenance). + console.warn( + `[host-mode] maintenanceTick: a ${this.surgeryHold.verb} for ${this.surgeryHold.agentName} holds the store — no pass executed`, + ); + return { ...this.getContextMaintenanceSnapshot(), ran: false }; + } this.startQueuedMaintenance(); await this.maintenancePass; return { ...this.getContextMaintenanceSnapshot(), ran: true }; diff --git a/test/host-quiesce.test.ts b/test/host-quiesce.test.ts index a4154e788..75f33c916 100644 --- a/test/host-quiesce.test.ts +++ b/test/host-quiesce.test.ts @@ -312,7 +312,7 @@ test('resume proceeds with a warn when the strategy cannot preview', async () => }); }); -test('resume proceeds without a verdict for an agent whose context manager refuses', async () => { +test('resume proceeds without a verdict for an agent whose context manager refuses, and warns naming it', async (t) => { const membrane = new MockMembrane(); await withFramework(membrane, async (framework) => { await framework.quiesce(); @@ -329,8 +329,20 @@ test('resume proceeds without a verdict for an agent whose context manager refus await assert.rejects(cm.setStrategy(failing), /strategy initialization failed for the test/); assert.throws(() => cm.getHotContextSettings(), /requires successful strategy initialization/); - const status = await framework.resume(); + const warn = t.mock.method(console, 'warn', () => {}); + let status: Awaited>; + try { + status = await framework.resume(); + } finally { + warn.mock.restore(); + } assert.equal(status.quiesced, false); + assert.ok(warn.mock.calls.some((call) => { + const line = String(call.arguments[0]); + return line.includes('feasibility preview unavailable for agent') + && line.includes('requires successful strategy initialization') + && line.includes('proceeding without a verdict'); + }), 'resume warns, naming the agent and the error'); }); }); @@ -363,6 +375,46 @@ test('maintenanceTick runs the real maintenance pass while quiesced', async () = }, { strategy }); }); +test('maintenanceTick runs no pass while a surgery holds the store, and says so', async (t) => { + class QueuedStrategy implements ContextStrategy { + readonly name = 'queued-test'; + ticks = 0; + checkReadiness(): ReadinessState { + return { ready: false, description: 'test maintenance queued' }; + } + async tick(_ctx: StrategyContext): Promise { + this.ticks++; + } + select( + _store: MessageStoreView, + _log: ContextLogView, + _budget: TokenBudget, + ): ContextEntry[] { + return []; + } + } + const strategy = new QueuedStrategy(); + const membrane = new MockMembrane(); + await withFramework(membrane, async (framework) => { + framework.start(); + await framework.quiesce({ reason: 'compression window' }); + const internals = framework as unknown as { surgeryHold: { verb: string; agentName: string; since: number } | null }; + internals.surgeryHold = { verb: 'rollback', agentName: 'agent', since: Date.now() }; + const warn = t.mock.method(console, 'warn', () => {}); + try { + const snapshot = await framework.maintenanceTick(); + assert.equal(snapshot.ran, false); + assert.equal(strategy.ticks, 0); + assert.equal(snapshot.history.length, 0); + assert.ok(warn.mock.calls.some((call) => + String(call.arguments[0]).includes('a rollback for agent holds the store — no pass executed'))); + } finally { + internals.surgeryHold = null; + warn.mock.restore(); + } + }, { strategy }); +}); + // --------------------------------------------------------------------------- // End-to-end MCPL leg: data plane held while quiesced, resume deliverable // over host/command (control plane), funnel reopens and flushes the backlog. From 2a9463cdf979011b14d74507e652c1f809d532b1 Mon Sep 17 00:00:00 2001 From: Ada-1017 Date: Wed, 14 Oct 2026 00:35:26 -0400 Subject: [PATCH 08/30] test(maintenance): hold the store before start(), whose first pass ticks at once The maintenanceTick hold test set the hold after start(), so the startup pass ticked the never-ready strategy once before it, and the test failed on ticks: 1 before reaching the change. --- test/host-quiesce.test.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test/host-quiesce.test.ts b/test/host-quiesce.test.ts index 75f33c916..848841592 100644 --- a/test/host-quiesce.test.ts +++ b/test/host-quiesce.test.ts @@ -396,10 +396,11 @@ test('maintenanceTick runs no pass while a surgery holds the store, and says so' const strategy = new QueuedStrategy(); const membrane = new MockMembrane(); await withFramework(membrane, async (framework) => { - framework.start(); - await framework.quiesce({ reason: 'compression window' }); + // Held before start(), whose first pass would otherwise tick at once. const internals = framework as unknown as { surgeryHold: { verb: string; agentName: string; since: number } | null }; internals.surgeryHold = { verb: 'rollback', agentName: 'agent', since: Date.now() }; + framework.start(); + await framework.quiesce({ reason: 'compression window' }); const warn = t.mock.method(console, 'warn', () => {}); try { const snapshot = await framework.maintenanceTick(); From ea4139ae8b907bec1a8f48ae74b089fec1744144 Mon Sep 17 00:00:00 2001 From: Ada-1017 Date: Wed, 14 Oct 2026 09:15:27 -0400 Subject: [PATCH 09/30] fix(maintenance, removals): surgeries wait for a running pass; a lasting failure alerts once; shards removed with their body The review of the context-manager 0.14 containment round found that an operator surgery reserved the store without waiting for a maintenance pass already running, so a strategy tick could run across the branch switch, and that a failure met while the hold was up was only logged, whatever its cause. It also found that a failure that lasts (a manager whose strategy failed to initialize) was logged, traced and alerted on every pass, every few seconds by default, and that agent-framework still removed a single shard through removeMessage in three places, which context-manager refuses for any message in a body. Maintenance and surgeries: - rollbackToMessage and suppressMessages, once they hold the store, wait for a running maintenance pass (settleMaintenancePass) before they switch or fork the branch. The pass starts no new tick while the hold is up, so the wait is at most the tick in flight. No maintenance work overlaps a move. - reportMaintenanceFailure no longer treats a failure met under a hold as the surgery's: with the wait, none is. The pass also reads an agent's readiness after its ticks while a surgery waits, since the branch hasn't moved yet. - A failure is logged and alerted once per agent until its recorded text changes, or until that agent's maintenance goes without a failure: a pass that finds it ready, or one whose ticks and readiness read succeed (maintenanceFailureReported). Each run still records it. - Whatever throws while one agent's maintenance is prepared (the tool refresh, the provider-gate read) is that agent's failure, so the pass runs the others. - resume() refuses with OperatorActionError('agent-busy') while a surgery holds the store, as the other operator calls do. It waits for a running pass through the same helper. Its warning for an agent whose settings can't be read says why that agent gets no verdict. Removals (src/stored-removal.ts): - removeStoredMessages removes a single message outside any body with removeMessage, whose remove event also drops the context-log entries copied from it, and a shard or a longer interval with removeMessages. - Offline recovery's suppressions go through it (the previous round had sent every single message as a range, which keeps those log entries), as do a Discord suppression resumed after a crash and the host hide command. - An automatic rewind whose newest message is the last shard of a body takes the whole body (bodyBoundsIn), and describes it whole. Tests: - maintenance-timer: a rollback and a suppression that begin during a gated tick keep the branch until the tick ends, start no tick under the hold, and make resume() refuse as agent-busy meanwhile; a lasting failure is alerted once and again after each way it can end (found ready; a pass without a failure); a throw while one agent's pass is prepared is that agent's alone; a readiness refusal after an agent's last tick is reported from the pass's finally, once; a run reports an agent's first failure only; the hold tests count the timer's passes instead of sleeping, and expect the readiness read and the alert the pass now makes. - offline-recovery-branch: a plain message is removed with removeMessage and a one-shard copy with removeMessages; an anchor at the end of a write cut short keeps the partial body and leaves out its retry. - discord-awareness-outbox: a resumed suppression removes a one-shard body as a range and a plain message alone. - operator-surgery: hide removes a one-shard body; a rewind takes the whole body of its newest shard. - host-quiesce: the resume warning's new wording. --- src/framework.ts | 159 +++++++++---- src/recovery/offline-branch.ts | 12 +- src/stored-removal.ts | 37 +++ test/discord-awareness-outbox.test.ts | 52 ++++ test/host-quiesce.test.ts | 4 +- test/maintenance-timer.test.ts | 326 ++++++++++++++++++++++++-- test/offline-recovery-branch.test.ts | 87 +++++++ test/operator-surgery.test.ts | 36 +++ 8 files changed, 640 insertions(+), 73 deletions(-) create mode 100644 src/stored-removal.ts diff --git a/src/framework.ts b/src/framework.ts index ace0c7382..707845300 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -19,6 +19,7 @@ import type { StoredMessage, TokenBudget, } from '@animalabs/context-manager'; +import { bodyStartIn, removeStoredMessages } from './stored-removal.js'; import type { FrameworkConfig, InferencePolicy, @@ -1422,6 +1423,9 @@ export class AgentFramework { private maintenanceRunId = 0; private currentMaintenanceRun: ContextMaintenanceRun | null = null; private maintenanceHistory: ContextMaintenanceRun[] = []; + /** Each agent's last reported maintenance failure, by its recorded text, + * until its maintenance goes without one (reportMaintenanceFailure). */ + private maintenanceFailureReported = new Map(); private providerGates: Map = new Map(); private providerAccelerationCooldowns: Map = new Map(); private providerAccelerationRecoveries: Map = new Map(); @@ -2785,11 +2789,18 @@ export class AgentFramework { }); } + /** Wait for a maintenance pass that is running, whatever its outcome. */ + private async settleMaintenancePass(): Promise { + const pass = this.maintenancePass; + if (pass) await pass.catch(() => {}); + } + private async runQueuedMaintenance(): Promise { - // A surgery holds the store while it switches or forks a branch, and a - // context manager mid-switch refuses its queries. Maintenance waits for - // the hold to end rather than report the surgery as a failure: no pass - // starts while it's up, and a running pass starts no new tick. + // A surgery holds the store while it switches or forks a branch. No pass + // starts while it's held (the timer's next pass after it runs as usual), + // and a running pass starts no new tick. The surgery waits for that pass + // to finish before it moves the branch, so no maintenance work overlaps + // the move, and a failure a pass meets is never the surgery's doing. if (this.surgeryHold) return; // A guard whose agent has left the map gets no activation to retry its // queue; retry it here. It leaves busyGuards once nothing of it is left. @@ -2802,18 +2813,49 @@ export class AgentFramework { console.error(`[tool-result-guard] agent=${name} storage retry failed during maintenance:`, error); } } - const queued = [...this.agents.values()].flatMap((agent) => { + for (const name of this.maintenanceFailureReported.keys()) { + if (!this.agents.has(name)) this.maintenanceFailureReported.delete(name); + } + type Queued = { agent: Agent; record: ContextMaintenanceAgentRun } + & ({ failed: true } | { failed: false; cm: ContextManager }); + const queued = [...this.agents.values()].flatMap((agent): Queued[] => { try { agent.toolResultGuard.flushUnrecorded(); } catch (error) { console.error(`[tool-result-guard] agent=${agent.name} storage retry failed during maintenance:`, error); } - const cm = agent.getContextManager(); - const tools = this.compressionToolsForAgent(agent.name); - cm.setToolDefinitions(tools); - if (this.providerGateBlocked(agent.name)) return []; - const state = this.contextReadiness(cm); - if ('ready' in state && state.ready) return []; + // Whatever throws while one agent's maintenance is prepared is that + // agent's failure, reported for it alone, while the others run. + const failedAgent = (error: unknown): Queued[] => { + const record: ContextMaintenanceAgentRun = { + agentName: agent.name, + startedAt: Date.now(), + ticks: 0, + readyBefore: false, + }; + record.finishedAt = record.startedAt; + this.reportMaintenanceFailure( + agent.name, + record, + error instanceof Error ? String(error.message) : failureDescription(error), + ); + return [{ agent, record, failed: true }]; + }; + let cm: ContextManager; + let state: ReturnType; + try { + cm = agent.getContextManager(); + const tools = this.compressionToolsForAgent(agent.name); + cm.setToolDefinitions(tools); + if (this.providerGateBlocked(agent.name)) return []; + state = this.contextReadiness(cm); + } catch (error) { + return failedAgent(error); + } + if ('ready' in state && state.ready) { + this.maintenanceFailureReported.delete(agent.name); + return []; + } const progress = this.contextProgress(cm); const record: ContextMaintenanceAgentRun = { agentName: agent.name, @@ -2828,7 +2870,7 @@ export class AgentFramework { // alone, while the others run. record.finishedAt = Date.now(); this.reportMaintenanceFailure(agent.name, record, state.error); - return [{ agent, cm, record, failed: true }]; + return [{ agent, record, failed: true }]; } return [{ agent, cm, record, failed: false }]; }); @@ -2842,7 +2884,8 @@ export class AgentFramework { this.currentMaintenanceRun = run; try { - await Promise.all(queued.filter((item) => !item.failed).map(async ({ agent, cm, record }) => { + const running = queued.filter((item): item is Extract => !item.failed); + await Promise.all(running.map(async ({ agent, cm, record }) => { try { for ( let i = 0; @@ -2860,14 +2903,15 @@ export class AgentFramework { ); } finally { record.finishedAt = Date.now(); - if (!this.surgeryHold) { - const state = this.contextReadiness(cm); - if ('error' in state) { - if (record.error === undefined) this.reportMaintenanceFailure(agent.name, record, state.error); - } else { - record.readyAfter = state.ready; - if (state.pending) record.pendingAfter = state.pending; - } + const state = this.contextReadiness(cm); + if ('error' in state) { + // A run records an agent's first failure: a readiness refusal + // after a tick that failed is read again by the next pass. + if (record.error === undefined) this.reportMaintenanceFailure(agent.name, record, state.error); + } else { + record.readyAfter = state.ready; + if (state.pending) record.pendingAfter = state.pending; + if (record.error === undefined) this.maintenanceFailureReported.delete(agent.name); } const progress = this.contextProgress(cm); if (progress) record.progressAfter = progress; @@ -2884,19 +2928,17 @@ export class AgentFramework { /** * One agent's maintenance failure: recorded on its run, bounded as every - * failure sink is (`record.error`), and alerted for that agent with the - * whole text, unless a surgery holds the store, in which case the surgery - * is what ended it and it is only logged. + * failure sink is (`record.error`), and logged and alerted for that agent + * with the whole text. A failure that lasts, such as a manager whose + * strategy failed to initialize, meets every pass, a few seconds apart: + * each run records it, and it is logged and alerted again only when its + * text changes, or after a pass in which that agent's maintenance went + * without a failure (maintenanceFailureReported). */ private reportMaintenanceFailure(agentName: string, record: ContextMaintenanceAgentRun, wholeError: string): void { record.error = boundFailureText(wholeError); - if (this.surgeryHold) { - console.error( - `[context-maintenance] agent=${agentName} stopped while a ${this.surgeryHold.verb} holds the store:`, - record.error, - ); - return; - } + if (this.maintenanceFailureReported.get(agentName) === record.error) return; + this.maintenanceFailureReported.set(agentName, record.error); console.error(`[context-maintenance] agent=${agentName} failed:`, record.error); this.opsAlert( 'context-maintenance-failed', @@ -3971,15 +4013,22 @@ export class AgentFramework { */ async resume(opts?: { force?: boolean }): Promise { if (!this.quiesced) return this.getHostModeStatus(); + // A surgery switches or forks the branch while it holds the store, and a + // manager whose initialization that left pending refuses its settings + // read: resume once it's done, as the other operator calls do. + if (this.surgeryHold) { + throw new OperatorActionError( + 'agent-busy', + `Cannot resume: a live ${this.surgeryHold.verb} for ${this.surgeryHold.agentName} holds the store — retry when it completes`, + ); + } // The canonical operator flow is `maintain; resume`, and a timer-driven // pass may still be mid-flight: previewContext refuses to overlap one // ("already running"), which used to degrade the gate to "no opinion". // Wait for the pass first, then retry a few times for the case where a // fresh pass started in between. - if (this.maintenancePass) { - await this.maintenancePass.catch(() => {}); - } + await this.settleMaintenancePass(); const failing: Array<{ agentName: string; preview: RuntimeSettingsPreview }> = []; for (const agentName of this.agents.keys()) { let preview: RuntimeSettingsPreview; @@ -3991,12 +4040,15 @@ export class AgentFramework { preview = this.previewAgentRuntimeSettings(agentName); } } catch (error) { - // The agent's settings can't be read (its context manager refuses while - // its strategy's initialization is pending or has failed): no verdict - // for it, as for any preview that isn't available. + // The agent's settings can't be read, as when its context manager's + // strategy failed to initialize: that manager refuses its compiles and + // writes until a switchBranch or setStrategy succeeds, and maintenance + // reports it. The gate guards against an agent that would run over + // budget, and this one can't run, so it gets no verdict, as a preview + // that isn't available gets none, and resume goes on for the others. console.warn( - `[host-mode] resume: feasibility preview unavailable for ${agentName} ` + - `(${error instanceof Error ? error.message : String(error)}) — proceeding without a verdict`, + `[host-mode] resume: ${agentName}'s settings can't be read ` + + `(${error instanceof Error ? error.message : String(error)}) — proceeding without a verdict for it`, ); continue; } @@ -5636,27 +5688,34 @@ export class AgentFramework { idx = i; break; } if (idx < 0) return null; + // The newest message can be the last shard of a body (a document stored + // in shards). context-manager removes a shard only with the rest of its + // body, so the rewind takes the whole body. + const first = bodyStartIn(all, idx); + const members = all.slice(first, idx + 1); const msg = all[idx]; const md = (msg.metadata ?? {}) as { messageId?: unknown; channelId?: unknown; system?: unknown; }; - const content = Array.isArray(msg.content) ? msg.content : []; + const content = members.flatMap((m) => (Array.isArray(m.content) ? m.content : [])); const toolBlocks = content.filter((b) => typeOf(b) === 'tool_result'); const images = content.filter((b) => typeOf(b) === 'image').length; const textLen = content .filter((b) => typeOf(b) === 'text') .reduce((n, b) => n + String((b as { text?: string }).text ?? '').length, 0); - // Determine the complete exchange to remove. - const removedIds: MessageId[] = [msg.id]; + // Determine the complete exchange to remove, newest first. + const removedIds: MessageId[] = members.map((m) => m.id).reverse(); + let paired: StoredMessage | undefined; let kind: RewindRecord['kind']; let descriptor: string; if (toolBlocks.length > 0) { kind = 'tool'; // Pair the tool_result with the tool_use assistant turn right before it. - if (idx - 1 >= 0 && hasBlock(all[idx - 1], 'tool_use')) { - removedIds.push(all[idx - 1].id); + if (first - 1 >= 0 && hasBlock(all[first - 1], 'tool_use')) { + paired = all[first - 1]; + removedIds.push(paired.id); } const sz = toolBlocks.reduce( (n, b) => n + String((b as { content?: unknown }).content ?? '').length, 0); @@ -5675,7 +5734,8 @@ export class AgentFramework { ? { channelId: String(md.channelId), messageId: String(md.messageId) } : undefined; - for (const id of removedIds) cm.removeMessage(id); + removeStoredMessages(cm, all[first], msg); + if (paired !== undefined) removeStoredMessages(cm, paired, paired); return { kind, descriptor, removedIds, discordRef }; } @@ -6077,7 +6137,7 @@ export class AgentFramework { } const refs = refsIn(fromIdx, fromIdx); const hiddenId = String(all[fromIdx].id); - cm.removeMessage(all[fromIdx].id); + removeStoredMessages(cm, all[fromIdx], all[fromIdx]); console.error( `[host-command] hide agent=${agentName} removed=1 (${params.fromMessageId}) ` + `by=${params.requesterName ?? params.requesterId ?? 'unknown'} (server=${serverId})`, @@ -6837,6 +6897,8 @@ export class AgentFramework { let branchName: string; let markers: SurgeryMarkerReceipt; try { + // A maintenance tick already running finishes before the switch. + await this.settleMaintenancePass(); // Prepare the external side effect before switching Chronicle. If the // process dies after the switch but before activate(), startup // promotes this batch by matching targetBranch to the active branch. @@ -7004,6 +7066,8 @@ export class AgentFramework { let createdBranch: string; let markers: SurgeryMarkerReceipt; try { + // A maintenance tick already running finishes before the fork. + await this.settleMaintenancePass(); const markerBatch = this.discordAwarenessOutbox?.prepare({ agentName, sourceBranch, @@ -14799,8 +14863,7 @@ export class AgentFramework { `${interval.fromId}..${interval.toId} is only partially present`, ); } - if (interval.fromId === interval.toId) cm.removeMessage(from.id); - else cm.removeMessages(from.id, to.id); + removeStoredMessages(cm, from, to); } this.discordAwarenessOutbox.activate(batch.id); console.error(`[discord-awareness] resumed suppression batch ${batch.id}`); diff --git a/src/recovery/offline-branch.ts b/src/recovery/offline-branch.ts index fbd315825..49137f1af 100644 --- a/src/recovery/offline-branch.ts +++ b/src/recovery/offline-branch.ts @@ -1,5 +1,6 @@ import { JsStore } from '@animalabs/chronicle'; import { ContextManager, continuesBody, type BodyShard } from '@animalabs/context-manager'; +import { removeStoredInterval } from '../stored-removal.js'; import { DEFAULT_DISCORD_AWARENESS_EMOJI, DiscordAwarenessOutbox, @@ -202,13 +203,12 @@ export async function createOfflineRecoveryBranch( const createdBranch = contextManager.branchAt(target.id, targetBranch); try { await contextManager.switchBranch(createdBranch); - // Work from newest to oldest so earlier redactions never perturb the - // live positions used by later range endpoints. - // Each interval goes as a range, a single message included: removing one - // message alone is refused for any shard, a one-shard body's too, and - // the range removal refuses a bisected body itself. + // Each interval is removed by its messages' ids, so the order doesn't + // change the result. removeStoredInterval takes a single message outside + // any body through removeMessage, and a shard or a longer interval as a + // range, which refuses one that splits a body. for (const interval of [...suppression.intervals].reverse()) { - contextManager.removeMessages(interval.fromId, interval.toId); + removeStoredInterval(contextManager, interval.fromId, interval.toId); } if (batch) outbox.activate(batch.id); } catch (error) { diff --git a/src/stored-removal.ts b/src/stored-removal.ts new file mode 100644 index 000000000..5c840774b --- /dev/null +++ b/src/stored-removal.ts @@ -0,0 +1,37 @@ +import { bodyBoundsIn, type BodyShard, type ContextManager, type MessageId } from '@animalabs/context-manager'; + +type StoredLike = BodyShard & { id: MessageId }; +type Removals = Pick; + +/** + * Remove the stored messages `first`..`last` (inclusive, in store order). + * + * A single message outside any body goes through `removeMessage`, whose + * remove event also drops the context-log entries copied from it. + * context-manager's `removeMessage` refuses any message that carries a body + * group id, a one-shard body's included, so a shard, and any longer interval, + * goes through `removeMessages`, which refuses an interval that splits a + * body. + */ +export function removeStoredMessages(cm: Removals, first: StoredLike, last: StoredLike): void { + if (first.id === last.id && !first.bodyGroupId) { + cm.removeMessage(first.id); + return; + } + cm.removeMessages(first.id, last.id); +} + +/** removeStoredMessages for an interval known by its ids. */ +export function removeStoredInterval(cm: ContextManager, fromId: MessageId, toId: MessageId): void { + const first = cm.getMessage(fromId) as StoredLike | null; + removeStoredMessages(cm, first ?? { id: fromId }, { id: toId }); +} + +/** + * The first position of the body that holds `messages[index]`, read the way + * context-manager 0.14.0 reads bodies (`continuesBody`, pairwise in store + * order). A message outside any body is a body of its own. + */ +export function bodyStartIn(messages: readonly unknown[], index: number): number { + return bodyBoundsIn(messages as readonly (BodyShard | null | undefined)[], index).from; +} diff --git a/test/discord-awareness-outbox.test.ts b/test/discord-awareness-outbox.test.ts index fb0f3bed1..94a6d5b95 100644 --- a/test/discord-awareness-outbox.test.ts +++ b/test/discord-awareness-outbox.test.ts @@ -143,6 +143,58 @@ test('framework resumes an interrupted suppression before activating its markers } }); +test('a resumed suppression removes a one-shard body as a range and a message outside any body alone', async () => { + const dir = mkdtempSync(join(tmpdir(), 'discord-awareness-resume-shard-')); + const path = join(dir, 'outbox.json'); + try { + const outbox = new DiscordAwarenessOutbox(path); + const batch = outbox.prepare({ + agentName: 'cairn', + sourceBranch: 'main', + targetBranch: 'recovery/cairn/suppressed', + activationPolicy: 'explicit', + suppressionIntervals: [ + { fromId: 'plain', toId: 'plain' }, + { fromId: 'shard', toId: 'shard' }, + ], + refs: [ + { serverId: 'discord', channelId: 'discord:g1:c1', messageId: 'm1' }, + ], + })!; + const present = new Set(['plain', 'shard']); + const removals: string[] = []; + // As context-manager 0.14 does, removeMessage refuses a shard, a + // one-shard body's included. + const cm = { + getMessage: (id: string) => present.has(id) + ? { id, ...(id === 'shard' ? { bodyGroupId: 'doc', shardIndex: 0 } : {}) } + : null, + removeMessage: (id: string) => { + if (id === 'shard') throw new Error(`Cannot remove shard ${id} in isolation: it is part of bodyGroup doc.`); + present.delete(id); + removals.push(id); + }, + removeMessages: (from: string, to: string) => { + present.delete(from); present.delete(to); removals.push(`${from}..${to}`); + }, + }; + const framework = Object.create(AgentFramework.prototype) as any; + framework.discordAwarenessOutbox = outbox; + framework.store = { + currentBranch: () => ({ name: 'recovery/cairn/suppressed' }), + listBranches: () => [], + }; + framework.agents = new Map([['cairn', { getContextManager: () => cm }]]); + + await framework.resumePreparedDiscordSuppressions(); + + assert.deepEqual(removals, ['shard..shard', 'plain']); + assert.equal(outbox.batches().find((candidate) => candidate.id === batch.id)?.status, 'active'); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); + test('framework drain records failures per ref and continues past permanent failures', async () => { const dir = mkdtempSync(join(tmpdir(), 'discord-awareness-drain-')); const path = join(dir, 'outbox.json'); diff --git a/test/host-quiesce.test.ts b/test/host-quiesce.test.ts index 848841592..55213e976 100644 --- a/test/host-quiesce.test.ts +++ b/test/host-quiesce.test.ts @@ -339,9 +339,9 @@ test('resume proceeds without a verdict for an agent whose context manager refus assert.equal(status.quiesced, false); assert.ok(warn.mock.calls.some((call) => { const line = String(call.arguments[0]); - return line.includes('feasibility preview unavailable for agent') + return line.includes("agent's settings can't be read") && line.includes('requires successful strategy initialization') - && line.includes('proceeding without a verdict'); + && line.includes('proceeding without a verdict for it'); }), 'resume warns, naming the agent and the error'); }); }); diff --git a/test/maintenance-timer.test.ts b/test/maintenance-timer.test.ts index 2f3141db7..7fad80be8 100644 --- a/test/maintenance-timer.test.ts +++ b/test/maintenance-timer.test.ts @@ -23,6 +23,7 @@ import type { ToolResult, } from '../src/index.js'; import { AgentFramework } from '../src/index.js'; +import { OperatorActionError } from '../src/operator-log.js'; import type { NormalizedRequest, NormalizedResponse } from '@animalabs/membrane'; class QueuedStrategy implements ContextStrategy { @@ -211,6 +212,86 @@ class HookedTickStrategy implements ContextStrategy { } } +/** A strategy with no maintenance to do. */ +class ReadyStrategy implements ContextStrategy { + readonly name = 'ready-test'; + + checkReadiness(): ReadinessState { + return { ready: true }; + } + + async tick(_ctx: StrategyContext): Promise {} + + select( + _store: MessageStoreView, + _log: ContextLogView, + _budget: TokenBudget, + ): ContextEntry[] { + return []; + } +} + +/** A strategy whose first tick waits on a gate, and that notes what it sees. */ +class GatedTickStrategy implements ContextStrategy { + readonly name = 'gated-tick-test'; + ticks = 0; + ticksUnderHold = 0; + branchAtFirstTickEnd?: string; + framework?: AgentFramework; + private open?: () => void; + private readonly gate = new Promise((resolve) => { this.open = resolve; }); + + release(): void { + this.open?.(); + } + + checkReadiness(): ReadinessState { + return { ready: false, description: 'gated tick queued' }; + } + + async tick(_ctx: StrategyContext): Promise { + this.ticks++; + if ((this.framework as unknown as { surgeryHold: unknown }).surgeryHold) this.ticksUnderHold++; + if (this.ticks === 1) { + await this.gate; + this.branchAtFirstTickEnd = this.framework!.getAgent('agent')!.getContextManager().currentBranch().name; + } + } + + select( + _store: MessageStoreView, + _log: ContextLogView, + _budget: TokenBudget, + ): ContextEntry[] { + return []; + } +} + +/** A strategy whose ticks succeed and whose readiness can't be read once all of a pass's ticks have run. */ +class RefusesAfterTicksStrategy implements ContextStrategy { + readonly name = 'refuses-after-ticks-test'; + ticks = 0; + failTick = false; + + checkReadiness(): ReadinessState { + if (this.ticks >= 8 || (this.failTick && this.ticks > 0)) throw new Error('readiness refused after the ticks'); + return { ready: false, description: 'work queued' }; + } + + async tick(_ctx: StrategyContext): Promise { + this.ticks++; + if (this.failTick) throw new Error('tick failed'); + } + + select( + _store: MessageStoreView, + _log: ContextLogView, + _budget: TokenBudget, + ): ContextEntry[] { + return []; + } +} + class ToolModule implements Module { readonly name = 'maintenance-tools'; @@ -438,11 +519,12 @@ describe('queued context maintenance timer', () => { pending: 'compression blocked', }, }); - await waitFor(() => framework.getContextMaintenanceSnapshot().history.length > 0); - assert.equal( - framework.getContextMaintenanceSnapshot().history[0].agents[0].error, - 'provider rejected maintenance request', - ); + // Every pass meets the same failure and records it; it is alerted once. + await waitFor(() => framework.getContextMaintenanceSnapshot().history.length >= 3); + for (const run of framework.getContextMaintenanceSnapshot().history) { + assert.equal(run.agents[0].error, 'provider rejected maintenance request'); + } + assert.equal(alerts.length, 1); } finally { await framework.stop(); rmSync(dir, { recursive: true, force: true }); @@ -577,6 +659,208 @@ describe('queued context maintenance timer', () => { } }); + it('whatever throws while one agent\'s maintenance is prepared is that agent\'s failure alone', async () => { + const dir = mkdtempSync(join(tmpdir(), 'af-maintenance-prepare-throws-')); + const healthy = new QueuedStrategy(); + const framework = await AgentFramework.create({ + storePath: join(dir, 'store'), + membrane, + agents: [ + { name: 'broken', model: 'test-model', systemPrompt: 'test', strategy: new QueuedStrategy() }, + { name: 'healthy', model: 'test-model', systemPrompt: 'test', strategy: healthy }, + ], + modules: [], + syncIntervalMs: 0, + maintenanceIntervalMs: 10, + }); + const alerts: Array<{ kind: string; agentName: string; data?: Record }> = []; + (framework as any).opsAlert = ( + kind: string, + agentName: string, + _message: string, + opts?: { data?: Record }, + ) => alerts.push({ kind, agentName, data: opts?.data }); + const internals = framework as unknown as { providerGateBlocked: (name: string) => boolean }; + const gateBlocked = internals.providerGateBlocked.bind(framework); + internals.providerGateBlocked = (name: string) => { + if (name === 'broken') throw new Error('gate read failed'); + return gateBlocked(name); + }; + try { + framework.start(); + await waitFor(() => healthy.ticks >= 2); + await waitFor(() => framework.getContextMaintenanceSnapshot().history.some( + (run) => run.agents.some((agent) => agent.agentName === 'broken' && agent.error !== undefined), + )); + const run = framework.getContextMaintenanceSnapshot().history.find( + (candidate) => candidate.agents.some((agent) => agent.agentName === 'broken'), + )!; + const broken = run.agents.find((agent) => agent.agentName === 'broken')!; + assert.equal(broken.error, 'gate read failed'); + assert.equal(broken.ticks, 0); + assert.ok(alerts.some((alert) => alert.agentName === 'broken' && alert.data?.scope === 'agent')); + assert.equal(alerts.some((alert) => alert.data?.scope === 'pass'), false); + } finally { + await framework.stop(); + rmSync(dir, { recursive: true, force: true }); + } + }); + + it('a failure that lasts is logged and alerted once, and again once the agent has recovered', async (t) => { + const dir = mkdtempSync(join(tmpdir(), 'af-maintenance-lasting-')); + const framework = await AgentFramework.create({ + storePath: join(dir, 'store'), + membrane, + agents: [{ name: 'fenced', model: 'test-model', systemPrompt: 'test', strategy: new QueuedStrategy() }], + modules: [], + syncIntervalMs: 0, + maintenanceIntervalMs: 10, + }); + const alerts: string[] = []; + (framework as any).opsAlert = (kind: string, agentName: string) => alerts.push(`${kind}:${agentName}`); + const internals = framework as unknown as { runQueuedMaintenance: () => Promise }; + let passes = 0; + const runPass = internals.runQueuedMaintenance.bind(framework); + internals.runQueuedMaintenance = () => { passes++; return runPass(); }; + const logged = t.mock.method(console, 'error', () => {}); + const failedLines = () => logged.mock.calls.filter((call) => + String(call.arguments[0]).includes('[context-maintenance] agent=fenced failed:')).length; + const runsWithError = () => framework.getContextMaintenanceSnapshot().history.filter((run) => + run.agents.some((agent) => agent.agentName === 'fenced' && agent.error !== undefined)).length; + const cm = framework.getAgent('fenced')!.getContextManager(); + const fence = () => assert.rejects(cm.setStrategy(new RefusingInitStrategy()), /strategy initialization failed for the test/); + await fence(); + try { + framework.start(); + await waitFor(() => runsWithError() >= 3); + assert.deepEqual(alerts, ['context-maintenance-failed:fenced']); + assert.equal(failedLines(), 1); + + // Recovered with nothing to do: passes find it ready and skip it. + await cm.setStrategy(new ReadyStrategy()); + const after = passes; + await waitFor(() => passes >= after + 3); + await fence(); + await waitFor(() => alerts.length === 2); + assert.equal(failedLines(), 2); + + // Recovered with work no tick finishes: a pass runs it without a failure. + await cm.setStrategy(new NeverReadyStrategy()); + await waitFor(() => framework.getContextMaintenanceSnapshot().history.some((run) => + run.agents.some((agent) => agent.agentName === 'fenced' && agent.ticks === 8 && agent.error === undefined))); + await fence(); + await waitFor(() => alerts.length === 3); + assert.equal(failedLines(), 3); + assert.deepEqual(new Set(alerts), new Set(['context-maintenance-failed:fenced'])); + } finally { + logged.mock.restore(); + await framework.stop(); + rmSync(dir, { recursive: true, force: true }); + } + }); + + it('a readiness refusal read after an agent\'s last tick is that agent\'s failure, alerted once', async () => { + const dir = mkdtempSync(join(tmpdir(), 'af-maintenance-refused-after-ticks-')); + const strategy = new RefusesAfterTicksStrategy(); + const framework = await AgentFramework.create({ + storePath: join(dir, 'store'), + membrane, + agents: [{ name: 'agent', model: 'test-model', systemPrompt: 'test', strategy }], + modules: [], + syncIntervalMs: 0, + // start() runs one pass at once; none follows within the test. + maintenanceIntervalMs: 60_000, + }); + const alerts: string[] = []; + (framework as any).opsAlert = (kind: string, agentName: string, message: string) => alerts.push(`${kind}:${agentName}:${message}`); + try { + framework.start(); + await waitFor(() => framework.getContextMaintenanceSnapshot().history.length >= 1); + const run = framework.getContextMaintenanceSnapshot().history[0].agents[0]; + assert.equal(run.ticks, 8); + assert.equal(run.error, 'readiness refused after the ticks'); + assert.equal(run.readyAfter, undefined); + assert.deepEqual(alerts, ['context-maintenance-failed:agent:readiness refused after the ticks']); + } finally { + await framework.stop(); + rmSync(dir, { recursive: true, force: true }); + } + }); + + it('a run records an agent\'s first failure: a readiness refusal after a failed tick isn\'t reported again', async () => { + const dir = mkdtempSync(join(tmpdir(), 'af-maintenance-first-failure-')); + const strategy = new RefusesAfterTicksStrategy(); + strategy.failTick = true; + const framework = await AgentFramework.create({ + storePath: join(dir, 'store'), + membrane, + agents: [{ name: 'agent', model: 'test-model', systemPrompt: 'test', strategy }], + modules: [], + syncIntervalMs: 0, + maintenanceIntervalMs: 60_000, + }); + const alerts: string[] = []; + (framework as any).opsAlert = (kind: string, agentName: string, message: string) => alerts.push(`${kind}:${agentName}:${message}`); + try { + framework.start(); + await waitFor(() => framework.getContextMaintenanceSnapshot().history.length >= 1); + const run = framework.getContextMaintenanceSnapshot().history[0].agents[0]; + assert.equal(run.ticks, 0); + assert.equal(run.error, 'tick failed'); + assert.deepEqual(alerts, ['context-maintenance-failed:agent:tick failed']); + } finally { + await framework.stop(); + rmSync(dir, { recursive: true, force: true }); + } + }); + + for (const surgery of [ + { verb: 'roll back', run: (framework: AgentFramework, first: string) => framework.rollbackToMessage('agent', { messageId: first }) }, + { verb: 'suppress', run: (framework: AgentFramework, first: string) => framework.suppressMessages('agent', { messageIds: [first] }) }, + ]) { + it(`a ${surgery.verb} that begins during a tick waits for it before it moves the branch, and resume waits for the ${surgery.verb}`, { timeout: 10_000 }, async () => { + const dir = mkdtempSync(join(tmpdir(), 'af-maintenance-surgery-waits-')); + const strategy = new GatedTickStrategy(); + const framework = await AgentFramework.create({ + storePath: join(dir, 'store'), + membrane, + agents: [{ name: 'agent', model: 'test-model', systemPrompt: 'test', strategy }], + modules: [], + syncIntervalMs: 0, + maintenanceIntervalMs: 10, + }); + strategy.framework = framework; + const cm = framework.getAgent('agent')!.getContextManager(); + const first = String(cm.addMessage('user', [{ type: 'text', text: 'm1' }])); + cm.addMessage('user', [{ type: 'text', text: 'm2' }]); + const internals = framework as unknown as { surgeryHold: unknown }; + try { + await framework.quiesce(); + framework.start(); + await waitFor(() => strategy.ticks === 1); + const moved = surgery.run(framework, first); + await waitFor(() => internals.surgeryHold !== null); + await assert.rejects( + framework.resume(), + (error: unknown) => error instanceof OperatorActionError && error.code === 'agent-busy' + && error.message.includes(`a live ${surgery.verb} for agent holds the store`), + ); + assert.equal(cm.currentBranch().name, 'main', `the ${surgery.verb} waits for the running tick`); + strategy.release(); + await moved; + assert.equal(strategy.branchAtFirstTickEnd, 'main'); + assert.equal(strategy.ticksUnderHold, 0, `the pass starts no tick while the ${surgery.verb} holds the store`); + assert.notEqual(cm.currentBranch().name, 'main'); + const status = await framework.resume(); + assert.equal(status.quiesced, false); + } finally { + strategy.release(); + await framework.stop(); + rmSync(dir, { recursive: true, force: true }); + } + }); + } + it('no pass starts while a surgery holds the store', async () => { const dir = mkdtempSync(join(tmpdir(), 'af-maintenance-surgery-hold-')); const strategy = new QueuedStrategy(); @@ -588,11 +872,18 @@ describe('queued context maintenance timer', () => { syncIntervalMs: 0, maintenanceIntervalMs: 10, }); - const internals = framework as unknown as { surgeryHold: { verb: string; agentName: string; since: number } | null }; + const internals = framework as unknown as { + surgeryHold: { verb: string; agentName: string; since: number } | null; + runQueuedMaintenance: () => Promise; + }; internals.surgeryHold = { verb: 'rollback', agentName: 'agent', since: Date.now() }; + // Count the passes the timer starts while the store is held. + let passes = 0; + const runPass = internals.runQueuedMaintenance.bind(framework); + internals.runQueuedMaintenance = () => { passes++; return runPass(); }; try { framework.start(); - await new Promise((resolve) => setTimeout(resolve, 150)); + await waitFor(() => passes >= 3); assert.equal(strategy.ticks, 0); assert.equal(framework.getContextMaintenanceSnapshot().history.length, 0); internals.surgeryHold = null; @@ -604,7 +895,7 @@ describe('queued context maintenance timer', () => { } }); - it('a pass starts no new tick once a surgery holds the store, and reads nothing after its ticks', async () => { + it('a pass starts no new tick once a surgery holds the store', async () => { const dir = mkdtempSync(join(tmpdir(), 'af-maintenance-surgery-mid-pass-')); const strategy = new HookedTickStrategy(); const framework = await AgentFramework.create({ @@ -628,8 +919,10 @@ describe('queued context maintenance timer', () => { assert.equal(strategy.ticks, 1); assert.equal(run.ticks, 1); assert.equal(run.error, undefined); - assert.equal(run.readyAfter, undefined); - assert.equal(run.pendingAfter, undefined); + // The surgery waits for the pass before it switches, so reading the + // agent's readiness after its tick is safe. + assert.equal(run.readyAfter, false); + assert.equal(run.pendingAfter, 'hooked tick queued'); assert.deepEqual(alerts, []); } finally { internals.surgeryHold = null; @@ -638,7 +931,7 @@ describe('queued context maintenance timer', () => { } }); - it('a failure met while a surgery holds the store is recorded and logged, without an alert', async () => { + it('a failure a pass meets while a surgery waits for it is reported as any failure', async () => { const dir = mkdtempSync(join(tmpdir(), 'af-maintenance-surgery-failure-')); const strategy = new HookedTickStrategy(); const framework = await AgentFramework.create({ @@ -654,7 +947,7 @@ describe('queued context maintenance timer', () => { (framework as any).opsAlert = (kind: string) => alerts.push(kind); strategy.onTick = () => { internals.surgeryHold = { verb: 'rollback', agentName: 'agent', since: Date.now() }; - throw new Error('the store switched branches under the tick'); + throw new Error('the tick failed while a surgery waited'); }; const logged: string[] = []; const consoleError = console.error; @@ -664,12 +957,11 @@ describe('queued context maintenance timer', () => { await waitFor(() => framework.getContextMaintenanceSnapshot().history.length >= 1); const run = framework.getContextMaintenanceSnapshot().history[0].agents[0]; assert.equal(run.ticks, 0); - assert.equal(run.error, 'the store switched branches under the tick'); - assert.equal(run.readyAfter, undefined); - assert.deepEqual(alerts, []); + assert.equal(run.error, 'the tick failed while a surgery waited'); + assert.deepEqual(alerts, ['context-maintenance-failed']); assert.ok(logged.some((line) => - line.includes('agent=agent stopped while a rollback holds the store') - && line.includes('the store switched branches under the tick'))); + line.includes('[context-maintenance] agent=agent failed:') + && line.includes('the tick failed while a surgery waited'))); } finally { console.error = consoleError; internals.surgeryHold = null; diff --git a/test/offline-recovery-branch.test.ts b/test/offline-recovery-branch.test.ts index 1f3e5bbf4..640cc0f59 100644 --- a/test/offline-recovery-branch.test.ts +++ b/test/offline-recovery-branch.test.ts @@ -511,3 +511,90 @@ test('suppressing a one-shard copy of a reposted document keeps the other', asyn rmSync(dir, { recursive: true, force: true }); } }); + +test('a suppressed message outside any body is removed alone, and a one-shard copy as a range', async (t) => { + const dir = mkdtempSync(join(tmpdir(), 'offline-recovery-removal-kind-')); + const storePath = join(dir, 'agent.chronicle'); + try { + const store = JsStore.openOrCreate({ path: storePath }); + const cm = await ContextManager.open({ store, namespace: 'agents/cairn' }); + cm.addMessage('user', [{ type: 'text', text: 'before' }], { + serverId: 'discord', channelId: 'discord:g1:c1', messageId: 'm-before', + }); + const plain = String(cm.addMessage('user', [{ type: 'text', text: 'plain' }], { + serverId: 'discord', channelId: 'discord:g1:c1', messageId: 'm-plain', + })); + appendShards(cm, 'one-shard-group', ['m-doc-1']); + const [copy] = appendShards(cm, 'one-shard-group', ['m-doc-2']); + cm.addMessage('user', [{ type: 'text', text: 'current' }], { + serverId: 'discord', channelId: 'discord:g1:c1', messageId: 'm-current', + }); + cm.close(); + store.close(); + + // removeMessage's remove event also drops the context-log entries copied + // from the message; context-manager refuses it for any shard. + const removeMessage = t.mock.method(ContextManager.prototype, 'removeMessage'); + const removeMessages = t.mock.method(ContextManager.prototype, 'removeMessages'); + const result = await createOfflineRecoveryBranch({ + storePath, + agentName: 'cairn', + messageId: 'm-current', + suppressMessageIds: ['m-plain', 'm-doc-2'], + }); + + assert.equal(result.messagesSuppressed, 2); + assert.deepEqual(removeMessage.mock.calls.map((call) => String(call.arguments[0])), [plain]); + assert.deepEqual(removeMessages.mock.calls.map((call) => call.arguments.map(String)), [[copy, copy]]); + removeMessage.mock.restore(); + removeMessages.mock.restore(); + const recoveredStore = JsStore.openOrCreate({ path: storePath }); + const recoveredCm = await ContextManager.open({ store: recoveredStore, namespace: 'agents/cairn' }); + assert.deepEqual( + recoveredCm.getAllMessages().map((message) => message.metadata?.messageId), + ['m-before', 'm-doc-1', 'm-current'], + ); + recoveredCm.close(); + recoveredStore.close(); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); + +test('an anchor at the end of a write that was cut short keeps that partial body and leaves out its retry', async () => { + const dir = mkdtempSync(join(tmpdir(), 'offline-recovery-cut-short-')); + const storePath = join(dir, 'agent.chronicle'); + try { + const store = JsStore.openOrCreate({ path: storePath }); + const cm = await ContextManager.open({ store, namespace: 'agents/cairn' }); + cm.addMessage('user', [{ type: 'text', text: 'before' }], { + serverId: 'discord', channelId: 'discord:g1:c1', messageId: 'm-before', + }); + // Two of a body's shards, written before an interruption, then its retry + // from shard 0: the retry is a body of its own. + const partial = appendShards(cm, 'cut-group', ['m-cut-0', 'm-cut-1']); + appendShards(cm, 'cut-group', ['m-retry-0', 'm-retry-1', 'm-retry-2']); + cm.close(); + store.close(); + + const result = await createOfflineRecoveryBranch({ + storePath, + agentName: 'cairn', + contextId: partial[1], + branchName: 'recovery/cairn/cut-short', + }); + + assert.equal(result.messagesRemoved, 3); + assert.deepEqual(result.refs.map((ref) => ref.messageId), ['m-retry-0', 'm-retry-1', 'm-retry-2']); + const recoveredStore = JsStore.openOrCreate({ path: storePath }); + const recoveredCm = await ContextManager.open({ store: recoveredStore, namespace: 'agents/cairn' }); + assert.deepEqual( + recoveredCm.getAllMessages().map((message) => message.metadata?.messageId), + ['m-before', 'm-cut-0', 'm-cut-1'], + ); + recoveredCm.close(); + recoveredStore.close(); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); diff --git a/test/operator-surgery.test.ts b/test/operator-surgery.test.ts index aeb162b59..64da53d59 100644 --- a/test/operator-surgery.test.ts +++ b/test/operator-surgery.test.ts @@ -372,6 +372,42 @@ describe('live operator surgery', () => { void s0; }); + it('hide removes a one-shard body, which removeMessage alone refuses', async () => { + const [lone] = (['lone'] as const).map((text) => + (cm() as unknown as ShardStore).messageStore.append( + 'user', + [{ type: 'text', text }], + { serverId: 'discord', channelId: 'discord:g1:c1', messageId: 'd-lone' }, + undefined, + { bodyGroupId: 'g-lone', shardIndex: 0 }, + ).id, + ); + const internals = framework as unknown as { + handleHostCommand(serverId: string, params: Record): Promise>; + }; + const r = await internals.handleHostCommand('discord', { + command: 'hide', + agentName: 'scout', + fromMessageId: 'd-lone', + requesterName: 'antra', + }); + assert.equal(r.ok, true, String(r.error)); + assert.equal(r.hidden, 1); + assert.equal(cm().getMessage(lone as never), null); + assert.deepEqual(remainingTexts(), ['m1', 'm2', 'm3', 'm4', 'm5']); + }); + + it('a rewind whose newest message is the last shard of a body takes the whole body', async () => { + const [s0, s1, s2] = appendShards('g-doc', ['d0', 'd1', 'd2']); + const internals = framework as unknown as { + shedNewestTurn(agent: unknown): { kind: string; removedIds: string[] } | null; + }; + const record = internals.shedNewestTurn(framework.getAgent('scout')); + assert.ok(record); + assert.deepEqual(record.removedIds, [s2, s1, s0]); + assert.deepEqual(remainingTexts(), ['m1', 'm2', 'm3', 'm4', 'm5']); + }); + // --- failure path ---------------------------------------------------------- it('a failed suppression restores the source branch, retires its outbox batch, and leaves the store bootable', async () => { From d24a9816cc0c19883cc450d8f7f2fdb16c6f7c65 Mon Sep 17 00:00:00 2001 From: Ada-1017 Date: Wed, 14 Oct 2026 09:26:57 -0400 Subject: [PATCH 10/30] fix(hide): a message stored as a body is hidden whole A long message is stored as a body of shards, each carrying the message's Discord id. hide found the first shard and removed it alone, which context-manager refuses for any shard, so hiding a long message failed, and a range whose end fell inside a body was refused as splitting it. - hide widens what it removes to whole bodies (bodyRangeIn): a single message to its body, a range's ends to their bodies' first and last shards. It reports the number removed and the body's one Discord ref. - stored-removal's bodyStartIn becomes bodyRangeIn (first and last), which the rewind reads the start of. - operator-surgery: hide removes a one-shard body and a three-shard body whole, by their Discord ids. --- src/framework.ts | 26 ++++++++++++++--------- src/stored-removal.ts | 11 +++++----- test/operator-surgery.test.ts | 40 +++++++++++++++++++++++------------ 3 files changed, 48 insertions(+), 29 deletions(-) diff --git a/src/framework.ts b/src/framework.ts index 707845300..43d6404b3 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -19,7 +19,7 @@ import type { StoredMessage, TokenBudget, } from '@animalabs/context-manager'; -import { bodyStartIn, removeStoredMessages } from './stored-removal.js'; +import { bodyRangeIn, removeStoredMessages } from './stored-removal.js'; import type { FrameworkConfig, InferencePolicy, @@ -5691,7 +5691,7 @@ export class AgentFramework { // The newest message can be the last shard of a body (a document stored // in shards). context-manager removes a shard only with the rest of its // body, so the rewind takes the whole body. - const first = bodyStartIn(all, idx); + const first = bodyRangeIn(all, idx).from; const members = all.slice(first, idx + 1); const msg = all[idx]; const md = (msg.metadata ?? {}) as { @@ -6107,10 +6107,13 @@ export class AgentFramework { if (toIdx < 0) { return { ok: false, error: `Message ${params.toMessageId} is not an addressable message in context.` }; } - const [lo, hi] = fromIdx <= toIdx ? [fromIdx, toIdx] : [toIdx, fromIdx]; + const [first, last] = fromIdx <= toIdx ? [fromIdx, toIdx] : [toIdx, fromIdx]; + // A message stored as a body (a long one, in shards) is hidden whole. + const lo = bodyRangeIn(all, first).from; + const hi = bodyRangeIn(all, last).to; const refs = refsIn(lo, hi); const rangeIds = all.slice(lo, hi + 1).map((m) => String(m.id)); - cm.removeMessages(all[lo].id, all[hi].id); + removeStoredMessages(cm, all[lo], all[hi]); console.error( `[host-command] hide agent=${agentName} range removed=${hi - lo + 1} ` + `(${params.fromMessageId}..${params.toMessageId}) by=${params.requesterName ?? params.requesterId ?? 'unknown'} (server=${serverId})`, @@ -6135,11 +6138,14 @@ export class AgentFramework { lastVisible: await this.lastVisiblePreview(agentName), }; } - const refs = refsIn(fromIdx, fromIdx); - const hiddenId = String(all[fromIdx].id); - removeStoredMessages(cm, all[fromIdx], all[fromIdx]); + // A message stored as a body (a long one, in shards) is hidden whole: + // each shard carries its Discord id, and the first is found. + const { from: lo, to: hi } = bodyRangeIn(all, fromIdx); + const refs = refsIn(lo, lo); + const hiddenIds = all.slice(lo, hi + 1).map((m) => String(m.id)); + removeStoredMessages(cm, all[lo], all[hi]); console.error( - `[host-command] hide agent=${agentName} removed=1 (${params.fromMessageId}) ` + + `[host-command] hide agent=${agentName} removed=${hi - lo + 1} (${params.fromMessageId}) ` + `by=${params.requesterName ?? params.requesterId ?? 'unknown'} (server=${serverId})`, ); this.recordOperatorAction({ @@ -6147,11 +6153,11 @@ export class AgentFramework { agent: agentName, requester: hostCommandRequester(serverId, params), params: { fromMessageId: params.fromMessageId }, - result: { branch: this.store.currentBranch().name, hidden: 1, removedIds: [hiddenId] }, + result: { branch: this.store.currentBranch().name, hidden: hi - lo + 1, removedIds: hiddenIds }, }); return { ok: true, - hidden: 1, + hidden: hi - lo + 1, hiddenRefs: refs, lastVisible: await this.lastVisiblePreview(agentName), }; diff --git a/src/stored-removal.ts b/src/stored-removal.ts index 5c840774b..93edff287 100644 --- a/src/stored-removal.ts +++ b/src/stored-removal.ts @@ -28,10 +28,11 @@ export function removeStoredInterval(cm: ContextManager, fromId: MessageId, toId } /** - * The first position of the body that holds `messages[index]`, read the way - * context-manager 0.14.0 reads bodies (`continuesBody`, pairwise in store - * order). A message outside any body is a body of its own. + * The first and last positions (inclusive) of the body that holds + * `messages[index]`, read the way context-manager 0.14.0 reads bodies + * (`continuesBody`, pairwise in store order). A message outside any body is a + * body of its own. */ -export function bodyStartIn(messages: readonly unknown[], index: number): number { - return bodyBoundsIn(messages as readonly (BodyShard | null | undefined)[], index).from; +export function bodyRangeIn(messages: readonly unknown[], index: number): { from: number; to: number } { + return bodyBoundsIn(messages as readonly (BodyShard | null | undefined)[], index); } diff --git a/test/operator-surgery.test.ts b/test/operator-surgery.test.ts index 64da53d59..fc9445fbf 100644 --- a/test/operator-surgery.test.ts +++ b/test/operator-surgery.test.ts @@ -372,28 +372,40 @@ describe('live operator surgery', () => { void s0; }); - it('hide removes a one-shard body, which removeMessage alone refuses', async () => { - const [lone] = (['lone'] as const).map((text) => - (cm() as unknown as ShardStore).messageStore.append( - 'user', - [{ type: 'text', text }], - { serverId: 'discord', channelId: 'discord:g1:c1', messageId: 'd-lone' }, - undefined, - { bodyGroupId: 'g-lone', shardIndex: 0 }, - ).id, - ); + it('hide removes a message stored as a body whole, a one-shard body included', async () => { + // Each shard of a long message carries its Discord message id. + const appendBody = (discordId: string, group: string, texts: string[]): string[] => + texts.map((text, shardIndex) => + (cm() as unknown as ShardStore).messageStore.append( + 'user', + [{ type: 'text', text }], + { serverId: 'discord', channelId: 'discord:g1:c1', messageId: discordId }, + undefined, + { bodyGroupId: group, shardIndex }, + ).id, + ); + const [lone] = appendBody('d-lone', 'g-lone', ['lone']); + const long = appendBody('d-long', 'g-long', ['l0', 'l1', 'l2']); const internals = framework as unknown as { handleHostCommand(serverId: string, params: Record): Promise>; }; - const r = await internals.handleHostCommand('discord', { + const hide = (fromMessageId: string) => internals.handleHostCommand('discord', { command: 'hide', agentName: 'scout', - fromMessageId: 'd-lone', + fromMessageId, requesterName: 'antra', }); - assert.equal(r.ok, true, String(r.error)); - assert.equal(r.hidden, 1); + + const one = await hide('d-lone'); + assert.equal(one.ok, true, String(one.error)); + assert.equal(one.hidden, 1); assert.equal(cm().getMessage(lone as never), null); + + const whole = await hide('d-long'); + assert.equal(whole.ok, true, String(whole.error)); + assert.equal(whole.hidden, 3); + assert.deepEqual((whole.hiddenRefs as Array<{ messageId: string }>).map((ref) => ref.messageId), ['d-long']); + for (const id of long) assert.equal(cm().getMessage(id as never), null); assert.deepEqual(remainingTexts(), ['m1', 'm2', 'm3', 'm4', 'm5']); }); From 2fd8d6e9f57e673d9985546fdaf9890e3c08a5f2 Mon Sep 17 00:00:00 2001 From: Ada-1017 Date: Wed, 14 Oct 2026 09:26:58 -0400 Subject: [PATCH 11/30] docs(changelog): the round's maintenance, resume and removal changes; what 0.14.0 changed for hosts, under 0.21.1 - cm-0.14.changed.md: a throw while preparing one agent's maintenance is that agent's failure; a lasting failure is logged and alerted once, then again when its text changes or after the agent's maintenance has gone without one; a surgery waits for a running pass before it moves the branch, and resume() refuses as agent-busy while a surgery holds the store. - cm-0.14.fixed.md: why resume() gives an agent whose settings can't be read no verdict; offline recovery's removal sentence moves to the new fragment. - body-removals.fixed.md (new): removing a single message of a body, a one-shard body's included, goes as a range in offline recovery and a resumed Discord suppression; hide hides a message stored as a body whole; a rewind takes the whole body of its newest shard. A message outside any body is still removed alone, with its context-log entries. - CHANGELOG 0.21.1: the operator note says how to tell the two kinds of store apart (the first holds no carriers and can't be told from its record; the second shows reset.discardedMultiplier, also in RenderStats.calibration). A second note, for hosts, carries what else context-manager 0.14.0 changed for them: dry-run compiles leave the estimate, calibration and draft markers alone; the state queries throw while initialization is pending or has failed; a reposted sharded document misses the prompt cache once. The earlier round had dropped these from its fragment without moving them. --- CHANGELOG.md | 27 ++++++++++++++++++++------- changelog.d/body-removals.fixed.md | 1 + changelog.d/cm-0.14.changed.md | 3 ++- changelog.d/cm-0.14.fixed.md | 4 ++-- 4 files changed, 25 insertions(+), 10 deletions(-) create mode 100644 changelog.d/body-removals.fixed.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 185ad3da5..a63468872 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -517,13 +517,26 @@ Releases up to and including 0.7.3 predate this file; for their contents see context-manager 0.13 or earlier** (added after the release): 0.14.0 stamps its persisted calibration multiplier with the pricing it was learned under. An unstamped multiplier is kept and re-stamped, unless the store holds - XML-history carriers, in which case it restarts from 1.0. Two kinds of - store can then refuse every compile with `OverBudgetError` before - inference: one whose multiplier was last learned before context-manager - 0.11.0, and one that holds XML-history carriers and had learned a - multiplier below 1. With the host stopped, reset the first kind's - multiplier once, and re-stamp the second kind's checked value, as the - **Operators** entry of context-manager 0.14.0's changelog describes. + XML-history carriers, in which case it restarts from 1.0 and the value it + discarded is kept as `reset.discardedMultiplier` in the calibration state + (shown in `RenderStats.calibration`). Two kinds of store can then refuse + every compile with `OverBudgetError` before inference. One holds no + carriers and kept a multiplier last learned before its host moved to + context-manager 0.11.0; its record can't show that, so tell it by when + the host moved. The other holds carriers and had learned a multiplier + below 1, which its `reset.discardedMultiplier` shows. With the host + stopped, reset the first kind's multiplier once, and re-stamp the second + kind's checked value, as the **Operators** entry of context-manager + 0.14.0's changelog describes. + - **What else hosts meet with context-manager 0.14.0** (added after the + release): a preview, or any compile made with `dryRun`, leaves the context + manager's estimate, its calibration and a pending kv-unified draft's marker + indices as it found them. The context manager's state queries (`isReady`, + `getPendingWork`, `getHotContextSettings`, `previewContext` and others) + throw while its strategy's initialization is pending or has failed, where + they used to answer. A reposted sharded document renders as two bodies, so + a stored conversation holding one misses the prompt cache once after the + upgrade. ## 0.21.0 — 2026-10-03 diff --git a/changelog.d/body-removals.fixed.md b/changelog.d/body-removals.fixed.md new file mode 100644 index 000000000..dd6716ab4 --- /dev/null +++ b/changelog.d/body-removals.fixed.md @@ -0,0 +1 @@ +- Removing a single message that belongs to a body, a one-shard body's included, no longer fails: context-manager's `removeMessage` refuses any message in a body, so agent-framework now removes one as a range in offline recovery's suppressions, and in a Discord suppression resumed after a crash. The host `hide` command now hides a message stored as a body (a long one, in shards) whole, and an automatic rewind whose newest message is a shard of a body takes the whole body. A message outside any body is still removed alone, so the context-log entries copied from it go with it. diff --git a/changelog.d/cm-0.14.changed.md b/changelog.d/cm-0.14.changed.md index d1dd33fcf..1a5c187e4 100644 --- a/changelog.d/cm-0.14.changed.md +++ b/changelog.d/cm-0.14.changed.md @@ -1 +1,2 @@ -- A context-maintenance pass no longer fails as a whole when one agent's readiness can't be read. context-manager 0.14, which agent-framework has required since 0.21.1, refuses its readiness queries while a strategy's initialization is pending (inside `switchBranch`, `fork`, `setStrategy` or `ContextManager.open`) or has failed, and a strategy can refuse on its own, as the autobiographical one does for a branch it hasn't reloaded. That agent's run now records the error, the agent gets its own `context-maintenance-failed` alert (`scope: 'agent'`), and the other agents run. Before, such a throw ended the pass for every agent, with one alert for the whole pass (`scope: 'pass'`). While an operator surgery holds the store, no pass starts and a running pass starts no new tick, a failure met then is logged without an alert, and `maintenanceTick()` answers `ran: false`. `getContextMaintenanceSnapshot()` reports such an agent as `ready: false` with an `error`, and a strategy that can't give a progress snapshot reads as having none. +- A context-maintenance pass no longer fails as a whole when one agent's readiness can't be read. context-manager 0.14, which agent-framework has required since 0.21.1, refuses its readiness queries while a strategy's initialization is pending (inside `switchBranch`, `fork`, `setStrategy` or `ContextManager.open`) or has failed, and a strategy can refuse on its own, as the autobiographical one does for a branch it hasn't reloaded. That agent's run now records the error, the agent gets its own `context-maintenance-failed` alert (`scope: 'agent'`), and the other agents run. Before, such a throw ended the pass for every agent, with one alert for the whole pass (`scope: 'pass'`). Anything else that throws while one agent's maintenance is prepared is that agent's failure in the same way. A failure that lasts, met by every pass, is recorded on each run, and logged and alerted once, then again when its text changes or after that agent's maintenance has gone without a failure. `getContextMaintenanceSnapshot()` reports such an agent as `ready: false` with an `error`, and a strategy that can't give a progress snapshot reads as having none. +- While an operator surgery (`rollbackToMessage` or `suppressMessages`) holds the store, no maintenance pass starts, a running pass starts no new tick, and the surgery waits for that pass to finish before it switches or forks the branch, so no maintenance work overlaps the move. `maintenanceTick()` answers `ran: false` meanwhile, and `resume()` refuses with `OperatorActionError` (`agent-busy`), as the other operator calls do. Resume once the surgery has finished. diff --git a/changelog.d/cm-0.14.fixed.md b/changelog.d/cm-0.14.fixed.md index c2f73fa5f..a4e121c59 100644 --- a/changelog.d/cm-0.14.fixed.md +++ b/changelog.d/cm-0.14.fixed.md @@ -1,2 +1,2 @@ -- `resume()` no longer throws when one agent's runtime settings can't be read, as when its context manager (0.14, required since 0.21.1) refuses while its strategy's initialization is pending or has failed. It warns, naming the agent and the error, and proceeds without a feasibility verdict for that agent, as it does for an agent whose strategy can't preview. -- Offline recovery (`createOfflineRecoveryBranch`) reads a sharded message's body as context-manager 0.14.0 does: a message continues the body before it only when it carries the same group id at a higher shard index. A document posted twice stores two copies under one group id, so an anchor at the end of one copy, or a suppression of one copy alone, is no longer refused. A selection inside one copy is still refused. An anchor at the end of a write that was cut short keeps that partial body and leaves out its retry, which context-manager 0.14.0 reads as a body of its own. Each suppressed interval is now removed as a range, so suppressing a one-shard copy no longer fails at removal. +- `resume()` no longer throws when one agent's runtime settings can't be read, as when its context manager (0.14, required since 0.21.1) refuses after its strategy failed to initialize. It warns, naming the agent and the error, and proceeds without a feasibility verdict for that agent: the verdict guards against an agent that would run over budget, and an agent whose manager refuses its compiles can't run. That manager refuses the agent's turns and writes until a `switchBranch` or `setStrategy` succeeds, and maintenance reports it. +- Offline recovery (`createOfflineRecoveryBranch`) reads a sharded message's body as context-manager 0.14.0 does: a message continues the body before it only when it carries the same group id at a higher shard index. A document posted twice stores two copies under one group id, so an anchor at the end of one copy, or a suppression of one copy alone, is no longer refused. A selection inside one copy is still refused. An anchor at the end of a write that was cut short keeps that partial body and leaves out its retry, which context-manager 0.14.0 reads as a body of its own. From 82859d5f60025dfb121079d49e7522800f86893d Mon Sep 17 00:00:00 2001 From: Ada-1017 Date: Wed, 14 Oct 2026 09:42:02 -0400 Subject: [PATCH 12/30] fix(resume, surgeries): wait only for a maintenance pass that is running The round awaited settleMaintenancePass() unconditionally. An await always yields, so resume() no longer reached its first preview in the call's synchronous part, which main's (#325) "a resume checks again after its preview retries, and refuses when stop() began during them" relies on: it asserts the preview ran before it calls stop(). The merge of main brought that test in, and it failed on both Nodes. resume() and both surgeries now await the pass only when one is running, as resume() did before the round. No pass starts while a surgery holds the store, so a surgery that finds none has nothing to wait for. --- src/framework.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/framework.ts b/src/framework.ts index b32287189..6d750e245 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -4657,7 +4657,7 @@ export class AgentFramework { // ("already running"), which used to degrade the gate to "no opinion". // Wait for the pass first, then retry a few times for the case where a // fresh pass started in between. - await this.settleMaintenancePass(); + if (this.maintenancePass) await this.settleMaintenancePass(); const failing: Array<{ agentName: string; preview: RuntimeSettingsPreview }> = []; for (const agentName of this.agents.keys()) { let preview: RuntimeSettingsPreview; @@ -7828,7 +7828,7 @@ export class AgentFramework { let markers: SurgeryMarkerReceipt; try { // A maintenance tick already running finishes before the switch. - await this.settleMaintenancePass(); + if (this.maintenancePass) await this.settleMaintenancePass(); // Prepare the external side effect before switching Chronicle. If the // process dies after the switch but before activate(), startup // promotes this batch by matching targetBranch to the active branch. @@ -7998,7 +7998,7 @@ export class AgentFramework { let markers: SurgeryMarkerReceipt; try { // A maintenance tick already running finishes before the fork. - await this.settleMaintenancePass(); + if (this.maintenancePass) await this.settleMaintenancePass(); const markerBatch = this.discordAwarenessOutbox?.prepare({ agentName, sourceBranch, From 9d61bbc47421f829f0d863baa45b898124ce74b0 Mon Sep 17 00:00:00 2001 From: Ada-1017 Date: Wed, 14 Oct 2026 09:52:06 -0400 Subject: [PATCH 13/30] fix(maintenance): switchBranch waits for a running pass before it switches, as the surgeries do main's switchBranch (#325) reserves the store and switches it at once; the merge of main brought it beside rollbackToMessage and suppressMessages, which this branch has wait for a running maintenance pass. It now waits the same way, so no strategy tick runs across an operator's switch either. The surgery-waits test gains a switchBranch variant (a branch made with branchAt, then switched to) and passes the reservation's 'every agent' to resume()'s refusal check. --- src/framework.ts | 2 ++ test/maintenance-timer.test.ts | 12 ++++++++++-- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/src/framework.ts b/src/framework.ts index 6d750e245..dabc9222a 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -7430,6 +7430,8 @@ export class AgentFramework { this.refuseOperatorAfterStop('switch branch'); const release = this.reserveStoreForSurgery('switch branch', 'every agent'); try { + // A maintenance tick already running finishes before the switch. + if (this.maintenancePass) await this.settleMaintenancePass(); this.store.switchBranch(name); // Reversible Discord awareness markers are part of the branch // projection. Their delivery starts behind the data-plane gate, which diff --git a/test/maintenance-timer.test.ts b/test/maintenance-timer.test.ts index 7fad80be8..a3ef18cf0 100644 --- a/test/maintenance-timer.test.ts +++ b/test/maintenance-timer.test.ts @@ -817,7 +817,15 @@ describe('queued context maintenance timer', () => { for (const surgery of [ { verb: 'roll back', run: (framework: AgentFramework, first: string) => framework.rollbackToMessage('agent', { messageId: first }) }, { verb: 'suppress', run: (framework: AgentFramework, first: string) => framework.suppressMessages('agent', { messageIds: [first] }) }, - ]) { + { + verb: 'switch branch', + who: 'every agent', + run: (framework: AgentFramework, first: string) => { + framework.getAgent('agent')!.getContextManager().branchAt(first as never, 'elsewhere'); + return framework.switchBranch('elsewhere'); + }, + }, + ] as Array<{ verb: string; who?: string; run: (framework: AgentFramework, first: string) => Promise }>) { it(`a ${surgery.verb} that begins during a tick waits for it before it moves the branch, and resume waits for the ${surgery.verb}`, { timeout: 10_000 }, async () => { const dir = mkdtempSync(join(tmpdir(), 'af-maintenance-surgery-waits-')); const strategy = new GatedTickStrategy(); @@ -843,7 +851,7 @@ describe('queued context maintenance timer', () => { await assert.rejects( framework.resume(), (error: unknown) => error instanceof OperatorActionError && error.code === 'agent-busy' - && error.message.includes(`a live ${surgery.verb} for agent holds the store`), + && error.message.includes(`a live ${surgery.verb} for ${surgery.who ?? 'agent'} holds the store`), ); assert.equal(cm.currentBranch().name, 'main', `the ${surgery.verb} waits for the running tick`); strategy.release(); From 41a2a3203380f7fd233ba52145d97ec04279a22d Mon Sep 17 00:00:00 2001 From: Ada-1017 Date: Wed, 14 Oct 2026 10:44:19 -0400 Subject: [PATCH 14/30] fix(maintenance, rewind): drain kicks are maintenance too; resume checks the hold after its waits; a rewind takes the whole exchange in one removal From the third review of the cm-0.14 round: - The over-budget drain kick ran up to eight strategy ticks in a loop that nothing waited for and no hold stopped, so a branch move could run across one. Its loop now starts no tick while a surgery holds the store, no kick starts under the hold, and the kicks are tracked: a surgery or switchBranch waits for the pass and every kick in flight (maintenanceRunning, settleMaintenancePass). - resume() checked the surgery hold only on entry; a surgery that took the store while resume waited for a pass or a preview went unnoticed. It checks again after the waits, before it reopens anything. - The pass waits for every agent's work to settle, not for the first rejection (Promise.allSettled), so a throw while reporting one agent's failure can't end the pass while others still tick. - A lasting failure's dedupe compares the whole text, not the bounded one, and alerts again once opsAlert's cooldown has passed, so the webhook, which posts a kind at most once per cooldown, hears a failure that lasts. - The rewind pairs a tool_result with the whole tool_use turn before it: a turn stored as a body carries its tool_use on its first shard. The exchange, the turn's first shard through the result's last, goes in one removeMessages, so a refusal can't leave it half applied. - Fragments: switchBranch and the drain kicks in the hold bullet, the cooldown in the lasting-failure sentence; the context-log clause in the body-removals fragment goes (context-manager 0.14.0 writes no persistent context-log entries, so there are none to keep or drop). - Tests: a rewind of a sharded tool exchange (one range); the surgery-wait test's name says resume refuses; the rewind harnesses' fake manager gains removeMessages, and the drain-breaker harness the drains set. --- changelog.d/body-removals.fixed.md | 2 +- changelog.d/cm-0.14.changed.md | 4 +- src/framework.ts | 102 ++++++++++++++++------- test/inference-exhaustion-rewind.test.ts | 7 ++ test/maintenance-timer.test.ts | 2 +- test/operator-surgery.test.ts | 31 +++++++ test/overbudget-drain-breaker.test.ts | 1 + test/refusal-rewind.test.ts | 7 ++ 8 files changed, 120 insertions(+), 36 deletions(-) diff --git a/changelog.d/body-removals.fixed.md b/changelog.d/body-removals.fixed.md index dd6716ab4..cb84f3877 100644 --- a/changelog.d/body-removals.fixed.md +++ b/changelog.d/body-removals.fixed.md @@ -1 +1 @@ -- Removing a single message that belongs to a body, a one-shard body's included, no longer fails: context-manager's `removeMessage` refuses any message in a body, so agent-framework now removes one as a range in offline recovery's suppressions, and in a Discord suppression resumed after a crash. The host `hide` command now hides a message stored as a body (a long one, in shards) whole, and an automatic rewind whose newest message is a shard of a body takes the whole body. A message outside any body is still removed alone, so the context-log entries copied from it go with it. +- Removing a single message that belongs to a body, a one-shard body's included, no longer fails: context-manager's `removeMessage` refuses any message in a body, so agent-framework now removes one as a range in offline recovery's suppressions, and in a Discord suppression resumed after a crash. The host `hide` command now hides a message stored as a body (a long one, in shards) whole, and an automatic rewind whose newest message is a shard of a body takes the whole body. diff --git a/changelog.d/cm-0.14.changed.md b/changelog.d/cm-0.14.changed.md index 1a5c187e4..bd71be775 100644 --- a/changelog.d/cm-0.14.changed.md +++ b/changelog.d/cm-0.14.changed.md @@ -1,2 +1,2 @@ -- A context-maintenance pass no longer fails as a whole when one agent's readiness can't be read. context-manager 0.14, which agent-framework has required since 0.21.1, refuses its readiness queries while a strategy's initialization is pending (inside `switchBranch`, `fork`, `setStrategy` or `ContextManager.open`) or has failed, and a strategy can refuse on its own, as the autobiographical one does for a branch it hasn't reloaded. That agent's run now records the error, the agent gets its own `context-maintenance-failed` alert (`scope: 'agent'`), and the other agents run. Before, such a throw ended the pass for every agent, with one alert for the whole pass (`scope: 'pass'`). Anything else that throws while one agent's maintenance is prepared is that agent's failure in the same way. A failure that lasts, met by every pass, is recorded on each run, and logged and alerted once, then again when its text changes or after that agent's maintenance has gone without a failure. `getContextMaintenanceSnapshot()` reports such an agent as `ready: false` with an `error`, and a strategy that can't give a progress snapshot reads as having none. -- While an operator surgery (`rollbackToMessage` or `suppressMessages`) holds the store, no maintenance pass starts, a running pass starts no new tick, and the surgery waits for that pass to finish before it switches or forks the branch, so no maintenance work overlaps the move. `maintenanceTick()` answers `ran: false` meanwhile, and `resume()` refuses with `OperatorActionError` (`agent-busy`), as the other operator calls do. Resume once the surgery has finished. +- A context-maintenance pass no longer fails as a whole when one agent's readiness can't be read. context-manager 0.14, which agent-framework has required since 0.21.1, refuses its readiness queries while a strategy's initialization is pending (inside `switchBranch`, `fork`, `setStrategy` or `ContextManager.open`) or has failed, and a strategy can refuse on its own, as the autobiographical one does for a branch it hasn't reloaded. That agent's run now records the error, the agent gets its own `context-maintenance-failed` alert (`scope: 'agent'`), and the other agents run. Before, such a throw ended the pass for every agent, with one alert for the whole pass (`scope: 'pass'`). Anything else that throws while one agent's maintenance is prepared is that agent's failure in the same way. A failure that lasts, met by every pass, is recorded on each run, and logged and alerted once, then again when its text changes, every 15 minutes while it lasts (the alert webhook's cooldown for it), or after that agent's maintenance has gone without a failure. `getContextMaintenanceSnapshot()` reports such an agent as `ready: false` with an `error`, and a strategy that can't give a progress snapshot reads as having none. +- While an operator surgery (`rollbackToMessage` or `suppressMessages`) or `switchBranch` holds the store, no maintenance pass starts and no over-budget drain kick is started, a running pass or drain kick starts no new tick, and the operator call waits for them to finish before it switches or forks the branch, so no maintenance work overlaps the move. `maintenanceTick()` answers `ran: false` meanwhile, and `resume()` refuses with `OperatorActionError` (`agent-busy`), as the other operator calls do, including when the surgery took the store while `resume()` waited. Resume once the surgery has finished. diff --git a/src/framework.ts b/src/framework.ts index dabc9222a..323c4f875 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -1596,9 +1596,10 @@ export class AgentFramework { private maintenanceRunId = 0; private currentMaintenanceRun: ContextMaintenanceRun | null = null; private maintenanceHistory: ContextMaintenanceRun[] = []; - /** Each agent's last reported maintenance failure, by its recorded text, - * until its maintenance goes without one (reportMaintenanceFailure). */ - private maintenanceFailureReported = new Map(); + /** Each agent's last reported maintenance failure, its whole text and when + * it was alerted, until its maintenance goes without one + * (reportMaintenanceFailure). */ + private maintenanceFailureReported = new Map(); private providerGates: Map = new Map(); private providerAccelerationCooldowns: Map = new Map(); private providerAccelerationRecoveries: Map = new Map(); @@ -1758,6 +1759,8 @@ export class AgentFramework { * tick counts in the success log stop meaning anything. One kick per agent * at a time; cleared when the kick settles (success or failure). */ private overBudgetDrainInFlight: Set = new Set(); + /** The over-budget drain kicks running, which a branch move waits for as it waits for a maintenance pass. */ + private overBudgetDrains = new Set>(); /** Per-agent current rewind episode: the single consolidated marker's id and * how many turns have been shed so far. One marker per episode, updated in * place (see updateRewindMarker); cleared when the episode ends. */ @@ -3394,10 +3397,23 @@ export class AgentFramework { }); } - /** Wait for a maintenance pass that is running, whatever its outcome. */ + /** resume()'s refusal while a surgery holds the store. */ + private refuseResumeUnderSurgery(): void { + if (!this.surgeryHold) return; + throw new OperatorActionError( + 'agent-busy', + `Cannot resume: a live ${this.surgeryHold.verb} for ${this.surgeryHold.agentName} holds the store — retry when it completes`, + ); + } + + /** Whether maintenance work is running: a pass, or an over-budget drain kick. */ + private maintenanceRunning(): boolean { + return this.maintenancePass !== null || this.overBudgetDrains.size > 0; + } + + /** Wait for the maintenance work that is running, whatever its outcome. */ private async settleMaintenancePass(): Promise { - const pass = this.maintenancePass; - if (pass) await pass.catch(() => {}); + await Promise.allSettled([...(this.maintenancePass ? [this.maintenancePass] : []), ...this.overBudgetDrains]); } private async runQueuedMaintenance(): Promise { @@ -3490,7 +3506,8 @@ export class AgentFramework { try { const running = queued.filter((item): item is Extract => !item.failed); - await Promise.all(running.map(async ({ agent, cm, record }) => { + // Settled, not all: the pass ends only when every agent's work has. + const outcomes = await Promise.allSettled(running.map(async ({ agent, cm, record }) => { try { for ( let i = 0; @@ -3522,6 +3539,11 @@ export class AgentFramework { if (progress) record.progressAfter = progress; } })); + for (const outcome of outcomes) { + if (outcome.status === 'rejected') { + console.error('[context-maintenance] reporting an agent\'s failure failed:', describeFailureWithFrames(outcome.reason)); + } + } } finally { run.finishedAt = Date.now(); run.agents.sort((a, b) => a.agentName.localeCompare(b.agentName)); @@ -3537,13 +3559,17 @@ export class AgentFramework { * with the whole text. A failure that lasts, such as a manager whose * strategy failed to initialize, meets every pass, a few seconds apart: * each run records it, and it is logged and alerted again only when its - * text changes, or after a pass in which that agent's maintenance went + * whole text changes, once opsAlert's cooldown for it has passed (so the + * webhook, which posts a kind at most once per cooldown, hears a failure + * that lasts), or after a pass in which that agent's maintenance went * without a failure (maintenanceFailureReported). */ private reportMaintenanceFailure(agentName: string, record: ContextMaintenanceAgentRun, wholeError: string): void { record.error = boundFailureText(wholeError); - if (this.maintenanceFailureReported.get(agentName) === record.error) return; - this.maintenanceFailureReported.set(agentName, record.error); + const reported = this.maintenanceFailureReported.get(agentName); + const now = Date.now(); + if (reported && reported.text === wholeError && now - reported.at < this.opsAlertCooldownMs) return; + this.maintenanceFailureReported.set(agentName, { text: wholeError, at: now }); console.error(`[context-maintenance] agent=${agentName} failed:`, record.error); this.opsAlert( 'context-maintenance-failed', @@ -4644,20 +4670,17 @@ export class AgentFramework { if (!this.quiesced) return this.getHostModeStatus(); // A surgery switches or forks the branch while it holds the store, and a // manager whose initialization that left pending refuses its settings - // read: resume once it's done, as the other operator calls do. - if (this.surgeryHold) { - throw new OperatorActionError( - 'agent-busy', - `Cannot resume: a live ${this.surgeryHold.verb} for ${this.surgeryHold.agentName} holds the store — retry when it completes`, - ); - } + // read: resume once it's done, as the other operator calls do. Checked + // again after the waits below, since a surgery can take the store while + // resume waits for a pass or a preview. + this.refuseResumeUnderSurgery(); // The canonical operator flow is `maintain; resume`, and a timer-driven // pass may still be mid-flight: previewContext refuses to overlap one // ("already running"), which used to degrade the gate to "no opinion". // Wait for the pass first, then retry a few times for the case where a // fresh pass started in between. - if (this.maintenancePass) await this.settleMaintenancePass(); + if (this.maintenanceRunning()) await this.settleMaintenancePass(); const failing: Array<{ agentName: string; preview: RuntimeSettingsPreview }> = []; for (const agentName of this.agents.keys()) { let preview: RuntimeSettingsPreview; @@ -4690,6 +4713,7 @@ export class AgentFramework { ); } } + this.refuseResumeUnderSurgery(); // stop() closes the queue first. If it has begun, before this call or // during the waits above, leave the host quiesced, as it will boot, // rather than reopen what stop() is closing. @@ -6537,15 +6561,22 @@ export class AgentFramework { // Determine the complete exchange to remove, newest first. const removedIds: MessageId[] = members.map((m) => m.id).reverse(); - let paired: StoredMessage | undefined; + // Where the exchange starts: this body, or the tool_use turn paired with it. + let from = first; let kind: RewindRecord['kind']; let descriptor: string; if (toolBlocks.length > 0) { kind = 'tool'; - // Pair the tool_result with the tool_use assistant turn right before it. - if (first - 1 >= 0 && hasBlock(all[first - 1], 'tool_use')) { - paired = all[first - 1]; - removedIds.push(paired.id); + // Pair the tool_result with the tool_use assistant turn right before + // it, the whole of it: a turn stored as a body carries its tool_use on + // its first shard, not its last. + if (first - 1 >= 0) { + const turn = bodyRangeIn(all, first - 1); + const turnMembers = all.slice(turn.from, turn.to + 1); + if (turnMembers.some((m) => hasBlock(m, 'tool_use'))) { + from = turn.from; + removedIds.push(...turnMembers.map((m) => m.id).reverse()); + } } const sz = toolBlocks.reduce( (n, b) => n + String((b as { content?: unknown }).content ?? '').length, 0); @@ -6564,8 +6595,8 @@ export class AgentFramework { ? { channelId: String(md.channelId), messageId: String(md.messageId) } : undefined; - removeStoredMessages(cm, all[first], msg); - if (paired !== undefined) removeStoredMessages(cm, paired, paired); + // One removal for the whole exchange, so it is never half applied. + removeStoredMessages(cm, all[from], msg); return { kind, descriptor, removedIds, discordRef }; } @@ -7431,7 +7462,7 @@ export class AgentFramework { const release = this.reserveStoreForSurgery('switch branch', 'every agent'); try { // A maintenance tick already running finishes before the switch. - if (this.maintenancePass) await this.settleMaintenancePass(); + if (this.maintenanceRunning()) await this.settleMaintenancePass(); this.store.switchBranch(name); // Reversible Discord awareness markers are part of the branch // projection. Their delivery starts behind the data-plane gate, which @@ -7830,7 +7861,7 @@ export class AgentFramework { let markers: SurgeryMarkerReceipt; try { // A maintenance tick already running finishes before the switch. - if (this.maintenancePass) await this.settleMaintenancePass(); + if (this.maintenanceRunning()) await this.settleMaintenancePass(); // Prepare the external side effect before switching Chronicle. If the // process dies after the switch but before activate(), startup // promotes this batch by matching targetBranch to the active branch. @@ -8000,7 +8031,7 @@ export class AgentFramework { let markers: SurgeryMarkerReceipt; try { // A maintenance tick already running finishes before the fork. - if (this.maintenancePass) await this.settleMaintenancePass(); + if (this.maintenanceRunning()) await this.settleMaintenancePass(); const markerBatch = this.discordAwarenessOutbox?.prepare({ agentName, sourceBranch, @@ -15653,23 +15684,30 @@ export class AgentFramework { { skipLog: true }, ); } - if (agent && overBudget && !this.overBudgetDrainInFlight.has(agentName)) { + if (agent && overBudget && !this.overBudgetDrainInFlight.has(agentName) && !this.surgeryHold) { this.overBudgetDrainInFlight.add(agentName); - void (async () => { + // Like a maintenance pass: it starts no tick while a surgery holds the + // store, and a branch move waits for the tick in flight. + const drain = (async () => { let ticks = 0; try { const cm = agent.getContextManager(); - while (ticks < 8) { + while (ticks < 8 && !this.surgeryHold) { await cm.tick(); ticks++; } - console.error(`[inference-failed] drain kicked for ${agentName} (OverBudget breaker, ${ticks} ticks)`); + console.error( + `[inference-failed] drain kicked for ${agentName} (OverBudget breaker, ${ticks} ticks` + + `${ticks < 8 ? ', stopped while a surgery holds the store' : ''})`, + ); } catch (err) { console.error(`[inference-failed] drain kick failed for ${agentName} after ${ticks} ticks: ${describeFailureWithFrames(err)}`); } finally { this.overBudgetDrainInFlight.delete(agentName); } })(); + this.overBudgetDrains.add(drain); + void drain.finally(() => { this.overBudgetDrains.delete(drain); }); } // (3) Hard-down escalation on repeated identical failure. diff --git a/test/inference-exhaustion-rewind.test.ts b/test/inference-exhaustion-rewind.test.ts index 0be5ad778..61f54b87d 100644 --- a/test/inference-exhaustion-rewind.test.ts +++ b/test/inference-exhaustion-rewind.test.ts @@ -32,6 +32,13 @@ function makeHarness(opts?: { maxRewinds?: number; messages?: any[] }) { const i = messages.findIndex((m) => m.id === id); if (i >= 0) messages.splice(i, 1); }, + // A range by its two ends, inclusive, as context-manager's removeMessages. + removeMessages: (fromId: string, toId: string) => { + const from = messages.findIndex((m) => m.id === fromId); + const to = messages.findIndex((m) => m.id === toId); + if (from < 0 || to < from) throw new Error(`no range ${fromId}..${toId}`); + for (const m of messages.splice(from, to - from + 1)) removed.push(m.id); + }, addMessage: (_p: string, content: any[], meta: any) => { const id = `marker-${added.length}`; added.push({ content, meta }); diff --git a/test/maintenance-timer.test.ts b/test/maintenance-timer.test.ts index a3ef18cf0..616703d03 100644 --- a/test/maintenance-timer.test.ts +++ b/test/maintenance-timer.test.ts @@ -826,7 +826,7 @@ describe('queued context maintenance timer', () => { }, }, ] as Array<{ verb: string; who?: string; run: (framework: AgentFramework, first: string) => Promise }>) { - it(`a ${surgery.verb} that begins during a tick waits for it before it moves the branch, and resume waits for the ${surgery.verb}`, { timeout: 10_000 }, async () => { + it(`a ${surgery.verb} that begins during a tick waits for it before it moves the branch, and resume refuses while the ${surgery.verb} holds the store`, { timeout: 10_000 }, async () => { const dir = mkdtempSync(join(tmpdir(), 'af-maintenance-surgery-waits-')); const strategy = new GatedTickStrategy(); const framework = await AgentFramework.create({ diff --git a/test/operator-surgery.test.ts b/test/operator-surgery.test.ts index fc9445fbf..90bc1c8cf 100644 --- a/test/operator-surgery.test.ts +++ b/test/operator-surgery.test.ts @@ -420,6 +420,37 @@ describe('live operator surgery', () => { assert.deepEqual(remainingTexts(), ['m1', 'm2', 'm3', 'm4', 'm5']); }); + it('a rewind of a tool result takes its tool_use turn whole, a turn stored as a body, in one removal', async () => { + // As context-manager's chunker stores them: the tool_use block on the + // turn's first shard, text after it; then the result, also a body. + const store = (cm() as unknown as ShardStore).messageStore; + const turn = [ + store.append('assistant', [{ type: 'tool_use', id: 't9', name: 'shell', input: {} }], {}, undefined, { bodyGroupId: 'g-turn', shardIndex: 0 }).id, + store.append('assistant', [{ type: 'text', text: 'narration' }], {}, undefined, { bodyGroupId: 'g-turn', shardIndex: 1 }).id, + ]; + const result = [ + store.append('user', [{ type: 'tool_result', toolUseId: 't9', content: 'r0' }], {}, undefined, { bodyGroupId: 'g-result', shardIndex: 0 }).id, + store.append('user', [{ type: 'text', text: 'r1' }], {}, undefined, { bodyGroupId: 'g-result', shardIndex: 1 }).id, + ]; + const c = cm() as unknown as { removeMessages: (a: string, b: string) => void }; + const ranges: Array<[string, string]> = []; + const original = c.removeMessages.bind(c); + c.removeMessages = (a: string, b: string) => { ranges.push([a, b]); original(a, b); }; + try { + const internals = framework as unknown as { + shedNewestTurn(agent: unknown): { kind: string; removedIds: string[] } | null; + }; + const record = internals.shedNewestTurn(framework.getAgent('scout')); + assert.ok(record); + assert.equal(record.kind, 'tool'); + assert.deepEqual(record.removedIds, [result[1], result[0], turn[1], turn[0]]); + assert.deepEqual(ranges, [[turn[0], result[1]]], "one removal, from the turn's first shard to the result's last"); + } finally { + c.removeMessages = original; + } + assert.deepEqual(remainingTexts(), ['m1', 'm2', 'm3', 'm4', 'm5']); + }); + // --- failure path ---------------------------------------------------------- it('a failed suppression restores the source branch, retires its outbox batch, and leaves the store bootable', async () => { diff --git a/test/overbudget-drain-breaker.test.ts b/test/overbudget-drain-breaker.test.ts index b21de6904..3defc651e 100644 --- a/test/overbudget-drain-breaker.test.ts +++ b/test/overbudget-drain-breaker.test.ts @@ -32,6 +32,7 @@ function makeHarness(tick: () => Promise) { fw.rewindEpisode = new Map(); fw.lastInferenceAt = new Map(); fw.overBudgetDrainInFlight = new Set(); + fw.overBudgetDrains = new Set(); fw.opsAlertLastSent = new Map(); fw.pendingRequests = []; fw.traceListeners = []; diff --git a/test/refusal-rewind.test.ts b/test/refusal-rewind.test.ts index a46a90f9f..0a8188e3a 100644 --- a/test/refusal-rewind.test.ts +++ b/test/refusal-rewind.test.ts @@ -21,6 +21,13 @@ function makeAgentHarness(messages: any[]) { const i = messages.findIndex((m) => m.id === id); if (i >= 0) messages.splice(i, 1); }, + // A range by its two ends, inclusive, as context-manager's removeMessages. + removeMessages: (fromId: string, toId: string) => { + const from = messages.findIndex((m) => m.id === fromId); + const to = messages.findIndex((m) => m.id === toId); + if (from < 0 || to < from) throw new Error(`no range ${fromId}..${toId}`); + for (const m of messages.splice(from, to - from + 1)) removed.push(m.id); + }, addMessage: (participant: string, content: any[], meta: any) => { const id = `marker-${added.length}`; added.push({ participant, content, meta }); From 525a31802969569b2cbb26f20cad6c50a263555a Mon Sep 17 00:00:00 2001 From: Ada-1017 Date: Wed, 14 Oct 2026 10:46:04 -0400 Subject: [PATCH 15/30] test(maintenance): a lasting failure's re-alerts follow its whole text and the cooldown; resume re-checks the hold after its wait - SayingStrategy refuses its readiness with a settable text: a lasting failure is alerted once, again when the text changes, again when two long texts differ only in their middle (which the run's bounded text leaves out), and again once opsAlert's cooldown has passed. - A rollback that takes the store while resume() waits for a tick makes resume() refuse as agent-busy after the wait, and the host stays quiesced. --- test/maintenance-timer.test.ts | 108 +++++++++++++++++++++++++++++++++ 1 file changed, 108 insertions(+) diff --git a/test/maintenance-timer.test.ts b/test/maintenance-timer.test.ts index 616703d03..e1c5f671e 100644 --- a/test/maintenance-timer.test.ts +++ b/test/maintenance-timer.test.ts @@ -147,6 +147,26 @@ class StaleStrategy implements ContextStrategy { } } +/** A strategy whose readiness query refuses with whatever text it's given. */ +class SayingStrategy implements ContextStrategy { + readonly name = 'saying-test'; + says = 'the first refusal'; + + checkReadiness(): ReadinessState { + throw new Error(this.says); + } + + async tick(_ctx: StrategyContext): Promise {} + + select( + _store: MessageStoreView, + _log: ContextLogView, + _budget: TokenBudget, + ): ContextEntry[] { + return []; + } +} + /** A strategy with work pending that no tick finishes. */ class NeverReadyStrategy implements ContextStrategy { readonly name = 'never-ready-test'; @@ -759,6 +779,53 @@ describe('queued context maintenance timer', () => { } }); + it('a failure that lasts is alerted again when its whole text changes, and again once the alert cooldown has passed', async () => { + const dir = mkdtempSync(join(tmpdir(), 'af-maintenance-lasting-text-')); + const strategy = new SayingStrategy(); + const framework = await AgentFramework.create({ + storePath: join(dir, 'store'), + membrane, + agents: [{ name: 'saying', model: 'test-model', systemPrompt: 'test', strategy }], + modules: [], + syncIntervalMs: 0, + maintenanceIntervalMs: 10, + }); + const alerts: string[] = []; + (framework as any).opsAlert = (_kind: string, _agentName: string, text: string) => alerts.push(text); + const internals = framework as unknown as { runQueuedMaintenance: () => Promise; opsAlertCooldownMs: number }; + let passes = 0; + const runPass = internals.runQueuedMaintenance.bind(framework); + internals.runQueuedMaintenance = () => { passes++; return runPass(); }; + const passed = async (n: number) => { const after = passes; await waitFor(() => passes >= after + n); }; + const quiet = console.error; + console.error = () => {}; + try { + framework.start(); + await waitFor(() => alerts.length === 1); + await passed(3); + assert.deepEqual(alerts, ['the first refusal']); + + // Two long texts that differ only in their middle, which the bounded + // text a run records leaves out: the alert follows the whole text. + const long = (middle: string) => `${'a'.repeat(3_000)} ${middle} ${'z'.repeat(3_000)}`; + strategy.says = long('one'); + await waitFor(() => alerts.length === 2); + strategy.says = long('two'); + await waitFor(() => alerts.length === 3); + await passed(3); + assert.equal(alerts.length, 3); + + // Unchanged, it is alerted again once the cooldown has passed. + internals.opsAlertCooldownMs = 0; + await waitFor(() => alerts.length >= 5); + assert.ok(alerts.slice(3).every((text) => text === long('two'))); + } finally { + console.error = quiet; + await framework.stop(); + rmSync(dir, { recursive: true, force: true }); + } + }); + it('a readiness refusal read after an agent\'s last tick is that agent\'s failure, alerted once', async () => { const dir = mkdtempSync(join(tmpdir(), 'af-maintenance-refused-after-ticks-')); const strategy = new RefusesAfterTicksStrategy(); @@ -869,6 +936,47 @@ describe('queued context maintenance timer', () => { }); } + it('a surgery that takes the store while resume waits for a tick is refused by resume after the wait', { timeout: 10_000 }, async () => { + const dir = mkdtempSync(join(tmpdir(), 'af-maintenance-resume-rechecks-')); + const strategy = new GatedTickStrategy(); + const framework = await AgentFramework.create({ + storePath: join(dir, 'store'), + membrane, + agents: [{ name: 'agent', model: 'test-model', systemPrompt: 'test', strategy }], + modules: [], + syncIntervalMs: 0, + maintenanceIntervalMs: 10, + }); + strategy.framework = framework; + const cm = framework.getAgent('agent')!.getContextManager(); + const first = String(cm.addMessage('user', [{ type: 'text', text: 'm1' }])); + cm.addMessage('user', [{ type: 'text', text: 'm2' }]); + const internals = framework as unknown as { surgeryHold: unknown }; + try { + await framework.quiesce(); + framework.start(); + await waitFor(() => strategy.ticks === 1); + // resume() finds no surgery and waits for the running tick; the + // rollback takes the store meanwhile. + const resuming = framework.resume(); + void resuming.catch(() => {}); + const moved = framework.rollbackToMessage('agent', { messageId: first }); + await waitFor(() => internals.surgeryHold !== null); + strategy.release(); + await assert.rejects( + resuming, + (error: unknown) => error instanceof OperatorActionError && error.code === 'agent-busy' + && error.message.includes('a live roll back for agent holds the store'), + ); + assert.equal(framework.getHostModeStatus().quiesced, true, 'the host stays quiesced'); + await moved; + } finally { + strategy.release(); + await framework.stop(); + rmSync(dir, { recursive: true, force: true }); + } + }); + it('no pass starts while a surgery holds the store', async () => { const dir = mkdtempSync(join(tmpdir(), 'af-maintenance-surgery-hold-')); const strategy = new QueuedStrategy(); From 758743f6ae3415c95d8b0960fca09a06b72add78 Mon Sep 17 00:00:00 2001 From: Ada-1017 Date: Wed, 14 Oct 2026 10:47:27 -0400 Subject: [PATCH 16/30] fix(hide): every stored copy of a message is hidden; a body counts once and its Discord ref once From the third review of the round: - hide found a Discord id's first stored copy only. A write cut short and retried stores a partial body and then the whole one, both carrying the id, so the retry stayed. hide now takes every copy, each widened to its body, and removes each contiguous run as one range. - The reply, the operator record and the log counted shards; they now count stored messages with a body counting once, so one long message is hidden: 1, and a partial copy plus its retry hidden: 2. - hiddenRefs listed a long message's ref once per shard; each Discord message is listed once. - Tests: one-shard, three-shard and cut-short-plus-retry copies hidden by their Discord ids; the range form (toMessageId) widening both ends to whole bodies, with a plain message between, counted and referenced once each. --- src/framework.ts | 59 +++++++++++++++++++++++++---------- test/operator-surgery.test.ts | 55 ++++++++++++++++++++++++++++---- 2 files changed, 91 insertions(+), 23 deletions(-) diff --git a/src/framework.ts b/src/framework.ts index 323c4f875..6b89d30fa 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -6983,16 +6983,31 @@ export class AgentFramework { // so the surface can mark them. channelId here is whatever the ingest // path stored (raw id or "discord:guild:channel"); the surface // normalizes it. - const refsIn = (lo: number, hi: number) => { + // Each Discord message once, though every shard of a body carries it. + const refsIn = (runs: Array<{ from: number; to: number }>) => { const refs: Array<{ channelId: string; messageId: string }> = []; - for (let i = lo; i <= hi; i++) { - const md = all[i].metadata as { messageId?: unknown; channelId?: unknown } | undefined; - if (md?.messageId && md?.channelId) { + const seen = new Set(); + for (const { from, to } of runs) { + for (let i = from; i <= to; i++) { + const md = all[i].metadata as { messageId?: unknown; channelId?: unknown } | undefined; + if (!md?.messageId || !md?.channelId) continue; + const key = `${String(md.channelId)}\u0000${String(md.messageId)}`; + if (seen.has(key)) continue; + seen.add(key); refs.push({ channelId: String(md.channelId), messageId: String(md.messageId) }); } } return refs; }; + // How many stored messages the runs hold, a body (a long message, in + // shards) counting once. + const messagesIn = (runs: Array<{ from: number; to: number }>) => { + let count = 0; + for (const { from, to } of runs) { + for (let i = from; i <= to; i = bodyRangeIn(all, i).to + 1) count++; + } + return count; + }; const fromIdx = byDiscordId(params.fromMessageId); if (fromIdx < 0) { @@ -7012,11 +7027,12 @@ export class AgentFramework { // A message stored as a body (a long one, in shards) is hidden whole. const lo = bodyRangeIn(all, first).from; const hi = bodyRangeIn(all, last).to; - const refs = refsIn(lo, hi); + const refs = refsIn([{ from: lo, to: hi }]); + const hidden = messagesIn([{ from: lo, to: hi }]); const rangeIds = all.slice(lo, hi + 1).map((m) => String(m.id)); removeStoredMessages(cm, all[lo], all[hi]); console.error( - `[host-command] hide agent=${agentName} range removed=${hi - lo + 1} ` + + `[host-command] hide agent=${agentName} range removed=${hidden} ` + `(${params.fromMessageId}..${params.toMessageId}) by=${params.requesterName ?? params.requesterId ?? 'unknown'} (server=${serverId})`, ); const loggedRange = capIds(rangeIds); @@ -7027,26 +7043,35 @@ export class AgentFramework { params: { fromMessageId: params.fromMessageId, toMessageId: params.toMessageId }, result: { branch: this.store.currentBranch().name, - hidden: hi - lo + 1, + hidden, removedIds: loggedRange.ids, ...(loggedRange.truncated ? { removedIdsTruncated: true } : {}), }, }); return { ok: true, - hidden: hi - lo + 1, + hidden, hiddenRefs: refs, lastVisible: await this.lastVisiblePreview(agentName), }; } - // A message stored as a body (a long one, in shards) is hidden whole: - // each shard carries its Discord id, and the first is found. - const { from: lo, to: hi } = bodyRangeIn(all, fromIdx); - const refs = refsIn(lo, lo); - const hiddenIds = all.slice(lo, hi + 1).map((m) => String(m.id)); - removeStoredMessages(cm, all[lo], all[hi]); + // A message stored as a body (a long one, in shards) is hidden whole, + // and so is every copy of it: a write cut short leaves a partial body + // before its retry, both carrying the Discord id. + const runs: Array<{ from: number; to: number }> = []; + all.forEach((m, i) => { + if (String((m.metadata as { messageId?: unknown } | undefined)?.messageId) !== String(params.fromMessageId)) return; + const { from, to } = bodyRangeIn(all, i); + const last = runs[runs.length - 1]; + if (last && from <= last.to + 1) last.to = Math.max(last.to, to); + else runs.push({ from, to }); + }); + const refs = refsIn(runs); + const hidden = messagesIn(runs); + const hiddenIds = runs.flatMap(({ from, to }) => all.slice(from, to + 1).map((m) => String(m.id))); + for (const { from, to } of runs) removeStoredMessages(cm, all[from], all[to]); console.error( - `[host-command] hide agent=${agentName} removed=${hi - lo + 1} (${params.fromMessageId}) ` + + `[host-command] hide agent=${agentName} removed=${hidden} (${params.fromMessageId}) ` + `by=${params.requesterName ?? params.requesterId ?? 'unknown'} (server=${serverId})`, ); this.recordOperatorAction({ @@ -7054,11 +7079,11 @@ export class AgentFramework { agent: agentName, requester: hostCommandRequester(serverId, params), params: { fromMessageId: params.fromMessageId }, - result: { branch: this.store.currentBranch().name, hidden: hi - lo + 1, removedIds: hiddenIds }, + result: { branch: this.store.currentBranch().name, hidden, removedIds: hiddenIds }, }); return { ok: true, - hidden: hi - lo + 1, + hidden, hiddenRefs: refs, lastVisible: await this.lastVisiblePreview(agentName), }; diff --git a/test/operator-surgery.test.ts b/test/operator-surgery.test.ts index 90bc1c8cf..27365ca04 100644 --- a/test/operator-surgery.test.ts +++ b/test/operator-surgery.test.ts @@ -372,20 +372,23 @@ describe('live operator surgery', () => { void s0; }); - it('hide removes a message stored as a body whole, a one-shard body included', async () => { + it('hide removes a message stored as a body whole, a one-shard body and every copy included', async () => { // Each shard of a long message carries its Discord message id. - const appendBody = (discordId: string, group: string, texts: string[]): string[] => - texts.map((text, shardIndex) => + const appendBody = (discordId: string, group: string, texts: string[], from = 0): string[] => + texts.map((text, i) => (cm() as unknown as ShardStore).messageStore.append( 'user', [{ type: 'text', text }], { serverId: 'discord', channelId: 'discord:g1:c1', messageId: discordId }, undefined, - { bodyGroupId: group, shardIndex }, + { bodyGroupId: group, shardIndex: from + i }, ).id, ); const [lone] = appendBody('d-lone', 'g-lone', ['lone']); const long = appendBody('d-long', 'g-long', ['l0', 'l1', 'l2']); + // A write cut short after two shards, then its retry from shard 0. + const partial = appendBody('d-cut', 'g-cut', ['c0', 'c1']); + const retry = appendBody('d-cut', 'g-cut', ['c0', 'c1', 'c2']); const internals = framework as unknown as { handleHostCommand(serverId: string, params: Record): Promise>; }; @@ -395,6 +398,8 @@ describe('live operator surgery', () => { fromMessageId, requesterName: 'antra', }); + const refsOf = (result: Record) => + (result.hiddenRefs as Array<{ messageId: string }>).map((ref) => ref.messageId); const one = await hide('d-lone'); assert.equal(one.ok, true, String(one.error)); @@ -403,9 +408,47 @@ describe('live operator surgery', () => { const whole = await hide('d-long'); assert.equal(whole.ok, true, String(whole.error)); - assert.equal(whole.hidden, 3); - assert.deepEqual((whole.hiddenRefs as Array<{ messageId: string }>).map((ref) => ref.messageId), ['d-long']); + assert.equal(whole.hidden, 1, 'one message, though stored in three shards'); + assert.deepEqual(refsOf(whole), ['d-long']); for (const id of long) assert.equal(cm().getMessage(id as never), null); + + const copies = await hide('d-cut'); + assert.equal(copies.ok, true, String(copies.error)); + assert.equal(copies.hidden, 2, 'the partial copy and its retry'); + assert.deepEqual(refsOf(copies), ['d-cut']); + for (const id of [...partial, ...retry]) assert.equal(cm().getMessage(id as never), null); + assert.deepEqual(remainingTexts(), ['m1', 'm2', 'm3', 'm4', 'm5']); + }); + + it('hide over a range widens both ends to whole bodies, and counts each message once', async () => { + const appendBody = (discordId: string, group: string, texts: string[]): string[] => + texts.map((text, shardIndex) => + (cm() as unknown as ShardStore).messageStore.append( + 'user', + [{ type: 'text', text }], + { serverId: 'discord', channelId: 'discord:g1:c1', messageId: discordId }, + undefined, + { bodyGroupId: group, shardIndex }, + ).id, + ); + appendBody('d-a', 'g-a', ['a0', 'a1']); + cm().addMessage('user', [{ type: 'text', text: 'between' }], { serverId: 'discord', channelId: 'discord:g1:c1', messageId: 'd-mid' }); + appendBody('d-b', 'g-b', ['b0', 'b1', 'b2']); + const internals = framework as unknown as { + handleHostCommand(serverId: string, params: Record): Promise>; + }; + // From the second body's first shard back to the first body's: both + // widen to their whole bodies. + const result = await internals.handleHostCommand('discord', { + command: 'hide', + agentName: 'scout', + fromMessageId: 'd-b', + toMessageId: 'd-a', + requesterName: 'antra', + }); + assert.equal(result.ok, true, String(result.error)); + assert.equal(result.hidden, 3); + assert.deepEqual((result.hiddenRefs as Array<{ messageId: string }>).map((ref) => ref.messageId), ['d-a', 'd-mid', 'd-b']); assert.deepEqual(remainingTexts(), ['m1', 'm2', 'm3', 'm4', 'm5']); }); From 7df647f313020f883c7de16e41454752b586ebb6 Mon Sep 17 00:00:00 2001 From: Ada-1017 Date: Wed, 14 Oct 2026 10:48:41 -0400 Subject: [PATCH 17/30] fix(removals): a module's and the coalescer's removal of a message in a body takes the whole body From the third review of the round: the module API's removeMessage (which the Discord module's delete, stale-message sweep and reconcile reach) and the MCPL coalescer removed by id with context-manager's removeMessage, which refuses any message in a body. A long Discord message stored in shards therefore stayed in context when it was deleted on Discord. removeStoredMessage(cm, id) removes a message outside any body alone, and a shard with the rest of its body: a module removes what it added by the id addMessage returned, the first shard's for a long message. Both paths use it. Test: a module's removal of a body's middle shard takes the body; a plain message goes alone. --- src/framework.ts | 6 +++--- src/stored-removal.ts | 21 +++++++++++++++++++-- test/operator-surgery.test.ts | 11 +++++++++++ 3 files changed, 33 insertions(+), 5 deletions(-) diff --git a/src/framework.ts b/src/framework.ts index 6b89d30fa..8da113d49 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -19,7 +19,7 @@ import type { StoredMessage, TokenBudget, } from '@animalabs/context-manager'; -import { bodyRangeIn, removeStoredMessages } from './stored-removal.js'; +import { bodyRangeIn, removeStoredMessage, removeStoredMessages } from './stored-removal.js'; import type { FrameworkConfig, InferencePolicy, @@ -10192,7 +10192,7 @@ export class AgentFramework { const agent = this.agents.get(p.agent); if (!agent) return false; try { - agent.getContextManager().removeMessage(p.messageId); + removeStoredMessage(agent.getContextManager(), p.messageId as MessageId); this.emitTrace({ type: 'message:removed', messageId: p.messageId, source: 'mcpl:coalescing' }); return true; } catch (err) { @@ -15224,7 +15224,7 @@ export class AgentFramework { if (!agent) { throw new Error('No agents configured'); } - agent.getContextManager().removeMessage(id); + removeStoredMessage(agent.getContextManager(), id); } private getMessage(id: MessageId): StoredMessage | null { diff --git a/src/stored-removal.ts b/src/stored-removal.ts index 93edff287..bd146579f 100644 --- a/src/stored-removal.ts +++ b/src/stored-removal.ts @@ -6,8 +6,7 @@ type Removals = Pick; /** * Remove the stored messages `first`..`last` (inclusive, in store order). * - * A single message outside any body goes through `removeMessage`, whose - * remove event also drops the context-log entries copied from it. + * A single message outside any body goes through `removeMessage`. * context-manager's `removeMessage` refuses any message that carries a body * group id, a one-shard body's included, so a shard, and any longer interval, * goes through `removeMessages`, which refuses an interval that splits a @@ -21,6 +20,24 @@ export function removeStoredMessages(cm: Removals, first: StoredLike, last: Stor cm.removeMessages(first.id, last.id); } +/** + * Remove one stored message by its id, with the rest of its body when it's + * in one: a module or a coalescer removes a message it added by the id + * `addMessage` returned, which for a long message stored in shards is its + * first shard's. + */ +export function removeStoredMessage(cm: ContextManager, id: MessageId): void { + const message = cm.getMessage(id) as StoredLike | null; + if (!message?.bodyGroupId) { + cm.removeMessage(id); + return; + } + const all = cm.getAllMessages(); + const at = all.findIndex((m) => m.id === id); + const { from, to } = bodyRangeIn(all, at); + removeStoredMessages(cm, all[from] as StoredLike, all[to] as StoredLike); +} + /** removeStoredMessages for an interval known by its ids. */ export function removeStoredInterval(cm: ContextManager, fromId: MessageId, toId: MessageId): void { const first = cm.getMessage(fromId) as StoredLike | null; diff --git a/test/operator-surgery.test.ts b/test/operator-surgery.test.ts index 27365ca04..a2b774ea6 100644 --- a/test/operator-surgery.test.ts +++ b/test/operator-surgery.test.ts @@ -452,6 +452,17 @@ describe('live operator surgery', () => { assert.deepEqual(remainingTexts(), ['m1', 'm2', 'm3', 'm4', 'm5']); }); + it("a module's removal of a message in a body takes the whole body, and a plain message alone", async () => { + const [b0, b1, b2] = appendShards('g-mod', ['b0', 'b1', 'b2']); + const plain = String(cm().addMessage('user', [{ type: 'text', text: 'plain' }])); + const internals = framework as unknown as { removeMessage(id: string): void }; + internals.removeMessage(b1); + for (const id of [b0, b1, b2]) assert.equal(cm().getMessage(id as never), null); + internals.removeMessage(plain); + assert.equal(cm().getMessage(plain as never), null); + assert.deepEqual(remainingTexts(), ['m1', 'm2', 'm3', 'm4', 'm5']); + }); + it('a rewind whose newest message is the last shard of a body takes the whole body', async () => { const [s0, s1, s2] = appendShards('g-doc', ['d0', 'd1', 'd2']); const internals = framework as unknown as { From 9de6d896e6ed8e04071547f232fdce2daf0607d8 Mon Sep 17 00:00:00 2001 From: Ada-1017 Date: Wed, 14 Oct 2026 10:49:14 -0400 Subject: [PATCH 18/30] test(maintenance): a rollback waits for an over-budget drain kick's tick, and the kick ticks no more under the hold --- test/maintenance-timer.test.ts | 41 ++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/test/maintenance-timer.test.ts b/test/maintenance-timer.test.ts index e1c5f671e..1fa681801 100644 --- a/test/maintenance-timer.test.ts +++ b/test/maintenance-timer.test.ts @@ -977,6 +977,47 @@ describe('queued context maintenance timer', () => { } }); + it('a rollback waits for an over-budget drain kick\'s tick in flight, and the kick ticks no more under the hold', { timeout: 10_000 }, async () => { + const dir = mkdtempSync(join(tmpdir(), 'af-maintenance-drain-hold-')); + const strategy = new GatedTickStrategy(); + const framework = await AgentFramework.create({ + storePath: join(dir, 'store'), + membrane, + agents: [{ name: 'agent', model: 'test-model', systemPrompt: 'test', strategy }], + modules: [], + syncIntervalMs: 0, + maintenanceIntervalMs: 0, + }); + strategy.framework = framework; + const cm = framework.getAgent('agent')!.getContextManager(); + const first = String(cm.addMessage('user', [{ type: 'text', text: 'm1' }])); + cm.addMessage('user', [{ type: 'text', text: 'm2' }]); + const internals = framework as unknown as { + surgeryHold: unknown; + noteInferenceExhausted(agentName: string, reason: string, retryable?: boolean, errorType?: string): void; + }; + const quiet = console.error; + console.error = () => {}; + try { + // A refused compile kicks the drain, whose first tick waits on the gate. + internals.noteInferenceExhausted('agent', 'compile refused', false, 'over_budget'); + await waitFor(() => strategy.ticks === 1); + const moved = framework.rollbackToMessage('agent', { messageId: first }); + await waitFor(() => internals.surgeryHold !== null); + assert.equal(cm.currentBranch().name, 'main', 'the rollback waits for the drain kick\'s tick'); + strategy.release(); + await moved; + assert.equal(strategy.branchAtFirstTickEnd, 'main'); + assert.equal(strategy.ticksUnderHold, 0, 'the kick starts no tick while the rollback holds the store'); + assert.notEqual(cm.currentBranch().name, 'main'); + } finally { + console.error = quiet; + strategy.release(); + await framework.stop(); + rmSync(dir, { recursive: true, force: true }); + } + }); + it('no pass starts while a surgery holds the store', async () => { const dir = mkdtempSync(join(tmpdir(), 'af-maintenance-surgery-hold-')); const strategy = new QueuedStrategy(); From a170308d9fb2cf80ddba10f1d9c0202e523d8198 Mon Sep 17 00:00:00 2001 From: Ada-1017 Date: Wed, 14 Oct 2026 10:49:51 -0400 Subject: [PATCH 19/30] docs(changelog), test: the operator note says where to tell the first kind of store; the lasting-failure test waits for a clean pass, not the tick cap - CHANGELOG 0.21.1's operator note: the first kind of store can't be told from its record, but the host's history can: a host that went from a context-manager older than 0.11.0 straight to 0.14.0 can hold one. - maintenance-timer: the recovered-with-work phase waits for a pass that ran the agent without a failure, not for exactly eight ticks, so a change to the per-pass cap fails an assertion rather than timing out. --- CHANGELOG.md | 6 ++++-- test/maintenance-timer.test.ts | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 38318b251..a7cf31a15 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -704,8 +704,10 @@ framework. (shown in `RenderStats.calibration`). Two kinds of store can then refuse every compile with `OverBudgetError` before inference. One holds no carriers and kept a multiplier last learned before its host moved to - context-manager 0.11.0; its record can't show that, so tell it by when - the host moved. The other holds carriers and had learned a multiplier + context-manager 0.11.0. Its record can't show that; the host's own + history can, in its lockfile's or its deployments' history: a host that + went from a context-manager older than 0.11.0 straight to 0.14.0 can + hold one. The other holds carriers and had learned a multiplier below 1, which its `reset.discardedMultiplier` shows. With the host stopped, reset the first kind's multiplier once, and re-stamp the second kind's checked value, as the **Operators** entry of context-manager diff --git a/test/maintenance-timer.test.ts b/test/maintenance-timer.test.ts index 1fa681801..40a133af2 100644 --- a/test/maintenance-timer.test.ts +++ b/test/maintenance-timer.test.ts @@ -767,7 +767,7 @@ describe('queued context maintenance timer', () => { // Recovered with work no tick finishes: a pass runs it without a failure. await cm.setStrategy(new NeverReadyStrategy()); await waitFor(() => framework.getContextMaintenanceSnapshot().history.some((run) => - run.agents.some((agent) => agent.agentName === 'fenced' && agent.ticks === 8 && agent.error === undefined))); + run.agents.some((agent) => agent.agentName === 'fenced' && agent.ticks > 0 && agent.error === undefined))); await fence(); await waitFor(() => alerts.length === 3); assert.equal(failedLines(), 3); From 2558dc9f24e610b2dc8ee8f5e8bcd9f0773fa11e Mon Sep 17 00:00:00 2001 From: Ada-1017 Date: Wed, 14 Oct 2026 10:51:00 -0400 Subject: [PATCH 20/30] test(discord-awareness): a resumed suppression's two crash states; the undo stub has no maintenance running - A resumed suppression skips an interval that committed before the crash (both ends gone) and refuses one only partly present, leaving the batch prepared. - The message-granular undo test's stub framework gains maintenancePass and overBudgetDrains, which a surgery now reads to wait for maintenance work. --- test/discord-awareness-outbox.test.ts | 49 +++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/test/discord-awareness-outbox.test.ts b/test/discord-awareness-outbox.test.ts index 82edf955e..482036780 100644 --- a/test/discord-awareness-outbox.test.ts +++ b/test/discord-awareness-outbox.test.ts @@ -195,6 +195,52 @@ test('a resumed suppression removes a one-shard body as a range and a message ou } }); +test('a resumed suppression skips an interval that committed before the crash, and refuses one only partly present', async () => { + const dir = mkdtempSync(join(tmpdir(), 'discord-awareness-resume-states-')); + try { + const run = async (present: Set) => { + const outbox = new DiscordAwarenessOutbox(join(dir, `outbox-${[...present].join('-') || 'none'}.json`)); + const batch = outbox.prepare({ + agentName: 'cairn', + sourceBranch: 'main', + targetBranch: 'recovery/cairn/suppressed', + activationPolicy: 'explicit', + suppressionIntervals: [ + { fromId: 'a0', toId: 'a1' }, + { fromId: 'b0', toId: 'b1' }, + ], + refs: [{ serverId: 'discord', channelId: 'discord:g1:c1', messageId: 'm1' }], + })!; + const removals: string[] = []; + const cm = { + getMessage: (id: string) => (present.has(id) ? { id } : null), + removeMessage: (id: string) => { present.delete(id); removals.push(id); }, + removeMessages: (from: string, to: string) => { present.delete(from); present.delete(to); removals.push(`${from}..${to}`); }, + }; + const framework = Object.create(AgentFramework.prototype) as any; + framework.discordAwarenessOutbox = outbox; + framework.store = { currentBranch: () => ({ name: 'recovery/cairn/suppressed' }), listBranches: () => [] }; + framework.agents = new Map([['cairn', { getContextManager: () => cm }]]); + const outcome = await framework.resumePreparedDiscordSuppressions().then(() => 'resumed', (error: Error) => error.message); + return { outcome, removals, status: outbox.batches().find((candidate) => candidate.id === batch.id)?.status }; + }; + + // The later interval committed before the crash: only the earlier is removed. + const committed = await run(new Set(['a0', 'a1'])); + assert.equal(committed.outcome, 'resumed'); + assert.deepEqual(committed.removals, ['a0..a1']); + assert.equal(committed.status, 'active'); + + // An interval with one end gone isn't safe to finish. + const partial = await run(new Set(['a0', 'a1', 'b0'])); + assert.match(partial.outcome, /interval b0\.\.b1 is only partially present/); + assert.deepEqual(partial.removals, []); + assert.equal(partial.status, 'prepared'); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); + test('framework drain records failures per ref and continues past permanent failures', async () => { const dir = mkdtempSync(join(tmpdir(), 'discord-awareness-drain-')); const path = join(dir, 'outbox.json'); @@ -317,6 +363,9 @@ test('message-granular undo prepares markers before switching branches', async ( framework.pendingAssistantBlocks = new Map(); // A host/command and a store surgery are operations stop() waits for. framework.operatorOperations = new Set(); + // A surgery waits for maintenance work that is running: here, none. + framework.maintenancePass = null; + framework.overBudgetDrains = new Set(); framework.operatorLog = new OperatorLog(undefined); const result = await framework.handleHostCommand('discord', { From 5165fe5e6b9d3d31160a809f1108903d7838b97e Mon Sep 17 00:00:00 2001 From: Ada-1017 Date: Wed, 14 Oct 2026 11:02:07 -0400 Subject: [PATCH 21/30] test(inference-failure-bounds): the stub framework has the drain-kick set the kick now records itself in The full suite at 22bce010 failed six of this file's tests: its stub framework (Object.create) lacked overBudgetDrains, which the over-budget drain kick now adds itself to so a branch move can wait for it. --- test/inference-failure-bounds.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/test/inference-failure-bounds.test.ts b/test/inference-failure-bounds.test.ts index 025d25256..6fe63cdbe 100644 --- a/test/inference-failure-bounds.test.ts +++ b/test/inference-failure-bounds.test.ts @@ -54,6 +54,7 @@ function makeHarness(opts?: { tick?: () => Promise; addMessage?: (p: strin fw.lastInferenceAt = new Map(); fw.pendingRequests = []; fw.overBudgetDrainInFlight = new Set(); + fw.overBudgetDrains = new Set(); fw.opsAlertLastSent = new Map(); fw.opsAlertCooldownMs = 15 * 60_000; const traces: any[] = []; From d71370d5cce20f1d4a1fe604d50e41133265d83e Mon Sep 17 00:00:00 2001 From: Ada-1017 Date: Wed, 14 Oct 2026 11:03:23 -0400 Subject: [PATCH 22/30] fix(hide): refused while a surgery holds the store; a missing Discord id matches nothing Gladys-2413's read of the round: - hide checked only that the agent was idle, which every agent is during a surgery, so it removed on the branch the surgery was leaving, or met a manager mid-switch. It now refuses as the other operator calls do, naming the surgery. - hide compared String(metadata.messageId), so a message with no Discord id read as "undefined", and the every-copy loop widened a fromMessageId of "undefined" to every such message: the agent's replies, tool results and notes. Only a present id matches now, in the lookup and the loop. - Test: hide under a reservation refuses; "undefined" matches neither of two id-less messages; the held message hides once the store is let go. --- src/framework.ts | 20 +++++++++++++++++--- test/operator-surgery.test.ts | 30 ++++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+), 3 deletions(-) diff --git a/src/framework.ts b/src/framework.ts index 8da113d49..45b59373f 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -6971,13 +6971,27 @@ export class AgentFramework { if (agent.state.status !== 'idle') { return { ok: false, error: `Cannot hide while agent is ${agent.state.status}` }; } + // A surgery switches or forks the branch while it holds the store, so + // a removal now would land on the branch it's leaving, or meet a + // manager mid-switch: refused as busy, as the other operator calls are. + if (this.surgeryHold) { + return { + ok: false, + error: `Cannot hide: a live ${this.surgeryHold.verb} for ${this.surgeryHold.agentName} holds the store — retry when it completes`, + }; + } if (!params.fromMessageId) { return { ok: false, error: 'hide: fromMessageId is required' }; } const cm = agent.getContextManager(); const all = cm.getAllMessages(); - const byDiscordId = (did: string) => - all.findIndex((m) => String((m.metadata as { messageId?: unknown } | undefined)?.messageId) === String(did)); + // Only a message that carries a Discord id matches: a missing one + // would otherwise read as the text "undefined". + const discordIdOf = (m: StoredMessage) => { + const id = (m.metadata as { messageId?: unknown } | undefined)?.messageId; + return id === undefined || id === null || id === '' ? undefined : String(id); + }; + const byDiscordId = (did: string) => all.findIndex((m) => discordIdOf(m) === String(did)); // Collect Discord refs of every message in [lo, hi] that carries one, // so the surface can mark them. channelId here is whatever the ingest @@ -7060,7 +7074,7 @@ export class AgentFramework { // before its retry, both carrying the Discord id. const runs: Array<{ from: number; to: number }> = []; all.forEach((m, i) => { - if (String((m.metadata as { messageId?: unknown } | undefined)?.messageId) !== String(params.fromMessageId)) return; + if (discordIdOf(m) !== String(params.fromMessageId)) return; const { from, to } = bodyRangeIn(all, i); const last = runs[runs.length - 1]; if (last && from <= last.to + 1) last.to = Math.max(last.to, to); diff --git a/test/operator-surgery.test.ts b/test/operator-surgery.test.ts index a2b774ea6..e19e6e337 100644 --- a/test/operator-surgery.test.ts +++ b/test/operator-surgery.test.ts @@ -420,6 +420,36 @@ describe('live operator surgery', () => { assert.deepEqual(remainingTexts(), ['m1', 'm2', 'm3', 'm4', 'm5']); }); + it('hide refuses while a surgery holds the store, and a missing Discord id matches nothing', async () => { + cm().addMessage('user', [{ type: 'text', text: 'held' }], { serverId: 'discord', channelId: 'discord:g1:c1', messageId: 'd-held' }); + const internals = framework as unknown as { + handleHostCommand(serverId: string, params: Record): Promise>; + reserveStoreForSurgery(verb: string, agentName: string): () => void; + }; + const hide = (fromMessageId: string) => internals.handleHostCommand('discord', { + command: 'hide', agentName: 'scout', fromMessageId, requesterName: 'antra', + }); + const release = internals.reserveStoreForSurgery('roll back', 'scout'); + try { + const refused = await hide('d-held'); + assert.equal(refused.ok, false); + assert.match(String(refused.error), /a live roll back for scout holds the store/); + } finally { + release(); + } + // Two messages with no Discord id, such as the agent's replies: the + // text "undefined" names neither of them. + cm().addMessage('scout', [{ type: 'text', text: 'reply-1' }]); + cm().addMessage('scout', [{ type: 'text', text: 'reply-2' }]); + const none = await hide('undefined'); + assert.equal(none.ok, false); + assert.match(String(none.error), /is not an addressable message in context/); + assert.deepEqual(remainingTexts(), ['m1', 'm2', 'm3', 'm4', 'm5', 'held', 'reply-1', 'reply-2']); + const hidden = await hide('d-held'); + assert.equal(hidden.ok, true, String(hidden.error)); + assert.deepEqual(remainingTexts(), ['m1', 'm2', 'm3', 'm4', 'm5', 'reply-1', 'reply-2']); + }); + it('hide over a range widens both ends to whole bodies, and counts each message once', async () => { const appendBody = (discordId: string, group: string, texts: string[]): string[] => texts.map((text, shardIndex) => From 3abf2bd1ccee0c7e680ac81f0aa999de532e340a Mon Sep 17 00:00:00 2001 From: Ada-1017 Date: Wed, 14 Oct 2026 12:31:36 -0400 Subject: [PATCH 23/30] fix(maintenance): a surgery withdraws maintenance provider calls rather than waiting on them at the provider gate A surgery waits for the maintenance work that is running before it moves the branch. A maintenance tick whose compression call parks at the agent's provider gate waits for the gate to reopen: for a cooldown to end, and, once a cooldown a primary call started has ended, for the retry turn it queued, which runs first. That turn can't start while the surgery holds the store, so the surgery and the parked call waited on each other for good, the store reserved and every agent paused. Now maintenance makes no provider call while a surgery holds the store: - withAuxiliaryAdmission refuses one with MaintenanceWithdrawnError when it's made under a hold, before it can park, and when a call that parked wakes to find one; - taking the hold (reserveStoreForSurgery) wakes every call parked at a provider gate, so each finds the hold and is withdrawn. A withdrawn call sent nothing. context-manager rethrows a plain error from a provider call with the work it was for kept queued and no attempt counted, so the next pass after the hold does it again. The pass treats a withdrawal as the end of that agent's work for the pass: no failure, no alert, and what the failure reports remember stays as it is. The over-budget drain kick logs it as stopped by the hold. Tests: a rollback, a suppression and a branch switch each finish while a maintenance call is parked at the gate behind a queued retry, the call is withdrawn having sent nothing, and nothing is reported as a failure; a call made under a hold is withdrawn whether the gate is open or closed, and admitted again once the hold is released. Without the wake, the three surgeries wait; without the refusal before parking, the closed-gate call parks. --- src/framework.ts | 58 +++++++++-- test/discord-awareness-outbox.test.ts | 2 + test/maintenance-timer.test.ts | 144 ++++++++++++++++++++++++++ 3 files changed, 196 insertions(+), 8 deletions(-) diff --git a/src/framework.ts b/src/framework.ts index 45b59373f..a1970c746 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -713,6 +713,19 @@ function logSeedOmissions( ); } +/** + * A maintenance provider call withdrawn because a surgery holds the store. It + * sent nothing: it was refused at the provider gate, or woken there from its + * wait. context-manager keeps the work it was for queued, with no attempt + * counted, and the next pass after the hold does it again. + */ +class MaintenanceWithdrawnError extends Error { + constructor(hold: { verb: string; agentName: string }) { + super(`maintenance call withdrawn: a live ${hold.verb} for ${hold.agentName} holds the store`); + this.name = 'MaintenanceWithdrawnError'; + } +} + class DiscordAwarenessAccountingError extends Error { constructor(operation: string, cause: unknown) { const detail = cause instanceof Error ? cause.message : String(cause); @@ -2878,6 +2891,7 @@ export class AgentFramework { private async withAuxiliaryAdmission(agentName: string, run: () => Promise, model?: string): Promise { const gate = this.providerGate(agentName); this.refuseHeldProviderModel(agentName, model); + this.withdrawMaintenanceUnderSurgery(); if (this.providerGateBlocked(agentName)) { gate.deferredAuxiliary++; try { @@ -2889,6 +2903,8 @@ export class AgentFramework { if (this.providerAdmissionClosed) { throw new Error(`Provider admission closed while stopping (${agentName})`); } + // A surgery that took the store while this call was parked withdraws it. + this.withdrawMaintenanceUnderSurgery(); // A wait recorded while this call was parked binds it too. this.refuseHeldProviderModel(agentName, model); gate.auxiliaryInFlight++; @@ -2910,6 +2926,16 @@ export class AgentFramework { if (gate.auxiliaryInFlight === 0) for (const resolve of gate.idleWaiters.splice(0)) resolve(); } } + /** + * Maintenance makes no provider call while a surgery holds the store. The + * surgery waits for the maintenance work that is running before it moves + * the branch, so a call that parked at the gate would make it wait on + * whatever reopens the gate: the end of a cooldown, or the retry turn a + * cooldown queues, which the hold itself keeps from starting. + */ + private withdrawMaintenanceUnderSurgery(): void { + if (this.surgeryHold) throw new MaintenanceWithdrawnError(this.surgeryHold); + } private auxiliaryMembraneFor(agentName: string): Membrane { return this.membraneAdmitting((run, model) => this.withAuxiliaryAdmission(agentName, run, model)); } @@ -3508,6 +3534,10 @@ export class AgentFramework { const running = queued.filter((item): item is Extract => !item.failed); // Settled, not all: the pass ends only when every agent's work has. const outcomes = await Promise.allSettled(running.map(async ({ agent, cm, record }) => { + // A tick whose provider call a surgery withdrew ends this agent's + // work for the pass, as the hold would have before its next tick: it + // is no failure, and what the failure reports remember stays as it is. + let withdrawn = false; try { for ( let i = 0; @@ -3518,22 +3548,27 @@ export class AgentFramework { record.ticks++; } } catch (error) { - this.reportMaintenanceFailure( - agent.name, - record, - error instanceof Error ? String(error.message) : failureDescription(error), - ); + if (error instanceof MaintenanceWithdrawnError) { + withdrawn = true; + console.error(`[context-maintenance] agent=${agent.name} stopped: ${error.message}`); + } else { + this.reportMaintenanceFailure( + agent.name, + record, + error instanceof Error ? String(error.message) : failureDescription(error), + ); + } } finally { record.finishedAt = Date.now(); const state = this.contextReadiness(cm); if ('error' in state) { // A run records an agent's first failure: a readiness refusal // after a tick that failed is read again by the next pass. - if (record.error === undefined) this.reportMaintenanceFailure(agent.name, record, state.error); + if (record.error === undefined && !withdrawn) this.reportMaintenanceFailure(agent.name, record, state.error); } else { record.readyAfter = state.ready; if (state.pending) record.pendingAfter = state.pending; - if (record.error === undefined) this.maintenanceFailureReported.delete(agent.name); + if (record.error === undefined && !withdrawn) this.maintenanceFailureReported.delete(agent.name); } const progress = this.contextProgress(cm); if (progress) record.progressAfter = progress; @@ -7760,6 +7795,9 @@ export class AgentFramework { // arrivals during the awaited switch (see surgeryHold). const hold = { verb, agentName, since: Date.now() }; this.surgeryHold = hold; + // Maintenance calls parked at a provider gate wake, find the hold and are + // withdrawn, so the wait for running maintenance can't wait on them. + for (const gate of this.providerGates.values()) for (const resolve of gate.auxiliaryWaiters.splice(0)) resolve(); // stop() waits for the release (see stopOwned): a surgery admitted before // stop() began finishes, flush included, before the store closes. let settle!: () => void; @@ -15740,7 +15778,11 @@ export class AgentFramework { `${ticks < 8 ? ', stopped while a surgery holds the store' : ''})`, ); } catch (err) { - console.error(`[inference-failed] drain kick failed for ${agentName} after ${ticks} ticks: ${describeFailureWithFrames(err)}`); + if (err instanceof MaintenanceWithdrawnError) { + console.error(`[inference-failed] drain kicked for ${agentName} (OverBudget breaker, ${ticks} ticks, stopped: ${err.message})`); + } else { + console.error(`[inference-failed] drain kick failed for ${agentName} after ${ticks} ticks: ${describeFailureWithFrames(err)}`); + } } finally { this.overBudgetDrainInFlight.delete(agentName); } diff --git a/test/discord-awareness-outbox.test.ts b/test/discord-awareness-outbox.test.ts index 482036780..c22e0a9ca 100644 --- a/test/discord-awareness-outbox.test.ts +++ b/test/discord-awareness-outbox.test.ts @@ -366,6 +366,8 @@ test('message-granular undo prepares markers before switching branches', async ( // A surgery waits for maintenance work that is running: here, none. framework.maintenancePass = null; framework.overBudgetDrains = new Set(); + // Its hold wakes maintenance calls parked at the provider gates: here, none. + framework.providerGates = new Map(); framework.operatorLog = new OperatorLog(undefined); const result = await framework.handleHostCommand('discord', { diff --git a/test/maintenance-timer.test.ts b/test/maintenance-timer.test.ts index 40a133af2..d620ee8b5 100644 --- a/test/maintenance-timer.test.ts +++ b/test/maintenance-timer.test.ts @@ -287,6 +287,48 @@ class GatedTickStrategy implements ContextStrategy { } } +/** + * A strategy whose first tick makes a provider call that parks at the agent's + * provider gate, as one does once a cooldown a primary call started has ended + * and the retry turn it queued hasn't run yet. + */ +class ParkedCallTickStrategy implements ContextStrategy { + readonly name = 'parked-call-tick-test'; + ticks = 0; + calls = 0; + outcome?: string; + framework?: AgentFramework; + + checkReadiness(): ReadinessState { + return { ready: false, description: 'parked call queued' }; + } + + async tick(_ctx: StrategyContext): Promise { + this.ticks++; + if (this.ticks > 1) return; + const internals = this.framework as unknown as { + providerGate(name: string): { primaryPending: boolean }; + withAuxiliaryAdmission(name: string, run: () => Promise): Promise; + }; + internals.providerGate('agent').primaryPending = true; + try { + await internals.withAuxiliaryAdmission('agent', async () => { this.calls++; }); + this.outcome = 'ran'; + } catch (error) { + this.outcome = (error as Error).name; + throw error; + } + } + + select( + _store: MessageStoreView, + _log: ContextLogView, + _budget: TokenBudget, + ): ContextEntry[] { + return []; + } +} + /** A strategy whose ticks succeed and whose readiness can't be read once all of a pass's ticks have run. */ class RefusesAfterTicksStrategy implements ContextStrategy { readonly name = 'refuses-after-ticks-test'; @@ -936,6 +978,108 @@ describe('queued context maintenance timer', () => { }); } + for (const surgery of [ + { verb: 'roll back', run: (framework: AgentFramework, first: string) => framework.rollbackToMessage('agent', { messageId: first }) }, + { verb: 'suppress', run: (framework: AgentFramework, first: string) => framework.suppressMessages('agent', { messageIds: [first] }) }, + { + verb: 'switch branch', + run: (framework: AgentFramework, first: string) => { + framework.getAgent('agent')!.getContextManager().branchAt(first as never, 'elsewhere'); + return framework.switchBranch('elsewhere'); + }, + }, + ] as Array<{ verb: string; run: (framework: AgentFramework, first: string) => Promise }>) { + it(`a ${surgery.verb} withdraws a maintenance call parked at the provider gate rather than waiting on it, and that is no failure`, { timeout: 10_000 }, async () => { + const dir = mkdtempSync(join(tmpdir(), 'af-maintenance-parked-call-')); + const strategy = new ParkedCallTickStrategy(); + const framework = await AgentFramework.create({ + storePath: join(dir, 'store'), + membrane, + agents: [{ name: 'agent', model: 'test-model', systemPrompt: 'test', strategy }], + modules: [], + syncIntervalMs: 0, + maintenanceIntervalMs: 10, + }); + strategy.framework = framework; + const alerts: string[] = []; + (framework as any).opsAlert = (kind: string, agentName: string, message: string) => alerts.push(`${kind}:${agentName}:${message}`); + const cm = framework.getAgent('agent')!.getContextManager(); + const first = String(cm.addMessage('user', [{ type: 'text', text: 'm1' }])); + cm.addMessage('user', [{ type: 'text', text: 'm2' }]); + const internals = framework as unknown as { providerGate(name: string): { auxiliaryWaiters: unknown[] } }; + const quiet = console.error; + console.error = () => {}; + try { + await framework.quiesce(); + framework.start(); + await waitFor(() => internals.providerGate('agent').auxiliaryWaiters.length === 1); + const moved = surgery.run(framework, first); + const outcome = await Promise.race([ + moved.then(() => 'moved'), + new Promise((resolve) => setTimeout(() => resolve(`the ${surgery.verb} still waits`), 3_000)), + ]); + assert.equal(outcome, 'moved'); + assert.equal(strategy.outcome, 'MaintenanceWithdrawnError'); + assert.equal(strategy.calls, 0, 'the withdrawn call sent nothing'); + assert.notEqual(cm.currentBranch().name, 'main'); + assert.deepEqual(alerts.filter((alert) => alert.startsWith('context-maintenance-failed')), []); + const run = framework.getContextMaintenanceSnapshot().history[0]!.agents[0]!; + assert.equal(run.error, undefined); + } finally { + console.error = quiet; + await framework.stop(); + rmSync(dir, { recursive: true, force: true }); + } + }); + } + + it('a maintenance provider call made while a surgery holds the store is withdrawn before it is sent, with the gate closed too', async () => { + const dir = mkdtempSync(join(tmpdir(), 'af-maintenance-withdrawn-')); + const framework = await AgentFramework.create({ + storePath: join(dir, 'store'), + membrane, + agents: [{ name: 'agent', model: 'test-model', systemPrompt: 'test', strategy: new GatedTickStrategy() }], + modules: [], + syncIntervalMs: 0, + maintenanceIntervalMs: 0, + }); + const internals = framework as unknown as { + reserveStoreForSurgery(verb: string, agentName: string): () => void; + withAuxiliaryAdmission(name: string, run: () => Promise): Promise; + providerGate(name: string): { primaryPending: boolean }; + }; + try { + for (const closed of [false, true]) { + let ran = false; + const release = internals.reserveStoreForSurgery('roll back', 'agent'); + // Closed: a cooldown's retry turn is queued, which the hold keeps from + // starting, so a call that parked would never be woken. + internals.providerGate('agent').primaryPending = closed; + try { + const admitted = internals.withAuxiliaryAdmission('agent', async () => { ran = true; }); + void admitted.catch(() => {}); + const outcome = await Promise.race([ + admitted.then(() => 'admitted', (error: unknown) => error), + new Promise((resolve) => setTimeout(() => resolve('parked'), 1_000)), + ]); + assert.ok( + outcome instanceof Error && outcome.name === 'MaintenanceWithdrawnError' + && outcome.message === 'maintenance call withdrawn: a live roll back for agent holds the store', + `${closed ? 'gate closed' : 'gate open'}: ${outcome instanceof Error ? outcome.message : String(outcome)}`, + ); + assert.equal(ran, false); + } finally { + internals.providerGate('agent').primaryPending = false; + release(); + } + } + assert.equal(await internals.withAuxiliaryAdmission('agent', async () => 7), 7, 'admitted again once the hold is released'); + } finally { + await framework.stop(); + rmSync(dir, { recursive: true, force: true }); + } + }); + it('a surgery that takes the store while resume waits for a tick is refused by resume after the wait', { timeout: 10_000 }, async () => { const dir = mkdtempSync(join(tmpdir(), 'af-maintenance-resume-rechecks-')); const strategy = new GatedTickStrategy(); From ff94afa3361dbf6d7e82461d25d3d5470f45b3cc Mon Sep 17 00:00:00 2001 From: Ada-1017 Date: Wed, 14 Oct 2026 12:34:41 -0400 Subject: [PATCH 24/30] fix(api): a branch switch that waits for running maintenance announces where it went, and the current branch is the store's switchBranch now moves the store only once maintenance already running has finished, so the move can come after the call returns. The API server read its destination right after calling it: a switch made during a pass announced main -> main and left the server's cached branch on main, so branch.current, the branch list, /health and /status kept naming it. Now the API server reads the store's branch whenever it reports one, and keeps the name it last read only for once the store has closed (lastKnownBranch). A rollback or a host's own switch can no longer leave it stale either, and undo and redo no longer set it by hand. A switch's branch:switched event names its own move, from the branch read when it began to the one it was asked for; one that fails after moving the store still announces it, and a refusal announces nothing, as before. Test: an API switch made while a pass runs leaves the store and the current branch on main until the pass ends, then announces main -> side, with the current branch side. It fails against the previous server. The two tests that modelled a stale cache now set the name kept for a closed store. --- src/api/server.ts | 47 +++++++++++++------------- test/stop-with-host-operations.test.ts | 33 ++++++++++++++++-- 2 files changed, 54 insertions(+), 26 deletions(-) diff --git a/src/api/server.ts b/src/api/server.ts index 0c7502229..0c6b0fed6 100644 --- a/src/api/server.ts +++ b/src/api/server.ts @@ -78,7 +78,20 @@ export class ApiServer { private wss: WebSocketServer | null = null; private httpServer: ReturnType | null = null; private clients: Set = new Set(); - private currentBranch = 'main'; + /** The store's branch as last read: what the API reports once the store has closed. */ + private lastKnownBranch = 'main'; + + /** + * The store's current branch, read from the store while it's open. A cached + * name would go stale behind any move the API server didn't make itself (a + * rollback, a host's switch), and behind a switch whose move comes after + * its call returns, as one does once running maintenance has finished. + */ + private get currentBranch(): string { + const store = this.framework.getStore(); + if (!store.isClosed()) this.lastKnownBranch = store.currentBranch().name; + return this.lastKnownBranch; + } /** Map from client to their subscription IDs */ private clientSubscriptions: Map> = new Map(); @@ -572,43 +585,31 @@ export class ApiServer { throw new Error('name is required'); } - const store = this.framework.getStore(); - const from = store.isClosed() ? this.currentBranch : store.currentBranch().name; + const from = this.currentBranch; // The framework switches the store, starts the delivery of its Discord // awareness marks behind the data-plane gate, and re-materializes the // config mount. It refuses during a live turn or once stop() has begun, - // and stop() waits for one already running. - // The store has moved when the call returns, before its promise settles - // (a refusal moves nothing). The branch read then is this switch's - // destination: another switch can move the store before this one's - // awaited steps end, so the event names what was read here. - const switching = this.framework.switchBranch(params.name); - void switching.catch(() => {}); // awaited below; handled even if the next line throws - this.followStoreBranch(); - const to = this.currentBranch; - const moved = to !== from; + // and stop() waits for one already running. It moves the store once any + // maintenance already running has finished, so the move can come after + // this call returns. The event names this switch's own move, from the + // branch read here to the one it was asked for: another switch can move + // the store again before this one continues. let switched = false; try { - await switching; + await this.framework.switchBranch(params.name); switched = true; } finally { // A later step can fail after the store has moved: clients still hear // where it went. A refusal moved nothing, so there's nothing to tell. - if (switched || moved) { - this.broadcast('branch:switched', { from, to }); + if (switched || (this.currentBranch === params.name && params.name !== from)) { + this.broadcast('branch:switched', { from, to: params.name }); } } return { switched: true }; } - /** Take the current branch from the store, while it's open. */ - private followStoreBranch(): void { - const store = this.framework.getStore(); - if (!store.isClosed()) this.currentBranch = store.currentBranch().name; - } - private async cmdBranchCurrent(): Promise<{ name: string }> { return { name: this.currentBranch }; } @@ -646,7 +647,6 @@ export class ApiServer { const result = this.framework.undoLastTurn(params.agentName); if (result.undone) { - this.currentBranch = result.toBranch!; // Marks follow behind the data-plane gate; the undo doesn't wait for Discord. this.framework.startDiscordAwarenessDelivery('undo', params.agentName); this.broadcast('branch:switched', { @@ -667,7 +667,6 @@ export class ApiServer { const result = this.framework.redo(params.agentName); if (result.redone) { - this.currentBranch = result.toBranch!; // Marks follow behind the data-plane gate; the redo doesn't wait for Discord. this.framework.startDiscordAwarenessDelivery('redo', params.agentName); this.broadcast('branch:switched', { diff --git a/test/stop-with-host-operations.test.ts b/test/stop-with-host-operations.test.ts index cfd9feba5..1685b13de 100644 --- a/test/stop-with-host-operations.test.ts +++ b/test/stop-with-host-operations.test.ts @@ -724,12 +724,41 @@ describe('stop() with an operator operation in flight', () => { const broadcasts: Array<[string, unknown]> = []; api.broadcast = (event, data) => { broadcasts.push([event, data]); }; // As after a rollback made outside the API server. - (api as unknown as { currentBranch: string }).currentBranch = 'stale'; + (api as unknown as { lastKnownBranch: string }).lastKnownBranch = 'stale'; await api.cmdBranchSwitch({ name: 'side' }); assert.deepEqual(broadcasts, [['branch:switched', { from: 'main', to: 'side' }]]); } finally { await framework.stop(); } }); + it('a switch from the API server that waits for running maintenance names the branch it moved to, and the current branch is the store\'s throughout', async () => { + const framework = await create(); + try { + const api = new ApiServer(framework) as unknown as { + cmdBranchCreate(params: { name: string }): Promise; + cmdBranchSwitch(params: { name: string }): Promise; + cmdBranchCurrent(): Promise<{ name: string }>; + broadcast(event: string, data: unknown): void; + }; + await api.cmdBranchCreate({ name: 'side' }); + const broadcasts: Array<[string, unknown]> = []; + api.broadcast = (event, data) => { broadcasts.push([event, data]); }; + // A maintenance pass still running: the switch moves the store once it ends. + let finish!: () => void; + internals(framework).maintenancePass = new Promise((resolve) => { finish = resolve; }); + const switching = api.cmdBranchSwitch({ name: 'side' }); + await new Promise((resolve) => setTimeout(resolve, 20)); + assert.equal(framework.getStore().currentBranch().name, 'main', 'the switch waits for the pass'); + assert.equal((await api.cmdBranchCurrent()).name, 'main'); + assert.deepEqual(broadcasts, []); + finish(); + internals(framework).maintenancePass = null; + await switching; + assert.equal(framework.getStore().currentBranch().name, 'side'); + assert.equal((await api.cmdBranchCurrent()).name, 'side'); + assert.deepEqual(broadcasts, [['branch:switched', { from: 'main', to: 'side' }]]); + } finally { await framework.stop(); } + }); + it('two API switches handled together each announce their own move', async () => { // With no workspace module, a switch holds the store for no await, so a // second one can move the store before the first's continuation runs. @@ -849,7 +878,7 @@ describe('stop() with an operator operation in flight', () => { const broadcasts: unknown[] = []; (api as unknown as { broadcast(event: string, data: unknown): void }).broadcast = (event, data) => { broadcasts.push([event, data]); }; // A stale cache, as after a rollback made outside the API server. - (api as unknown as { currentBranch: string }).currentBranch = 'stale'; + (api as unknown as { lastKnownBranch: string }).lastKnownBranch = 'stale'; // A turn holds its token from dequeue, before its stream registers. const tokens = internals(framework).activeTurnTokens; tokens.set('agent', -1); From ebef76ee3840ae919106824646e4216fe5523ec5 Mon Sep 17 00:00:00 2001 From: Ada-1017 Date: Wed, 14 Oct 2026 12:36:09 -0400 Subject: [PATCH 25/30] fix(maintenance): a lasting failure's reminder counts from the webhook's last post of it A failure that lasts is alerted again once opsAlert's cooldown for it has passed, so the webhook, which posts a kind at most once per cooldown, hears it. The cooldown was counted from the failure's last report, and a report of changed text moves that report's time even when the webhook, still within its own cooldown, posts nothing. A post at minute 0 and a changed text at minute 14 put the reminder due at 15 off until 29. Now the same failure again counts from the webhook's last post of it (opsAlertLastSent for the agent's context-maintenance-failed), and only with no webhook from its last report. Test, with the real opsAlert and a recorded webhook: a failure posted at minute 0 whose text changes at minute 14 is posted again, with the changed text, once 15 minutes have passed since the post. Counted from the report, it isn't. --- src/framework.ts | 17 +++++++--- test/maintenance-timer.test.ts | 58 ++++++++++++++++++++++++++++++++++ 2 files changed, 70 insertions(+), 5 deletions(-) diff --git a/src/framework.ts b/src/framework.ts index a1970c746..f5f8408fc 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -3594,16 +3594,23 @@ export class AgentFramework { * with the whole text. A failure that lasts, such as a manager whose * strategy failed to initialize, meets every pass, a few seconds apart: * each run records it, and it is logged and alerted again only when its - * whole text changes, once opsAlert's cooldown for it has passed (so the - * webhook, which posts a kind at most once per cooldown, hears a failure - * that lasts), or after a pass in which that agent's maintenance went - * without a failure (maintenanceFailureReported). + * whole text changes, once opsAlert's cooldown for it has passed since the + * webhook last posted it (so the webhook, which posts a kind at most once + * per cooldown, hears a failure that lasts), or after a pass in which that + * agent's maintenance went without a failure (maintenanceFailureReported). */ private reportMaintenanceFailure(agentName: string, record: ContextMaintenanceAgentRun, wholeError: string): void { record.error = boundFailureText(wholeError); const reported = this.maintenanceFailureReported.get(agentName); const now = Date.now(); - if (reported && reported.text === wholeError && now - reported.at < this.opsAlertCooldownMs) return; + if (reported && reported.text === wholeError) { + // The same failure again is alerted once the webhook's cooldown for it + // has passed, counted from the webhook's last post of it: a report of + // changed text that the webhook didn't post doesn't push the reminder + // back. With no webhook, it's counted from this failure's last report. + const since = this.opsAlertLastSent.get(`${agentName}:context-maintenance-failed`) ?? reported.at; + if (now - since < this.opsAlertCooldownMs) return; + } this.maintenanceFailureReported.set(agentName, { text: wholeError, at: now }); console.error(`[context-maintenance] agent=${agentName} failed:`, record.error); this.opsAlert( diff --git a/test/maintenance-timer.test.ts b/test/maintenance-timer.test.ts index d620ee8b5..963367ca5 100644 --- a/test/maintenance-timer.test.ts +++ b/test/maintenance-timer.test.ts @@ -868,6 +868,64 @@ describe('queued context maintenance timer', () => { } }); + it('a failure that lasts is posted to the webhook again a cooldown after its last post, whatever changed-text report came between', async () => { + const dir = mkdtempSync(join(tmpdir(), 'af-maintenance-webhook-reminder-')); + const strategy = new SayingStrategy(); + const framework = await AgentFramework.create({ + storePath: join(dir, 'store'), + membrane, + agents: [{ name: 'saying', model: 'test-model', systemPrompt: 'test', strategy }], + modules: [], + syncIntervalMs: 0, + maintenanceIntervalMs: 10, + }); + const internals = framework as unknown as { runQueuedMaintenance: () => Promise }; + let passes = 0; + const runPass = internals.runQueuedMaintenance.bind(framework); + internals.runQueuedMaintenance = () => { passes++; return runPass(); }; + const passed = async (n: number) => { const after = passes; await waitFor(() => passes >= after + n); }; + // The real opsAlert, with a webhook whose posts are recorded. + const posts: string[] = []; + const realFetch = globalThis.fetch; + globalThis.fetch = (async (_url: unknown, init?: { body?: string }) => { + posts.push(String(JSON.parse(String(init?.body)).content)); + return new Response('{}'); + }) as typeof fetch; + const hook = process.env.CONNECTOME_OPS_WEBHOOK; + process.env.CONNECTOME_OPS_WEBHOOK = 'http://webhook.invalid/hook'; + const realNow = Date.now; + const start = realNow(); + let offset = 0; + Date.now = () => realNow() + offset; + const minute = 60_000; + const maintenancePosts = () => posts.filter((post) => post.includes('context-maintenance-failed')); + const quiet = console.error; + console.error = () => {}; + try { + framework.start(); + await waitFor(() => maintenancePosts().length === 1); + assert.match(maintenancePosts()[0]!, /the first refusal/); + // 14 minutes on, the text changes: it's reported, and the webhook, + // still within its cooldown, doesn't post it. + offset = 14 * minute; + strategy.says = 'the second refusal'; + await passed(3); + assert.equal(maintenancePosts().length, 1); + // At 15 minutes the webhook's cooldown since its post has passed. + offset = 15 * minute + 1_000; + await waitFor(() => maintenancePosts().length === 2); + assert.match(maintenancePosts()[1]!, /the second refusal/); + assert.ok(Date.now() - start < 16 * minute); + } finally { + console.error = quiet; + Date.now = realNow; + globalThis.fetch = realFetch; + if (hook === undefined) delete process.env.CONNECTOME_OPS_WEBHOOK; else process.env.CONNECTOME_OPS_WEBHOOK = hook; + await framework.stop(); + rmSync(dir, { recursive: true, force: true }); + } + }); + it('a readiness refusal read after an agent\'s last tick is that agent\'s failure, alerted once', async () => { const dir = mkdtempSync(join(tmpdir(), 'af-maintenance-refused-after-ticks-')); const strategy = new RefusesAfterTicksStrategy(); From 5985987f6725c5991b1a7d287855ae153d471738 Mon Sep 17 00:00:00 2001 From: Ada-1017 Date: Wed, 14 Oct 2026 12:41:40 -0400 Subject: [PATCH 26/30] perf(removals): a message's body is found in blob-free windows, not a full read of the branch removeStoredMessage, which a module's removal, the coalescer's and the Discord history sweep use, read the whole branch with getAllMessages() to find a body's bounds. That inflates every attachment on the branch, and a sweep does it once for each message it removes. It now finds the message the way the live surgeries do: locateMessageIndex scans from the tail in blob-free windows, and bodyGroupRun takes the body's bounds from the store's own alignToBodyGroups walk, which reads a body as context-manager 0.14's body-runs rule does. The helpers (and the read surface they need) move from framework.ts to stored-removal.ts, beside the removal they now serve too, and framework.ts imports them. Test: a module's removal of a message in a body takes the whole body, and a plain message alone, with getAllMessages() refusing to be called. It fails against the full read. --- src/framework.ts | 75 ++++----------------------------- src/stored-removal.ts | 79 +++++++++++++++++++++++++++++++++-- test/operator-surgery.test.ts | 19 ++++++--- 3 files changed, 97 insertions(+), 76 deletions(-) diff --git a/src/framework.ts b/src/framework.ts index f5f8408fc..a60e49ae4 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -19,7 +19,14 @@ import type { StoredMessage, TokenBudget, } from '@animalabs/context-manager'; -import { bodyRangeIn, removeStoredMessage, removeStoredMessages } from './stored-removal.js'; +import { + bodyGroupRun, + bodyRangeIn, + locateMessageIndex, + locateMessageIndices, + removeStoredMessage, + removeStoredMessages, +} from './stored-removal.js'; import type { FrameworkConfig, InferencePolicy, @@ -856,72 +863,6 @@ class DefaultInferencePolicy implements InferencePolicy { } } -/** Minimal read surface the live-surgery helpers need from a ContextManager. */ -interface SurgeryReadableCm { - getMessageCount(): number; - getMessageWindow( - offset: number, - limit: number, - opts?: { resolveBlobs?: boolean; alignToBodyGroups?: boolean }, - ): { messages: Array<{ id: unknown; bodyGroupId?: string }>; startIndex: number }; -} - -const SURGERY_SCAN_WINDOW = 500; - -/** - * Slot index of `messageId` on the active branch, or -1. Scans from the tail - * in blob-free windows: operator targets are almost always recent, and a - * full `getAllMessages()` re-inflates every attachment on the branch. - */ -function locateMessageIndex(cm: SurgeryReadableCm, messageId: string): number { - const hit = locateMessageIndices(cm, new Set([messageId])).get(messageId); - return hit ? hit.index : -1; -} - -/** Tail-first scan resolving many ids at once; stops as soon as all are found. */ -function locateMessageIndices( - cm: SurgeryReadableCm, - ids: Set, -): Map { - const found = new Map(); - const total = cm.getMessageCount(); - for (let end = total; end > 0 && found.size < ids.size; end -= SURGERY_SCAN_WINDOW) { - const start = Math.max(0, end - SURGERY_SCAN_WINDOW); - const { messages, startIndex } = cm.getMessageWindow(start, end - start, { resolveBlobs: false }); - for (let i = messages.length - 1; i >= 0; i--) { - const id = String(messages[i].id); - if (ids.has(id) && !found.has(id)) { - found.set(id, { - index: startIndex + i, - ...(messages[i].bodyGroupId ? { bodyGroupId: messages[i].bodyGroupId } : {}), - }); - } - } - } - return found; -} - -/** - * Inclusive slot range of the body-group run containing `index` (the slot - * itself when it is not sharded). Delegates to the store's own - * `alignToBodyGroups` edge walk — one contiguity assumption, not two. - */ -function bodyGroupRun( - cm: SurgeryReadableCm, - index: number, -): { from: number; to: number; fromId: string; toId: string; group: boolean } { - const win = cm.getMessageWindow(index, 1, { alignToBodyGroups: true, resolveBlobs: false }); - const from = win.startIndex; - const to = win.startIndex + win.messages.length - 1; - return { - from, - to, - fromId: String(win.messages[0].id), - toId: String(win.messages[win.messages.length - 1].id), - group: win.messages[0].bodyGroupId !== undefined, - }; -} - function describeMarkers(m: SurgeryMarkerReceipt): string { if (m.status === 'queued') return `queued:${m.queued}`; if (m.status === 'not-scheduled') return 'not-scheduled(batch retired)'; diff --git a/src/stored-removal.ts b/src/stored-removal.ts index bd146579f..c0b73d8d0 100644 --- a/src/stored-removal.ts +++ b/src/stored-removal.ts @@ -3,6 +3,72 @@ import { bodyBoundsIn, type BodyShard, type ContextManager, type MessageId } fro type StoredLike = BodyShard & { id: MessageId }; type Removals = Pick; +/** Minimal read surface the live-surgery helpers need from a ContextManager. */ +export interface SurgeryReadableCm { + getMessageCount(): number; + getMessageWindow( + offset: number, + limit: number, + opts?: { resolveBlobs?: boolean; alignToBodyGroups?: boolean }, + ): { messages: Array<{ id: unknown; bodyGroupId?: string }>; startIndex: number }; +} + +const SURGERY_SCAN_WINDOW = 500; + +/** + * Slot index of `messageId` on the active branch, or -1. Scans from the tail + * in blob-free windows: operator targets are almost always recent, and a + * full `getAllMessages()` re-inflates every attachment on the branch. + */ +export function locateMessageIndex(cm: SurgeryReadableCm, messageId: string): number { + const hit = locateMessageIndices(cm, new Set([messageId])).get(messageId); + return hit ? hit.index : -1; +} + +/** Tail-first scan resolving many ids at once; stops as soon as all are found. */ +export function locateMessageIndices( + cm: SurgeryReadableCm, + ids: Set, +): Map { + const found = new Map(); + const total = cm.getMessageCount(); + for (let end = total; end > 0 && found.size < ids.size; end -= SURGERY_SCAN_WINDOW) { + const start = Math.max(0, end - SURGERY_SCAN_WINDOW); + const { messages, startIndex } = cm.getMessageWindow(start, end - start, { resolveBlobs: false }); + for (let i = messages.length - 1; i >= 0; i--) { + const id = String(messages[i].id); + if (ids.has(id) && !found.has(id)) { + found.set(id, { + index: startIndex + i, + ...(messages[i].bodyGroupId ? { bodyGroupId: messages[i].bodyGroupId } : {}), + }); + } + } + } + return found; +} + +/** + * Inclusive slot range of the body-group run containing `index` (the slot + * itself when it is not sharded). Delegates to the store's own + * `alignToBodyGroups` edge walk — one contiguity assumption, not two. + */ +export function bodyGroupRun( + cm: SurgeryReadableCm, + index: number, +): { from: number; to: number; fromId: string; toId: string; group: boolean } { + const win = cm.getMessageWindow(index, 1, { alignToBodyGroups: true, resolveBlobs: false }); + const from = win.startIndex; + const to = win.startIndex + win.messages.length - 1; + return { + from, + to, + fromId: String(win.messages[0].id), + toId: String(win.messages[win.messages.length - 1].id), + group: win.messages[0].bodyGroupId !== undefined, + }; +} + /** * Remove the stored messages `first`..`last` (inclusive, in store order). * @@ -32,10 +98,15 @@ export function removeStoredMessage(cm: ContextManager, id: MessageId): void { cm.removeMessage(id); return; } - const all = cm.getAllMessages(); - const at = all.findIndex((m) => m.id === id); - const { from, to } = bodyRangeIn(all, at); - removeStoredMessages(cm, all[from] as StoredLike, all[to] as StoredLike); + // Found in blob-free windows: a full read of the branch would inflate every + // attachment on it, once for each message a sweep removes. + const index = locateMessageIndex(cm, String(id)); + if (index < 0) { + cm.removeMessage(id); + return; + } + const run = bodyGroupRun(cm, index); + cm.removeMessages(run.fromId as MessageId, run.toId as MessageId); } /** removeStoredMessages for an interval known by its ids. */ diff --git a/test/operator-surgery.test.ts b/test/operator-surgery.test.ts index e19e6e337..b1eb37908 100644 --- a/test/operator-surgery.test.ts +++ b/test/operator-surgery.test.ts @@ -482,14 +482,23 @@ describe('live operator surgery', () => { assert.deepEqual(remainingTexts(), ['m1', 'm2', 'm3', 'm4', 'm5']); }); - it("a module's removal of a message in a body takes the whole body, and a plain message alone", async () => { + it("a module's removal of a message in a body takes the whole body, and a plain message alone, with no full read of the branch", async () => { const [b0, b1, b2] = appendShards('g-mod', ['b0', 'b1', 'b2']); const plain = String(cm().addMessage('user', [{ type: 'text', text: 'plain' }])); const internals = framework as unknown as { removeMessage(id: string): void }; - internals.removeMessage(b1); - for (const id of [b0, b1, b2]) assert.equal(cm().getMessage(id as never), null); - internals.removeMessage(plain); - assert.equal(cm().getMessage(plain as never), null); + // A full read inflates every attachment on the branch: the body is found + // in blob-free windows instead. + const reads = cm() as unknown as { getAllMessages(): unknown }; + const getAllMessages = reads.getAllMessages; + reads.getAllMessages = () => { throw new Error('read the whole branch'); }; + try { + internals.removeMessage(b1); + for (const id of [b0, b1, b2]) assert.equal(cm().getMessage(id as never), null); + internals.removeMessage(plain); + assert.equal(cm().getMessage(plain as never), null); + } finally { + reads.getAllMessages = getAllMessages; + } assert.deepEqual(remainingTexts(), ['m1', 'm2', 'm3', 'm4', 'm5']); }); From 609b247d62d39cf08e0fcbb6a28470745ff2345e Mon Sep 17 00:00:00 2001 From: Ada-1017 Date: Wed, 14 Oct 2026 12:42:53 -0400 Subject: [PATCH 27/30] fix(hide): a single hide's operator-log entry keeps at most 50 ids, as the range form's does A single hide now takes every stored copy of a message and each copy's whole body, so it can remove many stored messages, and its operator-log entry, which the operator:action trace carries too, listed every id. The range form already bounds them with capIds; the single form now does the same: the first 50 ids, and removedIdsTruncated when there were more. Test: hiding a message stored as a 60-shard body removes every shard and logs the first 50 ids with removedIdsTruncated. Uncapped, it logs all 60. --- src/framework.ts | 10 +++++++++- test/operator-surgery.test.ts | 11 +++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/src/framework.ts b/src/framework.ts index a60e49ae4..763499749 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -7071,12 +7071,20 @@ export class AgentFramework { `[host-command] hide agent=${agentName} removed=${hidden} (${params.fromMessageId}) ` + `by=${params.requesterName ?? params.requesterId ?? 'unknown'} (server=${serverId})`, ); + // Every shard of every copy goes, so the ids are bounded as the + // range form's are. + const loggedIds = capIds(hiddenIds); this.recordOperatorAction({ kind: 'hide', agent: agentName, requester: hostCommandRequester(serverId, params), params: { fromMessageId: params.fromMessageId }, - result: { branch: this.store.currentBranch().name, hidden, removedIds: hiddenIds }, + result: { + branch: this.store.currentBranch().name, + hidden, + removedIds: loggedIds.ids, + ...(loggedIds.truncated ? { removedIdsTruncated: true } : {}), + }, }); return { ok: true, diff --git a/test/operator-surgery.test.ts b/test/operator-surgery.test.ts index b1eb37908..46bacc245 100644 --- a/test/operator-surgery.test.ts +++ b/test/operator-surgery.test.ts @@ -418,6 +418,17 @@ describe('live operator surgery', () => { assert.deepEqual(refsOf(copies), ['d-cut']); for (const id of [...partial, ...retry]) assert.equal(cm().getMessage(id as never), null); assert.deepEqual(remainingTexts(), ['m1', 'm2', 'm3', 'm4', 'm5']); + + // A body of more shards than the operator log keeps ids for: its entry + // keeps the first 50 and says the list is cut. + const huge = appendBody('d-huge', 'g-huge', Array.from({ length: 60 }, (_, i) => `h${i}`)); + const big = await hide('d-huge'); + assert.equal(big.ok, true, String(big.error)); + assert.equal(big.hidden, 1); + const entry = readLog(framework.getOperatorLogPath()!).filter((e) => e.kind === 'hide').at(-1)!; + assert.deepEqual(entry.result?.removedIds, huge.slice(0, 50)); + assert.equal(entry.result?.removedIdsTruncated, true); + for (const id of huge) assert.equal(cm().getMessage(id as never), null); }); it('hide refuses while a surgery holds the store, and a missing Discord id matches nothing', async () => { From 194b17a697270d9ae451f378afa145d63089893d Mon Sep 17 00:00:00 2001 From: Ada-1017 Date: Wed, 14 Oct 2026 12:43:56 -0400 Subject: [PATCH 28/30] docs(changelog): the round's maintenance withdrawal, the API's branch reads, the webhook reminder, and hide's copies and log cap - cm-0.14.changed: while a surgery or switchBranch holds the store, maintenance makes no provider call; one made then, or waiting at the provider gate when the hold is taken, is withdrawn unsent, is no failure, and the next pass redoes the work, so the operator call never waits on a cooldown or the retry turn it queues. switchBranch can move the store after the call returns: the API server's branch.switch announces the move once made, and branch.current, the list, /health and /status read the store's branch. A lasting failure's 15-minute reminder counts from the webhook's last post of it. - body-removals.fixed: hide takes every stored copy of a body, refuses while a surgery holds the store, never matches a message with no Discord id, and logs at most 50 ids (removedIdsTruncated); a module's and the coalescer's removal take the whole body. --- changelog.d/body-removals.fixed.md | 2 +- changelog.d/cm-0.14.changed.md | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/changelog.d/body-removals.fixed.md b/changelog.d/body-removals.fixed.md index cb84f3877..d71c3d50a 100644 --- a/changelog.d/body-removals.fixed.md +++ b/changelog.d/body-removals.fixed.md @@ -1 +1 @@ -- Removing a single message that belongs to a body, a one-shard body's included, no longer fails: context-manager's `removeMessage` refuses any message in a body, so agent-framework now removes one as a range in offline recovery's suppressions, and in a Discord suppression resumed after a crash. The host `hide` command now hides a message stored as a body (a long one, in shards) whole, and an automatic rewind whose newest message is a shard of a body takes the whole body. +- Removing a single message that belongs to a body, a one-shard body's included, no longer fails: context-manager's `removeMessage` refuses any message in a body, so agent-framework now removes one as a range in offline recovery's suppressions, and in a Discord suppression resumed after a crash. The host `hide` command now hides a message stored as a body (a long one, in shards) whole, with every stored copy of it (a write cut short leaves a partial copy before its retry), and its operator-log entry lists at most 50 of the ids it removed, with `removedIdsTruncated` when there were more. It's refused as busy while a surgery holds the store, and a message with no Discord id never matches one. A module's removal of a message it added, and the coalescer's, take its whole body too, and an automatic rewind whose newest message is a shard of a body takes the whole body. diff --git a/changelog.d/cm-0.14.changed.md b/changelog.d/cm-0.14.changed.md index bd71be775..6b299779c 100644 --- a/changelog.d/cm-0.14.changed.md +++ b/changelog.d/cm-0.14.changed.md @@ -1,2 +1,2 @@ -- A context-maintenance pass no longer fails as a whole when one agent's readiness can't be read. context-manager 0.14, which agent-framework has required since 0.21.1, refuses its readiness queries while a strategy's initialization is pending (inside `switchBranch`, `fork`, `setStrategy` or `ContextManager.open`) or has failed, and a strategy can refuse on its own, as the autobiographical one does for a branch it hasn't reloaded. That agent's run now records the error, the agent gets its own `context-maintenance-failed` alert (`scope: 'agent'`), and the other agents run. Before, such a throw ended the pass for every agent, with one alert for the whole pass (`scope: 'pass'`). Anything else that throws while one agent's maintenance is prepared is that agent's failure in the same way. A failure that lasts, met by every pass, is recorded on each run, and logged and alerted once, then again when its text changes, every 15 minutes while it lasts (the alert webhook's cooldown for it), or after that agent's maintenance has gone without a failure. `getContextMaintenanceSnapshot()` reports such an agent as `ready: false` with an `error`, and a strategy that can't give a progress snapshot reads as having none. -- While an operator surgery (`rollbackToMessage` or `suppressMessages`) or `switchBranch` holds the store, no maintenance pass starts and no over-budget drain kick is started, a running pass or drain kick starts no new tick, and the operator call waits for them to finish before it switches or forks the branch, so no maintenance work overlaps the move. `maintenanceTick()` answers `ran: false` meanwhile, and `resume()` refuses with `OperatorActionError` (`agent-busy`), as the other operator calls do, including when the surgery took the store while `resume()` waited. Resume once the surgery has finished. +- A context-maintenance pass no longer fails as a whole when one agent's readiness can't be read. context-manager 0.14, which agent-framework has required since 0.21.1, refuses its readiness queries while a strategy's initialization is pending (inside `switchBranch`, `fork`, `setStrategy` or `ContextManager.open`) or has failed, and a strategy can refuse on its own, as the autobiographical one does for a branch it hasn't reloaded. That agent's run now records the error, the agent gets its own `context-maintenance-failed` alert (`scope: 'agent'`), and the other agents run. Before, such a throw ended the pass for every agent, with one alert for the whole pass (`scope: 'pass'`). Anything else that throws while one agent's maintenance is prepared is that agent's failure in the same way. A failure that lasts, met by every pass, is recorded on each run, and logged and alerted once, then again when its text changes, every 15 minutes while it lasts (the alert webhook's cooldown for it, counted from the webhook's last post of it), or after that agent's maintenance has gone without a failure. `getContextMaintenanceSnapshot()` reports such an agent as `ready: false` with an `error`, and a strategy that can't give a progress snapshot reads as having none. +- While an operator surgery (`rollbackToMessage` or `suppressMessages`) or `switchBranch` holds the store, no maintenance pass starts and no over-budget drain kick is started, a running pass or drain kick starts no new tick, and the operator call waits for them to finish before it switches or forks the branch, so no maintenance work overlaps the move. Maintenance makes no provider call meanwhile: one made then, or one waiting at the agent's provider gate when the hold is taken, is withdrawn without being sent, which is no maintenance failure, and the next pass after the hold does that work again. The operator call never waits on a provider cooldown, or on the retry turn a cooldown queues, which the hold keeps from starting. `switchBranch` can therefore move the store after the call returns, and the API server's `branch.switch` announces the move it made once it has, while `branch.current`, the branch list, `/health` and `/status` read the store's branch whenever they report one. `maintenanceTick()` answers `ran: false` meanwhile, and `resume()` refuses with `OperatorActionError` (`agent-busy`), as the other operator calls do, including when the surgery took the store while `resume()` waited. Resume once the surgery has finished. From fb8a6c4f98e58c91dd160e3909fb8f54fce72a9d Mon Sep 17 00:00:00 2001 From: Ada-1017 Date: Wed, 14 Oct 2026 13:00:43 -0400 Subject: [PATCH 29/30] fix(maintenance): a parked call woken with its gate still closed parks again; both of context-manager's lanes said as they are Gladys-2413's read of the round: - withAuxiliaryAdmission waited once and then went on. Only the gate opening, stop() closing admission and a surgery taking the store wake a parked call, and today each woken call meets one of them. It now loops: a call woken while the gate is still closed checks the hold and parks again, so no later waker can send one into a cooldown. - The text said context-manager keeps a withdrawn call's work queued, which holds for merges alone. A chunk's compression has already left the queue when its call is withdrawn: context-manager logs "Failed to compress chunk (hierarchical)" with the error and records the call, and the chunk, still uncompressed, is queued again at that manager's next rebuild (its initialization, a message, a sent compile, a released hold). MaintenanceWithdrawnError's doc and the changed fragment now say both, and name the log line. Test: a parked call woken while its gate is still closed parks again and goes once the gate opens. Waiting once, it goes out at the stray wake. --- changelog.d/cm-0.14.changed.md | 2 +- src/framework.ts | 18 +++++++++++++----- test/maintenance-timer.test.ts | 34 ++++++++++++++++++++++++++++++++++ 3 files changed, 48 insertions(+), 6 deletions(-) diff --git a/changelog.d/cm-0.14.changed.md b/changelog.d/cm-0.14.changed.md index 6b299779c..8e997f2ea 100644 --- a/changelog.d/cm-0.14.changed.md +++ b/changelog.d/cm-0.14.changed.md @@ -1,2 +1,2 @@ - A context-maintenance pass no longer fails as a whole when one agent's readiness can't be read. context-manager 0.14, which agent-framework has required since 0.21.1, refuses its readiness queries while a strategy's initialization is pending (inside `switchBranch`, `fork`, `setStrategy` or `ContextManager.open`) or has failed, and a strategy can refuse on its own, as the autobiographical one does for a branch it hasn't reloaded. That agent's run now records the error, the agent gets its own `context-maintenance-failed` alert (`scope: 'agent'`), and the other agents run. Before, such a throw ended the pass for every agent, with one alert for the whole pass (`scope: 'pass'`). Anything else that throws while one agent's maintenance is prepared is that agent's failure in the same way. A failure that lasts, met by every pass, is recorded on each run, and logged and alerted once, then again when its text changes, every 15 minutes while it lasts (the alert webhook's cooldown for it, counted from the webhook's last post of it), or after that agent's maintenance has gone without a failure. `getContextMaintenanceSnapshot()` reports such an agent as `ready: false` with an `error`, and a strategy that can't give a progress snapshot reads as having none. -- While an operator surgery (`rollbackToMessage` or `suppressMessages`) or `switchBranch` holds the store, no maintenance pass starts and no over-budget drain kick is started, a running pass or drain kick starts no new tick, and the operator call waits for them to finish before it switches or forks the branch, so no maintenance work overlaps the move. Maintenance makes no provider call meanwhile: one made then, or one waiting at the agent's provider gate when the hold is taken, is withdrawn without being sent, which is no maintenance failure, and the next pass after the hold does that work again. The operator call never waits on a provider cooldown, or on the retry turn a cooldown queues, which the hold keeps from starting. `switchBranch` can therefore move the store after the call returns, and the API server's `branch.switch` announces the move it made once it has, while `branch.current`, the branch list, `/health` and `/status` read the store's branch whenever they report one. `maintenanceTick()` answers `ran: false` meanwhile, and `resume()` refuses with `OperatorActionError` (`agent-busy`), as the other operator calls do, including when the surgery took the store while `resume()` waited. Resume once the surgery has finished. +- While an operator surgery (`rollbackToMessage` or `suppressMessages`) or `switchBranch` holds the store, no maintenance pass starts and no over-budget drain kick is started, a running pass or drain kick starts no new tick, and the operator call waits for them to finish before it switches or forks the branch, so no maintenance work overlaps the move. Maintenance makes no provider call meanwhile: one made then, or one waiting at the agent's provider gate when the hold is taken, is withdrawn without being sent. That's no maintenance failure, and context-manager counts no attempt for it. A merge keeps its place in the queue. A chunk's compression logs context-manager's "Failed to compress chunk (hierarchical)" line with the withdrawal, and the chunk is queued again at that manager's next rebuild, such as its next message or compile. The operator call never waits on a provider cooldown, or on the retry turn a cooldown queues, which the hold keeps from starting. `switchBranch` can therefore move the store after the call returns, and the API server's `branch.switch` announces the move it made once it has, while `branch.current`, the branch list, `/health` and `/status` read the store's branch whenever they report one. `maintenanceTick()` answers `ran: false` meanwhile, and `resume()` refuses with `OperatorActionError` (`agent-busy`), as the other operator calls do, including when the surgery took the store while `resume()` waited. Resume once the surgery has finished. diff --git a/src/framework.ts b/src/framework.ts index 763499749..919f722fe 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -723,8 +723,12 @@ function logSeedOmissions( /** * A maintenance provider call withdrawn because a surgery holds the store. It * sent nothing: it was refused at the provider gate, or woken there from its - * wait. context-manager keeps the work it was for queued, with no attempt - * counted, and the next pass after the hold does it again. + * wait. context-manager counts no attempt for it. A merge keeps its queue + * entry, so the next tick after the hold does it again. A chunk's + * compression has already left the queue: it logs "Failed to compress chunk + * (hierarchical)" with this error and records the call, and the chunk, still + * uncompressed, is queued again at that manager's next rebuild (its next + * initialization, message, sent compile or released hold). */ class MaintenanceWithdrawnError extends Error { constructor(hold: { verb: string; agentName: string }) { @@ -2833,19 +2837,23 @@ export class AgentFramework { const gate = this.providerGate(agentName); this.refuseHeldProviderModel(agentName, model); this.withdrawMaintenanceUnderSurgery(); - if (this.providerGateBlocked(agentName)) { + // A parked call is woken when the gate opens, when stop() closes + // admission, or when a surgery takes the store; one woken while the gate + // is still closed parks again. + while (this.providerGateBlocked(agentName)) { gate.deferredAuxiliary++; try { await new Promise((resolve) => gate.auxiliaryWaiters.push(resolve)); } finally { gate.deferredAuxiliary = Math.max(0, gate.deferredAuxiliary - 1); } + if (this.providerAdmissionClosed) break; + // A surgery that took the store while this call was parked withdraws it. + this.withdrawMaintenanceUnderSurgery(); } if (this.providerAdmissionClosed) { throw new Error(`Provider admission closed while stopping (${agentName})`); } - // A surgery that took the store while this call was parked withdraws it. - this.withdrawMaintenanceUnderSurgery(); // A wait recorded while this call was parked binds it too. this.refuseHeldProviderModel(agentName, model); gate.auxiliaryInFlight++; diff --git a/test/maintenance-timer.test.ts b/test/maintenance-timer.test.ts index 963367ca5..8a7f782ea 100644 --- a/test/maintenance-timer.test.ts +++ b/test/maintenance-timer.test.ts @@ -1138,6 +1138,40 @@ describe('queued context maintenance timer', () => { } }); + it('a parked maintenance call woken while its gate is still closed parks again, and goes once the gate opens', async () => { + const dir = mkdtempSync(join(tmpdir(), 'af-maintenance-parks-again-')); + const framework = await AgentFramework.create({ + storePath: join(dir, 'store'), + membrane, + agents: [{ name: 'agent', model: 'test-model', systemPrompt: 'test', strategy: new GatedTickStrategy() }], + modules: [], + syncIntervalMs: 0, + maintenanceIntervalMs: 0, + }); + const internals = framework as unknown as { + withAuxiliaryAdmission(name: string, run: () => Promise): Promise; + providerGate(name: string): { primaryPending: boolean; auxiliaryWaiters: Array<() => void> }; + flushAuxiliaryAdmission(name: string): void; + }; + try { + const gate = internals.providerGate('agent'); + gate.primaryPending = true; + let ran = false; + const call = internals.withAuxiliaryAdmission('agent', async () => { ran = true; return 7; }); + await waitFor(() => gate.auxiliaryWaiters.length === 1); + for (const resolve of gate.auxiliaryWaiters.splice(0)) resolve(); + await waitFor(() => gate.auxiliaryWaiters.length === 1); + assert.equal(ran, false, 'woken with the gate still closed, it parked again'); + gate.primaryPending = false; + internals.flushAuxiliaryAdmission('agent'); + assert.equal(await call, 7); + assert.equal(ran, true); + } finally { + await framework.stop(); + rmSync(dir, { recursive: true, force: true }); + } + }); + it('a surgery that takes the store while resume waits for a tick is refused by resume after the wait', { timeout: 10_000 }, async () => { const dir = mkdtempSync(join(tmpdir(), 'af-maintenance-resume-rechecks-')); const strategy = new GatedTickStrategy(); From ca2f80681d75c5661bdc93fe209845f37daec9e1 Mon Sep 17 00:00:00 2001 From: Ada-1017 Date: Wed, 14 Oct 2026 14:53:54 -0400 Subject: [PATCH 30/30] test(maintenance): read the first pass the timer recorded, not the newest The maintenance history lists the newest pass first. Six tests waited for at least one pass and then read history[0], meaning the first pass. Under load, a second pass can end before the test reads the history. In "a failure that begins during the pass is that agent's error and alert, not the pass's", that second pass sees the manager already fenced by the first one's tick, so it records 0 ticks, and the test failed (0 !== 1) in a cross-package run whose machine was busy. A firstPass() helper now reads the oldest entry, and each of the six uses it. With a 100 ms wait added before the read, so that later passes are recorded too, the old read fails and the new one passes. maintenance-timer passes 27/27 against context-manager 0.14.0 and against its main at 58a3f9e. --- test/maintenance-timer.test.ts | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/test/maintenance-timer.test.ts b/test/maintenance-timer.test.ts index 8a7f782ea..8b5e6612d 100644 --- a/test/maintenance-timer.test.ts +++ b/test/maintenance-timer.test.ts @@ -404,6 +404,13 @@ class CapturingMembrane { } } +/** The first pass the timer recorded. The history lists the newest first, and + * under load more passes can end before a test reads it. */ +function firstPass(framework: AgentFramework) { + const history = framework.getContextMaintenanceSnapshot().history; + return history[history.length - 1]!; +} + async function waitFor(predicate: () => boolean, timeoutMs = 1000): Promise { const deadline = Date.now() + timeoutMs; while (!predicate()) { @@ -667,7 +674,7 @@ describe('queued context maintenance timer', () => { try { framework.start(); await waitFor(() => framework.getContextMaintenanceSnapshot().history.length >= 1); - const run = framework.getContextMaintenanceSnapshot().history[0].agents[0]; + const run = firstPass(framework).agents[0]; assert.equal(run.agentName, 'agent'); assert.equal(run.ticks, 1); assert.match(run.error ?? '', /requires successful strategy initialization/); @@ -943,7 +950,7 @@ describe('queued context maintenance timer', () => { try { framework.start(); await waitFor(() => framework.getContextMaintenanceSnapshot().history.length >= 1); - const run = framework.getContextMaintenanceSnapshot().history[0].agents[0]; + const run = firstPass(framework).agents[0]; assert.equal(run.ticks, 8); assert.equal(run.error, 'readiness refused after the ticks'); assert.equal(run.readyAfter, undefined); @@ -971,7 +978,7 @@ describe('queued context maintenance timer', () => { try { framework.start(); await waitFor(() => framework.getContextMaintenanceSnapshot().history.length >= 1); - const run = framework.getContextMaintenanceSnapshot().history[0].agents[0]; + const run = firstPass(framework).agents[0]; assert.equal(run.ticks, 0); assert.equal(run.error, 'tick failed'); assert.deepEqual(alerts, ['context-maintenance-failed:agent:tick failed']); @@ -1081,7 +1088,7 @@ describe('queued context maintenance timer', () => { assert.equal(strategy.calls, 0, 'the withdrawn call sent nothing'); assert.notEqual(cm.currentBranch().name, 'main'); assert.deepEqual(alerts.filter((alert) => alert.startsWith('context-maintenance-failed')), []); - const run = framework.getContextMaintenanceSnapshot().history[0]!.agents[0]!; + const run = firstPass(framework).agents[0]!; assert.equal(run.error, undefined); } finally { console.error = quiet; @@ -1308,7 +1315,7 @@ describe('queued context maintenance timer', () => { try { framework.start(); await waitFor(() => framework.getContextMaintenanceSnapshot().history.length >= 1); - const run = framework.getContextMaintenanceSnapshot().history[0].agents[0]; + const run = firstPass(framework).agents[0]; assert.equal(strategy.ticks, 1); assert.equal(run.ticks, 1); assert.equal(run.error, undefined); @@ -1348,7 +1355,7 @@ describe('queued context maintenance timer', () => { try { framework.start(); await waitFor(() => framework.getContextMaintenanceSnapshot().history.length >= 1); - const run = framework.getContextMaintenanceSnapshot().history[0].agents[0]; + const run = firstPass(framework).agents[0]; assert.equal(run.ticks, 0); assert.equal(run.error, 'the tick failed while a surgery waited'); assert.deepEqual(alerts, ['context-maintenance-failed']);