From f7e14f6bc396cc3594f2d8d100544214903124e7 Mon Sep 17 00:00:00 2001 From: Basil-1007 Date: Wed, 7 Oct 2026 05:15:50 -0400 Subject: [PATCH 01/73] =?UTF-8?q?feat(chronicle):=20RecordJournal=20?= =?UTF-8?q?=E2=80=94=20typed=20records=20that=20don't=20follow=20branch=20?= =?UTF-8?q?switches?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Branch state (snapshot, tree and log states) rewinds with a historical rollback; typed chronicle records do not. getRecordIdsByType enumerates them from every branch, in global order (ids come from one store-wide counter), and they survive deleteBranch, reopen and SIGKILL. Three AF journals need exactly that — the workspace's disk-agreement evidence (shelf-383), delivery receipts (shelf-354) and prose-draft outcomes (shelf-351) — so they share one helper instead of three: append typed records, checkpoint the caller's reduced state, replay the tail after the latest checkpoint at open. Durability is the subtle part. A record append reaches the OS before it returns, while branch-state commits are buffered until store.sync(): a probe lost a tree commit made between two appends that both survived a SIGKILL. So the API names two barriers and implies neither from the returned {id, sequence}: - afterCommittedState: sync BEFORE appending, for an entry asserting state committed just before it (it can then never survive without that state); - durable: sync AFTER appending, for an entry that must be on stable storage before the caller acts (an attempt before its outbound request, an intent before its disk effect). One writer per record type: a checkpoint covers entries up to the last one this instance loaded or appended. Entry meaning, reducers, dedup and checkpoint cadence stay with each caller. Tests: empty load; append refs and replay order; entries from a rolled-back branch and main loading from either, through deleteBranch and reopen; checkpoint + tail with interleaved other record types and a tree commit, the newest checkpoint winning; the barrier call order; and a SIGKILL child that commits tree state then appends with afterCommittedState — the entry never survives without the state. (The same kill without the barrier leaves the entry and loses the state.) Agreed in room-220 #41647-#41713 with Oscar-1005, Petra-1012, Hugo-1004. Co-Authored-By: Claude Opus 5.5 --- src/record-journal.ts | 166 ++++++++++++++++++++++++++ test/fixtures/record-journal-kill.mjs | 18 +++ test/record-journal.test.ts | 161 +++++++++++++++++++++++++ 3 files changed, 345 insertions(+) create mode 100644 src/record-journal.ts create mode 100644 test/fixtures/record-journal-kill.mjs create mode 100644 test/record-journal.test.ts diff --git a/src/record-journal.ts b/src/record-journal.ts new file mode 100644 index 000000000..b3affd27e --- /dev/null +++ b/src/record-journal.ts @@ -0,0 +1,166 @@ +/** + * An append-only journal of typed Chronicle records that does not follow + * branch switches. + * + * Chronicle's branch state — snapshot, tree and log states — rewinds with a + * historical rollback (createBranchAt + switchBranch). Typed records do not: + * `getRecordIdsByType` enumerates them from every branch, in global order, and + * they survive `deleteBranch`, reopen and a hard kill. A journal is for state + * that must not rewind with the conversation but should live and die with its + * store: what a workspace's disk last agreed with, what reached a recipient, + * what became of a suppressed draft. + * + * Durability. A record append reaches the OS before `append` returns, so it + * survives a killed process, while branch-state commits are buffered until + * `store.sync()`: a record appended after a state commit can outlive that + * commit. Two barriers, each named where it's needed and neither implied by + * the `{id, sequence}` an append returns: + * - `afterCommittedState: true` syncs BEFORE appending — for an entry that + * asserts state committed just before it, so the entry can never survive + * without that state; + * - `durable: true` syncs AFTER appending — for an entry that must be on + * stable storage before the caller acts on it (an attempt before its + * outbound request, an intent before its disk effect). + * + * Checkpoints bound the replay at open: `checkpoint(snapshot)` records the + * caller's reduced state together with the last entry it covers, and `load()` + * returns the latest checkpoint's snapshot plus only the entries appended + * after it. A journal that never checkpoints replays everything. What entries + * mean, how they reduce, and when to checkpoint stay with the caller. + * + * One writer per record type: a checkpoint covers every entry up to the last + * one THIS journal loaded or appended, so entries another instance appended + * to the same type in between would be covered without being reduced. + */ + +import type { JsStore } from '@animalabs/chronicle'; + +/** Where an appended record landed. Not a durability receipt — see the barriers. */ +export interface JournalRecordRef { + /** Chronicle record id: global, increasing in append order. */ + id: string; + /** The record's sequence on the branch current when it was appended. */ + sequence: number; +} + +export interface JournalEntry extends JournalRecordRef { + entry: Entry; +} + +export interface JournalLoad { + /** The latest checkpoint's snapshot, or null when none was written. */ + snapshot: Snapshot | null; + /** Every entry appended after that checkpoint, in append order. */ + entries: Array>; +} + +export interface AppendOptions { + /** `store.sync()` before appending: the entry asserts state committed just before it. */ + afterCommittedState?: boolean; + /** `store.sync()` after appending: the entry must be on stable storage before the caller acts. */ + durable?: boolean; +} + +interface CheckpointPayload { + /** Id of the last entry the snapshot covers ("0" when it covers none). */ + through: string; + snapshot: Snapshot; +} + +/** Record ids are decimal strings from one store-wide counter. */ +function idValue(id: string): bigint { + if (!/^\d+$/.test(id)) throw new Error(`RecordJournal: unexpected chronicle record id "${id}"`); + return BigInt(id); +} + +function decode(store: JsStore, id: string): { value: T; sequence: number } { + const record = store.getRecord(id); + if (!record) throw new Error(`RecordJournal: record ${id} is listed but cannot be read`); + return { value: JSON.parse(Buffer.from(record.payload).toString('utf-8')) as T, sequence: record.sequence }; +} + +export class RecordJournal { + private readonly store: JsStore; + /** Record type of the entries. */ + readonly type: string; + /** Record type of the checkpoints. */ + readonly checkpointType: string; + /** Highest entry id this journal has loaded or appended (0 when none). */ + private lastEntryId = 0n; + private sinceCheckpoint = 0; + + constructor(store: JsStore, opts: { type: string; checkpointType?: string }) { + if (!opts.type) throw new Error('RecordJournal: a record type is required'); + this.store = store; + this.type = opts.type; + this.checkpointType = opts.checkpointType ?? `${opts.type}/checkpoint`; + if (this.checkpointType === this.type) { + throw new Error('RecordJournal: entries and checkpoints need distinct record types'); + } + } + + /** Entries appended since the last checkpoint (or since load, when there was none). */ + get entriesSinceCheckpoint(): number { + return this.sinceCheckpoint; + } + + /** + * The latest checkpoint and every entry after it, whichever branch each was + * appended on. Also positions the journal, so later checkpoints cover what + * was loaded. + */ + load(): JournalLoad { + let snapshot: Snapshot | null = null; + let through = 0n; + let latestCheckpoint: bigint | null = null; + let latestCheckpointId: string | null = null; + for (const id of this.store.getRecordIdsByType(this.checkpointType)) { + const value = idValue(id); + if (latestCheckpoint === null || value > latestCheckpoint) { + latestCheckpoint = value; + latestCheckpointId = id; + } + } + if (latestCheckpointId !== null) { + const payload = decode>(this.store, latestCheckpointId).value; + snapshot = payload.snapshot; + through = idValue(payload.through); + } + + const tail = this.store.getRecordIdsByType(this.type) + .map((id) => ({ id, value: idValue(id) })) + .filter(({ value }) => value > through) + .sort((a, b) => (a.value < b.value ? -1 : a.value > b.value ? 1 : 0)); + const entries = tail.map(({ id }) => { + const { value, sequence } = decode(this.store, id); + return { id, sequence, entry: value }; + }); + + this.lastEntryId = tail.length > 0 ? tail[tail.length - 1]!.value : through; + this.sinceCheckpoint = entries.length; + return { snapshot, entries }; + } + + /** Append one entry. The returned ref says where it landed, not that it's durable. */ + append(entry: Entry, opts: AppendOptions = {}): JournalRecordRef { + if (opts.afterCommittedState) this.store.sync(); + const record = this.store.appendJson(this.type, entry); + if (opts.durable) this.store.sync(); + const value = idValue(record.id); + if (value > this.lastEntryId) this.lastEntryId = value; + this.sinceCheckpoint++; + return { id: record.id, sequence: record.sequence }; + } + + /** + * Record the caller's reduced state as covering every entry this journal has + * loaded or appended. Like an entry, it's durable only with `durable`. + */ + checkpoint(snapshot: Snapshot, opts: { durable?: boolean } = {}): JournalRecordRef { + const payload: CheckpointPayload = { through: this.lastEntryId.toString(), snapshot }; + const record = this.store.appendJson(this.checkpointType, payload); + if (opts.durable) this.store.sync(); + this.sinceCheckpoint = 0; + return { id: record.id, sequence: record.sequence }; + } +} diff --git a/test/fixtures/record-journal-kill.mjs b/test/fixtures/record-journal-kill.mjs new file mode 100644 index 000000000..e0c04d802 --- /dev/null +++ b/test/fixtures/record-journal-kill.mjs @@ -0,0 +1,18 @@ +// Child process for record-journal.test.ts: commit branch state, append a +// journal entry that asserts it, then die without sync or close. +// node record-journal-kill.mjs +// mode "asserting": append with afterCommittedState (the barrier under test). +import { JsStore } from '@animalabs/chronicle'; +import { RecordJournal } from '../../src/record-journal.js'; + +const [storePath, mode] = process.argv.slice(2); +const store = JsStore.openOrCreate({ path: storePath }); +try { + store.registerState({ id: 'journal-test/tree', strategy: 'tree' }); +} catch { + // registered by the parent's setup +} +const journal = new RecordJournal(store, { type: 'journal-test/entry' }); +store.treeSet('journal-test/tree', 'committed.txt', { blobHash: 'c'.repeat(64), size: 1, mode: 0o644 }); +journal.append({ asserts: 'committed.txt' }, { afterCommittedState: mode === 'asserting' }); +process.kill(process.pid, 'SIGKILL'); diff --git a/test/record-journal.test.ts b/test/record-journal.test.ts new file mode 100644 index 000000000..f74513559 --- /dev/null +++ b/test/record-journal.test.ts @@ -0,0 +1,161 @@ +/** + * RecordJournal — typed chronicle records that don't follow branch switches, + * replayed from the latest checkpoint, with the two durability barriers named. + */ + +import { describe, it } from 'node:test'; +import assert from 'node:assert'; +import { spawnSync } from 'node:child_process'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { JsStore } from '@animalabs/chronicle'; +import { RecordJournal } from '../src/record-journal.js'; + +const KILL_FIXTURE = join(import.meta.dirname, 'fixtures/record-journal-kill.mjs'); + +function withStore(fn: (store: JsStore, path: string) => T): T { + const dir = mkdtempSync(join(tmpdir(), 'record-journal-')); + const path = join(dir, 'store'); + const store = JsStore.openOrCreate({ path }); + try { + return fn(store, path); + } finally { + if (!store.isClosed()) store.close(); + rmSync(dir, { recursive: true, force: true }); + } +} + +type Entry = { n: string }; +const entriesOf = (journal: RecordJournal): string[] => journal.load().entries.map((e) => e.entry.n); + +describe('RecordJournal', () => { + it('an empty journal loads nothing', () => { + withStore((store) => { + const journal = new RecordJournal(store, { type: 'journal-test/entry' }); + assert.deepStrictEqual(journal.load(), { snapshot: null, entries: [] }); + assert.strictEqual(journal.entriesSinceCheckpoint, 0); + }); + }); + + it('append returns where the record landed; a fresh load replays entries in append order', () => { + withStore((store) => { + const journal = new RecordJournal(store, { type: 'journal-test/entry' }); + const a = journal.append({ n: 'a' }); + const b = journal.append({ n: 'b' }); + assert.match(a.id, /^\d+$/); + assert.ok(BigInt(b.id) > BigInt(a.id), 'ids increase in append order'); + assert.strictEqual(typeof a.sequence, 'number'); + const loaded = new RecordJournal(store, { type: 'journal-test/entry' }).load(); + assert.deepStrictEqual(loaded.entries, [ + { id: a.id, sequence: a.sequence, entry: { n: 'a' } }, + { id: b.id, sequence: b.sequence, entry: { n: 'b' } }, + ]); + }); + }); + + it('entries from every branch load from any branch, in global order, through deleteBranch and reopen', () => { + withStore((store, path) => { + const journal = new RecordJournal(store, { type: 'journal-test/entry' }); + journal.append({ n: 'main-0' }); + const fork = store.currentSequence(); + journal.append({ n: 'main-1' }); + const main = store.currentBranch().name; + store.createBranchAt('rollback', main, fork); + store.switchBranch('rollback'); + journal.append({ n: 'rollback-2' }); + assert.deepStrictEqual(entriesOf(journal), ['main-0', 'main-1', 'rollback-2'], + 'the rolled-back branch still sees what main appended after the fork'); + store.switchBranch(main); + journal.append({ n: 'main-3' }); + store.deleteBranch('rollback'); + assert.deepStrictEqual(entriesOf(journal), ['main-0', 'main-1', 'rollback-2', 'main-3']); + store.close(); + const reopened = JsStore.openOrCreate({ path }); + try { + assert.deepStrictEqual(entriesOf(new RecordJournal(reopened, { type: 'journal-test/entry' })), + ['main-0', 'main-1', 'rollback-2', 'main-3']); + } finally { + reopened.close(); + } + }); + }); + + it('load returns the latest checkpoint and only the entries after it; other records do not disturb it', () => { + withStore((store) => { + store.registerState({ id: 'journal-test/tree', strategy: 'tree' }); + const journal = new RecordJournal(store, { type: 'journal-test/entry' }); + const other = new RecordJournal(store, { type: 'journal-test/other' }); + journal.append({ n: 'a' }); + other.append({ n: 'x' }); + journal.append({ n: 'b' }); + journal.checkpoint({ seen: ['a', 'b'] }); + assert.strictEqual(journal.entriesSinceCheckpoint, 0); + store.treeSet('journal-test/tree', 'f.txt', { blobHash: 'f'.repeat(64), size: 1, mode: 0o644 }); + journal.append({ n: 'c' }); + other.append({ n: 'y' }); + assert.strictEqual(journal.entriesSinceCheckpoint, 1); + + const reader = new RecordJournal(store, { type: 'journal-test/entry' }); + const first = reader.load(); + assert.deepStrictEqual(first.snapshot, { seen: ['a', 'b'] }); + assert.deepStrictEqual(first.entries.map((e) => e.entry.n), ['c']); + assert.strictEqual(reader.entriesSinceCheckpoint, 1); + + // A checkpoint after load covers what was loaded; the newest one wins. + reader.checkpoint({ seen: ['a', 'b', 'c'] }); + const second = new RecordJournal(store, { type: 'journal-test/entry' }).load(); + assert.deepStrictEqual(second, { snapshot: { seen: ['a', 'b', 'c'] }, entries: [] }); + assert.deepStrictEqual(entriesOf(new RecordJournal(store, { type: 'journal-test/other' })), ['x', 'y']); + }); + }); + + it('names its two barriers: afterCommittedState syncs before the append, durable after it, neither by default', () => { + withStore((store) => { + const calls: string[] = []; + const recording = { + getRecordIdsByType: (type: string) => store.getRecordIdsByType(type), + getRecord: (id: string) => store.getRecord(id), + appendJson: (type: string, data: unknown) => { calls.push('append'); return store.appendJson(type, data); }, + sync: () => { calls.push('sync'); store.sync(); }, + } as unknown as JsStore; + const journal = new RecordJournal(recording, { type: 'journal-test/entry' }); + journal.append({ n: 'plain' }); + assert.deepStrictEqual(calls.splice(0), ['append'], 'a plain append does not sync'); + journal.append({ n: 'asserting' }, { afterCommittedState: true }); + assert.deepStrictEqual(calls.splice(0), ['sync', 'append']); + journal.append({ n: 'durable' }, { durable: true }); + assert.deepStrictEqual(calls.splice(0), ['append', 'sync']); + journal.append({ n: 'both' }, { afterCommittedState: true, durable: true }); + assert.deepStrictEqual(calls.splice(0), ['sync', 'append', 'sync']); + journal.checkpoint({ n: 1 }, { durable: true }); + assert.deepStrictEqual(calls.splice(0), ['append', 'sync']); + }); + }); + + it('after a hard kill, an afterCommittedState entry never survives without the state it asserts', () => { + withStore((store, path) => { + store.registerState({ id: 'journal-test/tree', strategy: 'tree' }); + store.sync(); + store.close(); + const child = spawnSync(process.execPath, [KILL_FIXTURE, path, 'asserting'], { encoding: 'utf-8' }); + assert.strictEqual(child.signal, 'SIGKILL', `the child died by SIGKILL (stderr: ${child.stderr})`); + const reopened = JsStore.openOrCreate({ path }); + try { + try { reopened.registerState({ id: 'journal-test/tree', strategy: 'tree' }); } catch { /* exists */ } + const entries = new RecordJournal<{ asserts: string }, unknown>(reopened, { type: 'journal-test/entry' }).load().entries; + assert.strictEqual(entries.length, 1, 'the appended entry survived the kill'); + assert.notStrictEqual(reopened.treeGet('journal-test/tree', 'committed.txt'), null, + 'the state it asserts survived with it'); + } finally { + reopened.close(); + } + }); + }); + + it('refuses a checkpoint type equal to the entry type', () => { + withStore((store) => { + assert.throws(() => new RecordJournal(store, { type: 't', checkpointType: 't' }), /distinct record types/); + }); + }); +}); From 16a639266726d7296b5a1e6e93eac6691791d532 Mon Sep 17 00:00:00 2001 From: Ada-1017 Date: Wed, 7 Oct 2026 05:17:23 -0400 Subject: [PATCH 02/73] fix(surgery): return once marks are scheduled, not after Discord accepts them Field incident (brosefo's household, 2026-10-07): a web-UI rollback removed 918 Discord messages from a resident's context. rollbackToMessage awaited syncDiscordAwarenessMarkers() inside reserveStoreForSurgery, so the serial reaction drain (~1/s at Discord's pace) held the store-wide reservation and the operator's surgery-result for its whole length; the web client gave up after 60 s while the branch had already switched. A composed fixture against the shipped 0.21.0 source reproduced it: the rollback promise resolved only after the last add_reaction, and channel traffic sent during the drain was answered only after it. Now both live surgeries split body from marks: - The reservation covers fork/switch (or redaction), durable batch activation, and any source restoration on error. After the body change lands, activateSurgeryMarkers() records the batch active and returns a SurgeryMarkerReceipt: none | queued (count, batchId) | not-scheduled. - deliverDiscordAwarenessInBackground() calls syncDiscordAwarenessMarkers() without awaiting it, before release(). The MCPL data-plane gate is installed synchronously inside it, so MCPL-borne wakes stay behind the gate exactly as before; only the response and the store reservation stop waiting on remote reactions. Failures are logged and raised as an ops alert ('discord-awareness-delivery'); the gate's own failure handling (connection recycle) is unchanged. - A ledger failure in activate() after the body change no longer fails an applied rollback (previously a raw throw after the switch, logged as an error) or undoes a fully redacted suppression (previously activate() sat in the redaction try block, so it retired the batch and restored the source). The batch is retired so the reconciliation that delivery runs next cannot promote it behind a "not scheduled" receipt; if retiring fails too, the receipt names the batch left prepared. - `markers` is returned by both surgeries, by message-granular host/command undo, recorded in operator-actions.jsonl, and printed on the [operator] stderr line. It is a scheduling statement, never a delivery claim. Agreed seam: Kit-1008 and Agnes-1009 (shelf-380 body contract, room-203 #41462/#41558) stack their safe-boundary lease on this split; room-225's marks contract (opt-in scope, one-shot, background delivery that does not gate MCPL traffic) is a separate, pending change. Tests: test/surgery-marks-receipt.test.ts (6) with a fake Discord MCPL server that withholds reaction replies. Red on origin/main 64c480b: 0/6 (rollback still pending after 3 s with reactions held; activation failures throw; suppression restored; no markers field). Green: 6/6. Related files (operator-surgery, discord-awareness-outbox, mcpl-awareness-barrier, offline-recovery-branch, host-quiesce, puppet-turn-reservation): 87/87. Co-Authored-By: Claude Opus 5.5 --- .../fix-surgery-marks-receipt.fixed.md | 13 + src/framework.ts | 133 +++++++- src/index.ts | 7 +- src/operator-log.ts | 21 ++ test/fixtures/surgery-marks-mcpl-server.mjs | 113 +++++++ test/surgery-marks-receipt.test.ts | 290 ++++++++++++++++++ 6 files changed, 568 insertions(+), 9 deletions(-) create mode 100644 changelog.d/fix-surgery-marks-receipt.fixed.md create mode 100644 test/fixtures/surgery-marks-mcpl-server.mjs create mode 100644 test/surgery-marks-receipt.test.ts diff --git a/changelog.d/fix-surgery-marks-receipt.fixed.md b/changelog.d/fix-surgery-marks-receipt.fixed.md new file mode 100644 index 000000000..ae2c47d1e --- /dev/null +++ b/changelog.d/fix-surgery-marks-receipt.fixed.md @@ -0,0 +1,13 @@ +- Live `rollbackToMessage` and `suppressMessages` (and message-granular + `host/command` undo, which rides on rollback) return once the body change + has landed and the Discord awareness batch is durably active, instead of + after every 💤 reaction has been delivered. Previously the store stayed + reserved, and the operator got no answer, for the whole serial drain: about + 15 minutes for a rollback that removed 918 channel messages. Delivery now + continues in the background, behind the same MCPL data-plane gate as + before. The result's new `markers` field (`none` | `queued` | + `not-scheduled`) says whether marks were scheduled, never that Discord + accepted them. If the ledger write after the switch fails, the result + reports `not-scheduled` and retires the batch rather than failing a + rollback that already applied, and a suppression whose redactions all + committed is no longer undone by that failure. diff --git a/src/framework.ts b/src/framework.ts index c527c488f..3f100dcce 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -406,12 +406,14 @@ import { DiscordAwarenessOutbox, defaultDiscordAwarenessOutboxPath, extractDiscordAwarenessRefs, + type DiscordAwarenessBatch, } from './recovery/discord-awareness-outbox.js'; import { OperatorActionError, OperatorLog, capIds, defaultOperatorLogPath, + type SurgeryMarkerReceipt, type OperatorLogEntry, type OperatorLogInput, type OperatorRequester, @@ -717,6 +719,14 @@ function bodyGroupRun( }; } +function describeMarkers(m: SurgeryMarkerReceipt): string { + if (m.status === 'queued') return `queued:${m.queued}`; + if (m.status === 'not-scheduled') { + return m.batchId ? `not-scheduled(batch ${m.batchId} left prepared)` : 'not-scheduled(batch retired)'; + } + return 'none'; +} + function describeRequester(r: OperatorRequester | undefined): string { if (!r) return 'unknown'; return `${r.name ?? r.id ?? 'unknown'} (${r.via})`; @@ -5068,6 +5078,9 @@ export class AgentFramework { /** Discord addresses removed by message-granular undo. The durable outbox * owns eventual delivery; this is also returned for immediate surfaces. */ removedRefs?: Array<{ serverId: string; channelId: string; messageId: string }>; + /** For message-granular undo: whether awareness marks were scheduled + * (not whether Discord has accepted them). */ + markers?: SurgeryMarkerReceipt; hidden?: number; /** For `hide`: the Discord (channelId, messageId) of each removed message * that carried one — so the surface can mark them with a reaction. */ @@ -5311,6 +5324,7 @@ export class AgentFramework { ok: true, messagesRemoved: r.messagesRemoved, removedRefs: r.removedRefs, + markers: r.markers, lastVisible: r.lastVisible, }; } catch (error) { @@ -5942,6 +5956,11 @@ export class AgentFramework { * Discord messages that left the live context get awareness markers via the * durable outbox, exactly as message-granular `undo` does. * + * Returns once the switch has landed and the marker batch is durably active + * (`markers`), without waiting for Discord to accept the reactions: delivery + * runs after the store reservation is released, behind the MCPL data-plane + * gate, and its per-message outcomes stay in the outbox ledger. + * * Throws `OperatorActionError` (`agent-busy`, `unknown-message`, …) — the * agent must be idle; nothing is queued. */ @@ -5964,6 +5983,8 @@ export class AgentFramework { * the group's last shard so the sharded message stays whole. */ tailMessageId: string; removedRefs: Array<{ serverId: string; channelId: string; messageId: string }>; + /** Whether awareness marks were scheduled; never a delivery claim. */ + markers: SurgeryMarkerReceipt; lastVisible: { participant?: string; role?: string; preview?: string } | null; }> { const logBase: OperatorLogInput = { @@ -6001,9 +6022,11 @@ export class AgentFramework { } // Gate + reserve the whole store (see reserveStoreForSurgery); held - // until the switch has landed or been rolled back. + // until the switch has landed and its marker batch is durably active, + // or the switch has been rolled back. Remote delivery is not waited on. const release = this.reserveStoreForSurgery('roll back', agentName); let branchName: string; + let markers: SurgeryMarkerReceipt; try { // Prepare the external side effect before switching Chronicle. If the // process dies after the switch but before activate(), startup @@ -6034,8 +6057,13 @@ export class AgentFramework { { cause: error }, ); } - if (markerBatch) this.discordAwarenessOutbox!.activate(markerBatch.id); - await this.syncDiscordAwarenessMarkers(); + // The body change has landed. Nothing after this point turns the + // rollback into a failure: marks are an outbox outcome, reported + // apart from the body (`markers`). + markers = this.activateSurgeryMarkers(markerBatch, 'rollback', agentName); + // Installs the MCPL data-plane gate synchronously, before the store is + // released; the drain itself runs after this call returns. + this.deliverDiscordAwarenessInBackground('rollback', agentName); this.materializeConfigMountAfterBranchSwitch(); } finally { release(); @@ -6044,7 +6072,8 @@ export class AgentFramework { console.error( `[operator] rollback agent=${agentName} to=${tailMessageId}` + `${tailMessageId !== opts.messageId ? ` (requested ${opts.messageId}, snapped to end of body group)` : ''}` + - ` removed=${messagesRemoved} branch=${branchName} by=${describeRequester(opts.requester)}`, + ` removed=${messagesRemoved} branch=${branchName} marks=${describeMarkers(markers)}` + + ` by=${describeRequester(opts.requester)}`, ); this.recordOperatorAction({ ...logBase, @@ -6054,6 +6083,7 @@ export class AgentFramework { tailMessageId, messagesRemoved, discordRefs: removedRefs.length, + markers, }, }); return { @@ -6063,6 +6093,7 @@ export class AgentFramework { messagesRemoved, tailMessageId, removedRefs, + markers, lastVisible: await this.lastVisiblePreview(agentName), }; } catch (error) { @@ -6078,6 +6109,8 @@ export class AgentFramework { * removed together. Discord originals get awareness markers via the outbox; * the batch is `explicit`-activated only after every removal succeeded, and * a crash mid-way is finished at next boot (resumePreparedDiscordSuppressions). + * Like rollbackToMessage, it returns once the body change and the batch's + * activation are durable (`markers`); delivery runs in the background. * * Not retroactive over derived state: a message already folded into an * autobiographical summary stays in that summary — roll back to before it @@ -6099,6 +6132,8 @@ export class AgentFramework { messagesRemoved: number; removedIds: string[]; removedRefs: Array<{ serverId: string; channelId: string; messageId: string }>; + /** Whether awareness marks were scheduled; never a delivery claim. */ + markers: SurgeryMarkerReceipt; lastVisible: { participant?: string; role?: string; preview?: string } | null; }> { const requestedIds = [...new Set(opts.messageIds.map(String))]; @@ -6151,9 +6186,11 @@ export class AgentFramework { throw new OperatorActionError('invalid', 'Suppression branch name must differ from the active branch'); } // Gate + reserve the whole store (see reserveStoreForSurgery); held - // until the fork is fully redacted or the source is restored. + // until the fork is fully redacted and its marker batch durably + // active, or the source is restored. Remote delivery is not waited on. const release = this.reserveStoreForSurgery('suppress', agentName); let createdBranch: string; + let markers: SurgeryMarkerReceipt; try { const markerBatch = this.discordAwarenessOutbox?.prepare({ agentName, @@ -6194,7 +6231,6 @@ export class AgentFramework { if (iv.group || iv.fromId !== iv.toId) cm.removeMessages(iv.fromId as MessageId, iv.toId as MessageId); else cm.removeMessage(iv.fromId as MessageId); } - if (markerBatch) this.discordAwarenessOutbox!.activate(markerBatch.id); } catch (error) { // A partially suppressed branch is not safe to serve from. Keep it // for diagnosis, put the agent back on the untouched source — and @@ -6207,7 +6243,16 @@ export class AgentFramework { { cause: error }, ); } - await this.syncDiscordAwarenessMarkers(); + // Every redaction committed: the body change has landed and the fork + // is safe to serve from. Activating the marker batch is bookkeeping + // about a world-side effect; if it fails, the suppression still + // stands and the receipt says the marks were not scheduled. (No + // interval needs resuming any more, so retiring the batch there + // loses nothing.) + markers = this.activateSurgeryMarkers(markerBatch, 'suppress', agentName); + // Installs the MCPL data-plane gate synchronously, before the store is + // released; the drain itself runs after this call returns. + this.deliverDiscordAwarenessInBackground('suppress', agentName); this.materializeConfigMountAfterBranchSwitch(); } finally { release(); @@ -6215,7 +6260,7 @@ export class AgentFramework { console.error( `[operator] suppress agent=${agentName} removed=${messagesRemoved} branch=${createdBranch} ` + - `by=${describeRequester(opts.requester)}`, + `marks=${describeMarkers(markers)} by=${describeRequester(opts.requester)}`, ); const loggedIds = capIds(removedIds); this.recordOperatorAction({ @@ -6227,6 +6272,7 @@ export class AgentFramework { removedIds: loggedIds.ids, ...(loggedIds.truncated ? { removedIdsTruncated: true } : {}), discordRefs: removedRefs.length, + markers, }, }); return { @@ -6236,6 +6282,7 @@ export class AgentFramework { messagesRemoved, removedIds, removedRefs, + markers, lastVisible: await this.lastVisiblePreview(agentName), }; } catch (error) { @@ -6244,6 +6291,76 @@ export class AgentFramework { } } + /** + * Record a surgery's prepared marker batch as active once its body change + * has landed, and say what happened. Never throws: the body is already + * applied, so a ledger failure here is reported as `not-scheduled` rather + * than as a failed surgery an operator might retry. A batch that could not + * be activated is retired, so that the reconciliation which delivery runs + * next cannot promote it behind a receipt saying it was not scheduled; only + * when retiring fails too does it stay prepared (named by `batchId`). A + * batch with no refs (a suppression journal with nothing addressable) is + * still activated, which closes its resume journal; it schedules no marks. + */ + private activateSurgeryMarkers( + batch: DiscordAwarenessBatch | null, + verb: 'rollback' | 'suppress', + agentName: string, + ): SurgeryMarkerReceipt { + if (!batch) return { status: 'none', queued: 0 }; + try { + this.discordAwarenessOutbox!.activate(batch.id); + } catch (error) { + const detail = error instanceof Error ? error.message : String(error); + let retired = false; + let retireDetail = ''; + try { + retired = this.discordAwarenessOutbox!.discard(batch.id); + } catch (discardError) { + retireDetail = discardError instanceof Error ? discardError.message : String(discardError); + } + console.error( + `[discord-awareness] ${verb} agent=${agentName}: body applied, but marker batch ${batch.id} ` + + `could not be activated (${detail}); ` + + (retired ? 'batch retired' : `batch left prepared${retireDetail ? ` (retire failed: ${retireDetail})` : ''}`), + ); + if (batch.refs.length === 0) return { status: 'none', queued: 0 }; + this.opsAlert( + 'discord-awareness-not-scheduled', + agentName, + `${verb} applied, but its ${batch.refs.length} awareness mark(s) were not scheduled: ${detail}`, + { data: { batchId: batch.id, retired } }, + ); + return retired + ? { status: 'not-scheduled', queued: 0, error: detail } + : { status: 'not-scheduled', queued: 0, error: detail, batchId: batch.id }; + } + if (batch.refs.length === 0) return { status: 'none', queued: 0 }; + return { status: 'queued', queued: batch.refs.length, batchId: batch.id }; + } + + /** + * Hand awareness delivery to the background, without holding the caller. + * `syncDiscordAwarenessMarkers` installs the MCPL data-plane gate + * synchronously, before its first await, so calling this while a surgery + * still holds the store keeps MCPL-borne wakes behind the gate exactly as + * before. Only the surgery's response and its store reservation stop + * waiting on remote reactions. A failure is accounted by the gate itself + * (connections recycled); here it is logged and raised as an ops alert. + */ + private deliverDiscordAwarenessInBackground(verb: 'rollback' | 'suppress', agentName: string): void { + if (!this.discordAwarenessOutbox) return; + this.syncDiscordAwarenessMarkers().catch((error) => { + const detail = error instanceof Error ? error.message : String(error); + console.error(`[discord-awareness] delivery after ${verb} agent=${agentName} failed: ${detail}`); + this.opsAlert( + 'discord-awareness-delivery', + agentName, + `Awareness mark delivery after ${verb} failed: ${detail}`, + ); + }); + } + /** Re-materialize config files from the (new) active branch — fire and * forget; the gate picks changes up via mtime. Mirrors undoLastTurn. */ private materializeConfigMountAfterBranchSwitch(): void { diff --git a/src/index.ts b/src/index.ts index bd07d3547..939a4ac56 100644 --- a/src/index.ts +++ b/src/index.ts @@ -142,7 +142,12 @@ export type { } from './recovery/discord-awareness-outbox.js'; // Live operator surgery (rollback / suppress) and its durable action log export { OperatorLog, OperatorActionError, defaultOperatorLogPath } from './operator-log.js'; -export type { OperatorLogEntry, OperatorLogInput, OperatorRequester } from './operator-log.js'; +export type { + OperatorLogEntry, + OperatorLogInput, + OperatorRequester, + SurgeryMarkerReceipt, +} from './operator-log.js'; export { createOfflineRecoveryBranch } from './recovery/offline-branch.js'; export type { OfflineRecoveryBranchOptions, diff --git a/src/operator-log.ts b/src/operator-log.ts index 710e6b505..48de210f0 100644 --- a/src/operator-log.ts +++ b/src/operator-log.ts @@ -118,6 +118,27 @@ export function capIds(ids: string[]): { ids: string[]; count: number; truncated : { ids, count: ids.length }; } +/** + * What a live surgery did about Discord awareness marks, reported apart from + * the body change itself. A surgery that returns has applied its body change; + * this says only whether marks were scheduled: + * + * - `none` — the change removed no addressable Discord message. + * - `queued` — the outbox batch is durably active and its delivery has been + * handed to the background drain. It is not a delivery claim: per-message + * outcomes stay in the outbox ledger. + * - `not-scheduled` — the body change landed, but recording the batch as + * active failed, so no mark will be delivered for it: the batch is retired + * from the ledger. Only if retiring it failed as well is `batchId` present: + * that batch stays prepared, and a later reconciliation (or, for a + * suppression, the next startup) that can write the ledger may still + * promote it. + */ +export type SurgeryMarkerReceipt = + | { status: 'none'; queued: 0 } + | { status: 'queued'; queued: number; batchId: string } + | { status: 'not-scheduled'; queued: 0; error: string; batchId?: string }; + /** Thrown by live surgery methods when a request cannot be honored. `code` * lets surfaces distinguish "agent busy — quiesce first" from bad input. */ export class OperatorActionError extends Error { diff --git a/test/fixtures/surgery-marks-mcpl-server.mjs b/test/fixtures/surgery-marks-mcpl-server.mjs new file mode 100644 index 000000000..9c5314901 --- /dev/null +++ b/test/fixtures/surgery-marks-mcpl-server.mjs @@ -0,0 +1,113 @@ +// Fake Discord MCPL server for test/surgery-marks-receipt.test.ts. +// +// Every tools/call is appended to CALLS_PATH. While HOLD_PATH is set and that +// file does not exist, replies are withheld, standing in for Discord pacing a +// long reaction drain. When PROBE_PATH appears, the server sends one +// inference-bearing channels/incoming request and appends to EVENTS_PATH when +// the host answers it, so a test can see whether the MCPL data plane was held. +import { appendFileSync, existsSync } from 'node:fs'; + +const callsPath = process.env.CALLS_PATH; +const eventsPath = process.env.EVENTS_PATH; +const holdPath = process.env.HOLD_PATH; +const probePath = process.env.PROBE_PATH; +const channelId = process.env.CHANNEL_ID ?? 'discord:g1:c1'; + +const send = (message) => process.stdout.write(JSON.stringify(message) + '\n'); +const reply = (id, result) => send({ jsonrpc: '2.0', id, result }); +const request = (id, method, params) => send({ jsonrpc: '2.0', id, method, params }); +const event = (name, extra = {}) => { + if (eventsPath) appendFileSync(eventsPath, JSON.stringify({ event: name, at: Date.now(), ...extra }) + '\n'); +}; + +const held = []; +let probed = false; +let buf = ''; + +function answer(message) { + event('reaction-answered', { messageId: message.params?.arguments?.messageId }); + reply(message.id, { content: [{ type: 'text', text: 'Reaction applied' }] }); +} + +function handle(message) { + if (message.method === 'initialize') { + reply(message.id, { + capabilities: { + experimental: { + mcpl: { + version: '0.5', + pushEvents: true, + channels: { register: true, incoming: true }, + featureSets: { chat: { description: 'chat', uses: ['pushEvents'] } }, + }, + }, + }, + }); + return; + } + if (message.method === 'featureSets/update') { + if (message.id !== undefined && message.id !== null) reply(message.id, { accepted: true }); + return; + } + if (message.method === 'notifications/initialized') { + request(100, 'channels/register', { + channels: [{ id: channelId, type: 'discord', label: 'busy', direction: 'bidirectional' }], + }); + return; + } + if (message.method === 'tools/list') { + reply(message.id, { + tools: [ + { name: 'add_reaction', description: 'add', inputSchema: { type: 'object' } }, + { name: 'remove_reaction', description: 'remove', inputSchema: { type: 'object' } }, + ], + }); + return; + } + if (message.method === 'tools/call') { + if (callsPath) { + appendFileSync(callsPath, JSON.stringify({ at: Date.now(), name: message.params?.name, args: message.params?.arguments }) + '\n'); + } + if (holdPath && !existsSync(holdPath)) held.push(message); + else answer(message); + return; + } + if (message.id === 900) { + event('probe-answered', { error: message.error ?? null }); + } +} + +process.stdin.on('data', (chunk) => { + buf += chunk.toString('utf8'); + let newline; + while ((newline = buf.indexOf('\n')) >= 0) { + const line = buf.slice(0, newline); + buf = buf.slice(newline + 1); + if (!line.trim()) continue; + try { + handle(JSON.parse(line)); + } catch (error) { + event('server-error', { message: error instanceof Error ? error.message : String(error) }); + } + } +}); + +const timer = setInterval(() => { + if (held.length > 0 && holdPath && existsSync(holdPath)) { + for (const message of held.splice(0)) answer(message); + } + if (!probed && probePath && existsSync(probePath)) { + probed = true; + event('probe-sent'); + request(900, 'channels/incoming', { + messages: [{ + channelId, + messageId: 'probe-during-delivery', + author: { id: 'bystander', name: 'Bystander' }, + timestamp: new Date().toISOString(), + content: [{ type: 'text', text: 'ordinary traffic while marks are delivered' }], + }], + }); + } +}, 5); +timer.unref(); diff --git a/test/surgery-marks-receipt.test.ts b/test/surgery-marks-receipt.test.ts new file mode 100644 index 000000000..13e8793dd --- /dev/null +++ b/test/surgery-marks-receipt.test.ts @@ -0,0 +1,290 @@ +/** + * Live surgery reports its body change apart from the Discord awareness marks + * it schedules, and does not wait on Discord to accept them. + * + * Field incident (2026-10-07): a web-UI rollback removed 918 Discord messages + * from a resident's context. rollbackToMessage awaited the serial reaction + * drain inside its store reservation, so the operator's surgery-result never + * arrived within the client's 60 s, the store stayed reserved, and every MCPL + * data plane stayed paused for the whole drain. These tests hold the fake + * server's reaction replies to model that drain. + * + * The receipt is a scheduling statement (`markers`), never a delivery claim, + * and a ledger failure after the body change landed is reported as + * `not-scheduled` instead of a failed surgery an operator might retry. + */ +import { describe, it, beforeEach, afterEach } from 'node:test'; +import assert from 'node:assert/strict'; +import { fileURLToPath } from 'node:url'; +import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { AgentFramework, DiscordAwarenessOutbox } from '../src/index.js'; +import type { OperatorLogEntry } from '../src/index.js'; +import { MockMembrane } from './helpers/mock-membrane.js'; + +const FIXTURE = fileURLToPath(new URL('./fixtures/surgery-marks-mcpl-server.mjs', import.meta.url)); + +function jsonl>(path: string): T[] { + if (!existsSync(path)) return []; + return readFileSync(path, 'utf8').split('\n').filter(Boolean).map((line) => JSON.parse(line) as T); +} + +async function waitFor(description: string, predicate: () => boolean, timeoutMs = 5_000): Promise { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + if (predicate()) return; + await new Promise((resolve) => setTimeout(resolve, 10)); + } + assert.fail(`timed out waiting for ${description}`); +} + +/** Resolve with the promise's value, or fail if it is still pending after ms. */ +async function settlesWithin(promise: Promise, ms: number, what: string): Promise { + let timer: ReturnType | undefined; + try { + return await Promise.race([ + promise, + new Promise((_, reject) => { + timer = setTimeout(() => reject(new Error(`${what} still pending after ${ms} ms`)), ms); + }), + ]); + } finally { + if (timer) clearTimeout(timer); + } +} + +describe('surgery marker receipt', () => { + let dir: string; + let storePath: string; + let callsPath: string; + let eventsPath: string; + let holdPath: string; + let probePath: string; + let framework: AgentFramework; + + const cm = () => framework.getAgent('resident')!.getContextManager(); + const outboxPath = () => join(storePath, 'recovery', 'discord-awareness-outbox.json'); + const ledger = () => new DiscordAwarenessOutbox(outboxPath()).batches(); + + async function start(withDiscord: boolean): Promise { + framework = await AgentFramework.create({ + storePath, + membrane: new MockMembrane().asMembrane(), + agents: [{ name: 'resident', model: 'test-model', systemPrompt: 'You are a resident.' }], + modules: [], + maintenanceIntervalMs: 0, + ...(withDiscord ? { + mcplServers: [{ + id: 'discord', + command: process.execPath, + args: [FIXTURE], + env: { + CALLS_PATH: callsPath, + EVENTS_PATH: eventsPath, + HOLD_PATH: holdPath, + PROBE_PATH: probePath, + }, + enabledFeatureSets: ['chat'], + }], + } : {}), + }); + } + + /** A kept message, the rollback target, then `n` addressable messages. */ + function seed(n: number): { tail: string; removed: string[] } { + cm().addMessage('Operator', [{ type: 'text', text: 'kept' }], { + serverId: 'discord', channelId: 'discord:g1:dm', messageId: 'kept', + }); + const tail = String(cm().addMessage('Operator', [{ type: 'text', text: 'last good' }], { + serverId: 'discord', channelId: 'discord:g1:dm', messageId: 'tail', + })); + const removed: string[] = []; + for (let i = 0; i < n; i++) { + removed.push(String(cm().addMessage(`Member${i}`, [{ type: 'text', text: `ambient ${i}` }], { + serverId: 'discord', channelId: 'discord:g1:c1', messageId: `amb-${i}`, + }))); + } + return { tail, removed }; + } + + beforeEach(() => { + dir = mkdtempSync(join(tmpdir(), 'surgery-marks-')); + storePath = join(dir, 'store'); + callsPath = join(dir, 'calls.jsonl'); + eventsPath = join(dir, 'events.jsonl'); + holdPath = join(dir, 'release-reactions'); + probePath = join(dir, 'send-probe'); + }); + + afterEach(async () => { + writeFileSync(holdPath, '1'); + await framework?.stop(); + rmSync(dir, { recursive: true, force: true }); + }); + + it('rollback returns once the batch is active, before Discord has accepted any reaction', async () => { + await start(true); + const { tail } = seed(3); + + const result = await settlesWithin( + framework.rollbackToMessage('resident', { messageId: tail }), + 3_000, + 'rollbackToMessage with reactions held', + ); + + assert.equal(result.messagesRemoved, 3); + assert.equal(result.markers.status, 'queued'); + assert.equal(result.markers.queued, 3); + const [batch] = ledger(); + assert.equal(batch.status, 'active', 'the batch is durably active before the call returns'); + assert.equal(result.markers.status === 'queued' && result.markers.batchId, batch.id); + assert.equal( + jsonl(eventsPath).filter((e) => e.event === 'reaction-answered').length, + 0, + 'no reaction had been accepted when the rollback returned', + ); + + // Delivery continues in the background and completes once Discord answers. + writeFileSync(holdPath, '1'); + await waitFor('all three reactions answered', () => + jsonl(eventsPath).filter((e) => e.event === 'reaction-answered').length === 3); + await framework.syncDiscordAwarenessMarkers(); + const entries = ledger()[0].refs; + assert.deepEqual(entries.map((ref) => ref.deliveryStatus), ['applied', 'applied', 'applied']); + assert.deepEqual( + jsonl<{ name: string; args: { emoji: string } }>(callsPath).map((call) => `${call.name}:${call.args.emoji}`), + ['add_reaction:💤', 'add_reaction:💤', 'add_reaction:💤'], + ); + + const logged = new Map( + framework.getOperatorLog({ limit: 10 }).map((entry) => [entry.kind, entry]), + ); + assert.deepEqual(logged.get('rollback')?.result?.markers, result.markers); + }); + + it('releases the store while delivery runs, but keeps MCPL traffic behind the delivery gate', async () => { + await start(true); + const { removed } = seed(4); + + await settlesWithin( + framework.rollbackToMessage('resident', { messageId: removed[1] }), + 3_000, + 'rollbackToMessage with reactions held', + ); + await waitFor('the first reaction call to reach the server', () => jsonl(callsPath).length >= 1); + + // The store reservation is released: a second surgery is admitted while + // the first one's reactions are still unanswered. + const second = await settlesWithin( + framework.suppressMessages('resident', { messageIds: [removed[0]] }), + 3_000, + 'a second surgery during delivery', + ); + assert.equal(second.messagesRemoved, 1); + assert.equal(second.markers.status, 'queued'); + + // The gate was installed before release: ordinary channel traffic sent + // during delivery is not answered until the reactions are. + writeFileSync(probePath, '1'); + await waitFor('the probe to be sent', () => jsonl(eventsPath).some((e) => e.event === 'probe-sent')); + await new Promise((resolve) => setTimeout(resolve, 200)); + assert.equal( + jsonl(eventsPath).some((e) => e.event === 'probe-answered'), + false, + 'channel traffic waits behind the delivery gate', + ); + + writeFileSync(holdPath, '1'); + await waitFor('the probe to be answered after delivery', () => + jsonl(eventsPath).some((e) => e.event === 'probe-answered')); + const events = jsonl<{ event: string; at: number }>(eventsPath); + const lastReaction = Math.max(...events.filter((e) => e.event === 'reaction-answered').map((e) => e.at)); + const probe = events.find((e) => e.event === 'probe-answered')!; + assert.ok(probe.at >= lastReaction, 'the probe is answered only after the held reactions'); + }); + + it('a rollback whose marker bookkeeping fails after the switch reports the body as applied', async () => { + await start(false); + const { tail } = seed(2); + const original = DiscordAwarenessOutbox.prototype.activate; + DiscordAwarenessOutbox.prototype.activate = function () { + throw new Error('injected ledger failure'); + }; + let result: Awaited>; + try { + result = await framework.rollbackToMessage('resident', { messageId: tail }); + } finally { + DiscordAwarenessOutbox.prototype.activate = original; + } + + assert.equal(cm().currentBranch().name, result.targetBranch, 'the rollback stands'); + assert.equal(result.messagesRemoved, 2); + assert.equal(result.markers.status, 'not-scheduled'); + assert.match(result.markers.status === 'not-scheduled' ? result.markers.error : '', /injected ledger failure/); + assert.equal(result.markers.status === 'not-scheduled' && result.markers.batchId, undefined); + // The unactivated batch is retired, so the reconciliation that delivery + // runs cannot promote it behind a receipt that said "not scheduled". + await framework.syncDiscordAwarenessMarkers(); + assert.deepEqual(ledger(), []); + + const entry = framework.getOperatorLog({ limit: 10 }).find((e) => e.kind === 'rollback'); + assert.equal(entry?.error, undefined, 'logged as an applied rollback, not a failure'); + assert.equal((entry?.result?.markers as { status?: string } | undefined)?.status, 'not-scheduled'); + }); + + it('names a batch it could neither activate nor retire, which stays prepared', async () => { + await start(false); + const { tail } = seed(2); + const activate = DiscordAwarenessOutbox.prototype.activate; + const discard = DiscordAwarenessOutbox.prototype.discard; + DiscordAwarenessOutbox.prototype.activate = function () { + throw new Error('injected ledger failure'); + }; + DiscordAwarenessOutbox.prototype.discard = function () { + throw new Error('injected retire failure'); + }; + let result: Awaited>; + try { + result = await framework.rollbackToMessage('resident', { messageId: tail }); + } finally { + DiscordAwarenessOutbox.prototype.activate = activate; + DiscordAwarenessOutbox.prototype.discard = discard; + } + assert.equal(cm().currentBranch().name, result.targetBranch); + assert.equal(result.markers.status, 'not-scheduled'); + const batches = ledger(); + assert.equal(batches.length, 1); + assert.equal(result.markers.status === 'not-scheduled' && result.markers.batchId, batches[0].id); + }); + + it('a suppression whose marker bookkeeping fails after the last redaction stands', async () => { + await start(false); + const { removed } = seed(3); + const original = DiscordAwarenessOutbox.prototype.activate; + DiscordAwarenessOutbox.prototype.activate = function () { + throw new Error('injected ledger failure'); + }; + let result: Awaited>; + try { + result = await framework.suppressMessages('resident', { messageIds: [removed[1]] }); + } finally { + DiscordAwarenessOutbox.prototype.activate = original; + } + + assert.equal(cm().currentBranch().name, result.targetBranch, 'the suppression is not undone'); + assert.deepEqual(result.removedIds, [removed[1]]); + assert.ok(!cm().getAllMessages().some((m) => String(m.id) === removed[1])); + assert.equal(result.markers.status, 'not-scheduled'); + assert.deepEqual(ledger(), [], 'every interval committed, so the retired journal loses nothing'); + }); + + it('a surgery that removes nothing addressable schedules no marks', async () => { + await start(false); + const keep = String(cm().addMessage('Operator', [{ type: 'text', text: 'kept' }], {})); + cm().addMessage('resident', [{ type: 'text', text: 'local only' }], {}); + const result = await framework.rollbackToMessage('resident', { messageId: keep }); + assert.deepEqual(result.markers, { status: 'none', queued: 0 }); + assert.equal(ledger().length, 0); + }); +}); From 2153e3d3627d3a40657d01f5e38be956d44689e5 Mon Sep 17 00:00:00 2001 From: Basil-1007 Date: Wed, 7 Oct 2026 05:21:31 -0400 Subject: [PATCH 03/73] fix(chronicle): a RecordJournal write that fails after reaching the store must be reconciled MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Hugo-1004's review of f7e14f6 found a replay-loss path: append(A, {durable:true}) hands A to the store, then sync() throws; the caller sees failure and never reduces A. A later successful append(B) advanced lastEntryId past A, and a checkpoint({seen:[B]}) then covered A without A ever entering the reducer — load() returned that snapshot with an empty tail. Now any failure after a record reaches the store (the appendJson itself, or the durable barrier after it, for entries and checkpoints alike) marks the journal unreconciled: append and checkpoint throw until load() has replayed the tail, the possibly-written record included, for the caller to rebuild from. A failure before the record reaches the store (the afterCommittedState sync) wrote nothing and leaves the journal usable. load() documents that the caller rebuilds its reduced state from exactly the snapshot and tail it returns. needsReconcile exposes the state. Regression test (fails on f7e14f6, which let append(B) and the covering checkpoint through): a durable barrier failing after A, writes refused until load(), load() replaying A, the journal usable again; and a pre-append barrier failure leaving it usable. Co-Authored-By: Claude Opus 5.5 --- src/record-journal.ts | 67 ++++++++++++++++++++++++++++++------- test/record-journal.test.ts | 42 +++++++++++++++++++++++ 2 files changed, 97 insertions(+), 12 deletions(-) diff --git a/src/record-journal.ts b/src/record-journal.ts index b3affd27e..0565e5422 100644 --- a/src/record-journal.ts +++ b/src/record-journal.ts @@ -31,6 +31,17 @@ * One writer per record type: a checkpoint covers every entry up to the last * one THIS journal loaded or appended, so entries another instance appended * to the same type in between would be covered without being reduced. + * + * An ambiguous write must be reconciled before the journal writes again. Once + * `append` or `checkpoint` has handed a record to the store, any failure from + * there on (the append itself, or a barrier after it) leaves it unknown + * whether that record exists, and the caller didn't reduce it. A later + * checkpoint would then cover it without its ever entering the reducer. So + * such a failure marks the journal unreconciled: every further `append` and + * `checkpoint` throws until `load()` has replayed the tail — the possibly + * written record included — for the caller to reduce. A failure BEFORE the + * record reaches the store (the `afterCommittedState` sync) wrote nothing and + * leaves the journal usable. */ import type { JsStore } from '@animalabs/chronicle'; @@ -88,6 +99,8 @@ export class RecordJournal { /** Highest entry id this journal has loaded or appended (0 when none). */ private lastEntryId = 0n; private sinceCheckpoint = 0; + /** Set by a failure after a record reached the store; cleared by load(). */ + private unreconciled: Error | null = null; constructor(store: JsStore, opts: { type: string; checkpointType?: string }) { if (!opts.type) throw new Error('RecordJournal: a record type is required'); @@ -106,8 +119,9 @@ export class RecordJournal { /** * The latest checkpoint and every entry after it, whichever branch each was - * appended on. Also positions the journal, so later checkpoints cover what - * was loaded. + * appended on. The caller rebuilds its reduced state from exactly these — + * at open, and to reconcile after an ambiguous write. Also positions the + * journal, so later checkpoints cover what was loaded. */ load(): JournalLoad { let snapshot: Snapshot | null = null; @@ -138,18 +152,24 @@ export class RecordJournal { this.lastEntryId = tail.length > 0 ? tail[tail.length - 1]!.value : through; this.sinceCheckpoint = entries.length; + this.unreconciled = null; return { snapshot, entries }; } + /** True after an ambiguous write, until load() has replayed it. */ + get needsReconcile(): boolean { + return this.unreconciled !== null; + } + /** Append one entry. The returned ref says where it landed, not that it's durable. */ append(entry: Entry, opts: AppendOptions = {}): JournalRecordRef { + this.assertReconciled(); if (opts.afterCommittedState) this.store.sync(); - const record = this.store.appendJson(this.type, entry); - if (opts.durable) this.store.sync(); - const value = idValue(record.id); - if (value > this.lastEntryId) this.lastEntryId = value; - this.sinceCheckpoint++; - return { id: record.id, sequence: record.sequence }; + return this.write(this.type, entry, opts.durable === true, (ref) => { + const value = idValue(ref.id); + if (value > this.lastEntryId) this.lastEntryId = value; + this.sinceCheckpoint++; + }); } /** @@ -157,10 +177,33 @@ export class RecordJournal { * loaded or appended. Like an entry, it's durable only with `durable`. */ checkpoint(snapshot: Snapshot, opts: { durable?: boolean } = {}): JournalRecordRef { + this.assertReconciled(); const payload: CheckpointPayload = { through: this.lastEntryId.toString(), snapshot }; - const record = this.store.appendJson(this.checkpointType, payload); - if (opts.durable) this.store.sync(); - this.sinceCheckpoint = 0; - return { id: record.id, sequence: record.sequence }; + return this.write(this.checkpointType, payload, opts.durable === true, () => { + this.sinceCheckpoint = 0; + }); + } + + /** Hand one record to the store; any failure from here on is ambiguous. */ + private write(type: string, payload: unknown, durable: boolean, onWritten: (ref: JournalRecordRef) => void): JournalRecordRef { + try { + const record = this.store.appendJson(type, payload); + const ref = { id: record.id, sequence: record.sequence }; + onWritten(ref); + if (durable) this.store.sync(); + return ref; + } catch (err) { + this.unreconciled = err instanceof Error ? err : new Error(String(err)); + throw err; + } + } + + private assertReconciled(): void { + if (this.unreconciled) { + throw new Error( + `RecordJournal(${this.type}): an earlier write failed after reaching the store ` + + `(${this.unreconciled.message}); load() and reduce its tail before writing again`, + ); + } } } diff --git a/test/record-journal.test.ts b/test/record-journal.test.ts index f74513559..3618ae077 100644 --- a/test/record-journal.test.ts +++ b/test/record-journal.test.ts @@ -153,6 +153,48 @@ describe('RecordJournal', () => { }); }); + it('a write that fails after reaching the store blocks further writes until load() replays it', () => { + withStore((store) => { + let failNextSync = false; + const flaky = { + getRecordIdsByType: (type: string) => store.getRecordIdsByType(type), + getRecord: (id: string) => store.getRecord(id), + appendJson: (type: string, data: unknown) => store.appendJson(type, data), + sync: () => { + if (failNextSync) { failNextSync = false; throw new Error('disk full'); } + store.sync(); + }, + } as unknown as JsStore; + const journal = new RecordJournal(flaky, { type: 'journal-test/entry' }); + + // The durable barrier fails after A reached the store: the caller never reduces A. + failNextSync = true; + assert.throws(() => journal.append({ n: 'A' }, { durable: true }), /disk full/); + assert.strictEqual(journal.needsReconcile, true); + // Without reconciliation, a later write and checkpoint would cover A unreduced. + assert.throws(() => journal.append({ n: 'B' }), /load\(\) and reduce its tail before writing again/); + assert.throws(() => journal.checkpoint({ seen: ['B'] }), /load\(\) and reduce its tail/); + + const reconciled = journal.load(); + assert.deepStrictEqual(reconciled.entries.map((e) => e.entry.n), ['A'], 'load replays the ambiguous record'); + assert.strictEqual(journal.needsReconcile, false); + journal.append({ n: 'B' }); + journal.checkpoint({ seen: ['A', 'B'] }); + assert.deepStrictEqual(new RecordJournal(store, { type: 'journal-test/entry' }).load(), + { snapshot: { seen: ['A', 'B'] }, entries: [] }); + + // A failure BEFORE the record reaches the store wrote nothing and leaves the journal usable. + failNextSync = true; + assert.throws(() => journal.append({ n: 'C' }, { afterCommittedState: true }), /disk full/); + assert.strictEqual(journal.needsReconcile, false); + journal.append({ n: 'D' }); + assert.deepStrictEqual( + new RecordJournal(store, { type: 'journal-test/entry' }).load().entries.map((e) => e.entry.n), + ['D'], + ); + }); + }); + it('refuses a checkpoint type equal to the entry type', () => { withStore((store) => { assert.throws(() => new RecordJournal(store, { type: 't', checkpointType: 't' }), /distinct record types/); From 82a267a81203308418136c7582d70cb34dfe6484 Mon Sep 17 00:00:00 2001 From: Ada-1017 Date: Wed, 7 Oct 2026 05:36:12 -0400 Subject: [PATCH 04/73] fix(surgery): report unresolved marker bookkeeping as unresolved Review (June-1016, room-225 #41840): when both activate() and discard() failed, the receipt said `not-scheduled` while the batch stayed prepared, and the reconciliation that the surgery's own background delivery runs promoted it at once: June's probe saw a `not-scheduled, queued: 0` receipt followed by an active ledger batch and two real add_reaction calls. The fourth test's title claimed "stays prepared" without asserting it. SurgeryMarkerReceipt now has four outcomes, each with a guarantee that holds on every later path: - `not-scheduled`: the batch was retired from the ledger, so no reconciliation or restart can deliver it (no batchId is offered). - `unresolved`: the ledger could record neither activation nor retirement; the receipt names the batch and promises nothing about it. The ops alert is 'discord-awareness-unresolved' and says it may still be delivered. - `queued` / `none` unchanged. Tests: the not-scheduled case now runs against the fake Discord server and asserts no reaction is sent across the delivery reconciliation and a full restart on the same store; the double-failure case asserts `unresolved`, names the batch still in the ledger, and makes no prepared/held claim. Focused + related files: 87/87 plus 6/6. Co-Authored-By: Claude Opus 5.5 --- .../fix-surgery-marks-receipt.fixed.md | 11 ++++--- src/framework.ts | 31 +++++++++++-------- src/operator-log.ts | 14 +++++---- test/surgery-marks-receipt.test.ts | 30 +++++++++++------- 4 files changed, 51 insertions(+), 35 deletions(-) diff --git a/changelog.d/fix-surgery-marks-receipt.fixed.md b/changelog.d/fix-surgery-marks-receipt.fixed.md index ae2c47d1e..caef410eb 100644 --- a/changelog.d/fix-surgery-marks-receipt.fixed.md +++ b/changelog.d/fix-surgery-marks-receipt.fixed.md @@ -6,8 +6,9 @@ 15 minutes for a rollback that removed 918 channel messages. Delivery now continues in the background, behind the same MCPL data-plane gate as before. The result's new `markers` field (`none` | `queued` | - `not-scheduled`) says whether marks were scheduled, never that Discord - accepted them. If the ledger write after the switch fails, the result - reports `not-scheduled` and retires the batch rather than failing a - rollback that already applied, and a suppression whose redactions all - committed is no longer undone by that failure. + `not-scheduled` | `unresolved`) says whether marks were scheduled, never + that Discord accepted them. If the ledger write after the switch fails, the + rollback that already applied is no longer reported as failed: the batch is + retired (`not-scheduled`), or, if even that write fails, the result says + the batch may still be delivered (`unresolved`). A suppression whose + redactions all committed is no longer undone by that failure. diff --git a/src/framework.ts b/src/framework.ts index 3f100dcce..f296c4ebe 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -721,9 +721,8 @@ function bodyGroupRun( function describeMarkers(m: SurgeryMarkerReceipt): string { if (m.status === 'queued') return `queued:${m.queued}`; - if (m.status === 'not-scheduled') { - return m.batchId ? `not-scheduled(batch ${m.batchId} left prepared)` : 'not-scheduled(batch retired)'; - } + if (m.status === 'not-scheduled') return 'not-scheduled(batch retired)'; + if (m.status === 'unresolved') return `unresolved(batch ${m.batchId} may still be delivered)`; return 'none'; } @@ -6294,13 +6293,15 @@ export class AgentFramework { /** * Record a surgery's prepared marker batch as active once its body change * has landed, and say what happened. Never throws: the body is already - * applied, so a ledger failure here is reported as `not-scheduled` rather - * than as a failed surgery an operator might retry. A batch that could not - * be activated is retired, so that the reconciliation which delivery runs - * next cannot promote it behind a receipt saying it was not scheduled; only - * when retiring fails too does it stay prepared (named by `batchId`). A - * batch with no refs (a suppression journal with nothing addressable) is - * still activated, which closes its resume journal; it schedules no marks. + * applied, so a ledger failure here is reported in the receipt rather than + * as a failed surgery an operator might retry. A batch that could not be + * activated is retired, so no later reconciliation can deliver it behind a + * receipt saying it was not scheduled (`not-scheduled`). If retiring fails + * too, the receipt says only that the outcome is `unresolved`: the batch is + * still prepared, and the reconciliation that delivery runs next may well + * promote it. A batch with no refs (a suppression journal with nothing + * addressable) is still activated, which closes its resume journal; it + * schedules no marks. */ private activateSurgeryMarkers( batch: DiscordAwarenessBatch | null, @@ -6326,14 +6327,18 @@ export class AgentFramework { ); if (batch.refs.length === 0) return { status: 'none', queued: 0 }; this.opsAlert( - 'discord-awareness-not-scheduled', + retired ? 'discord-awareness-not-scheduled' : 'discord-awareness-unresolved', agentName, - `${verb} applied, but its ${batch.refs.length} awareness mark(s) were not scheduled: ${detail}`, + retired + ? `${verb} applied, but its ${batch.refs.length} awareness mark(s) were not scheduled ` + + `(batch ${batch.id} retired): ${detail}` + : `${verb} applied, but awareness batch ${batch.id} (${batch.refs.length} mark(s)) could be ` + + `neither activated nor retired; it may still be delivered: ${detail}`, { data: { batchId: batch.id, retired } }, ); return retired ? { status: 'not-scheduled', queued: 0, error: detail } - : { status: 'not-scheduled', queued: 0, error: detail, batchId: batch.id }; + : { status: 'unresolved', queued: 0, batchId: batch.id, error: detail }; } if (batch.refs.length === 0) return { status: 'none', queued: 0 }; return { status: 'queued', queued: batch.refs.length, batchId: batch.id }; diff --git a/src/operator-log.ts b/src/operator-log.ts index 48de210f0..2a50106c5 100644 --- a/src/operator-log.ts +++ b/src/operator-log.ts @@ -128,16 +128,18 @@ export function capIds(ids: string[]): { ids: string[]; count: number; truncated * handed to the background drain. It is not a delivery claim: per-message * outcomes stay in the outbox ledger. * - `not-scheduled` — the body change landed, but recording the batch as - * active failed, so no mark will be delivered for it: the batch is retired - * from the ledger. Only if retiring it failed as well is `batchId` present: - * that batch stays prepared, and a later reconciliation (or, for a - * suppression, the next startup) that can write the ledger may still - * promote it. + * active failed, so it was retired from the ledger: none of its marks will + * be delivered. + * - `unresolved` — the ledger could record neither the activation nor the + * retirement. The batch (`batchId`) is still in the ledger in its prepared + * state, and any later reconciliation or restart that can read it may + * promote and deliver it; nothing here promises otherwise. */ export type SurgeryMarkerReceipt = | { status: 'none'; queued: 0 } | { status: 'queued'; queued: number; batchId: string } - | { status: 'not-scheduled'; queued: 0; error: string; batchId?: string }; + | { status: 'not-scheduled'; queued: 0; error: string } + | { status: 'unresolved'; queued: 0; batchId: string; error: string }; /** Thrown by live surgery methods when a request cannot be honored. `code` * lets surfaces distinguish "agent busy — quiesce first" from bad input. */ diff --git a/test/surgery-marks-receipt.test.ts b/test/surgery-marks-receipt.test.ts index 13e8793dd..c71ac0f31 100644 --- a/test/surgery-marks-receipt.test.ts +++ b/test/surgery-marks-receipt.test.ts @@ -205,7 +205,7 @@ describe('surgery marker receipt', () => { }); it('a rollback whose marker bookkeeping fails after the switch reports the body as applied', async () => { - await start(false); + await start(true); const { tail } = seed(2); const original = DiscordAwarenessOutbox.prototype.activate; DiscordAwarenessOutbox.prototype.activate = function () { @@ -222,18 +222,23 @@ describe('surgery marker receipt', () => { assert.equal(result.messagesRemoved, 2); assert.equal(result.markers.status, 'not-scheduled'); assert.match(result.markers.status === 'not-scheduled' ? result.markers.error : '', /injected ledger failure/); - assert.equal(result.markers.status === 'not-scheduled' && result.markers.batchId, undefined); - // The unactivated batch is retired, so the reconciliation that delivery - // runs cannot promote it behind a receipt that said "not scheduled". - await framework.syncDiscordAwarenessMarkers(); - assert.deepEqual(ledger(), []); - const entry = framework.getOperatorLog({ limit: 10 }).find((e) => e.kind === 'rollback'); assert.equal(entry?.error, undefined, 'logged as an applied rollback, not a failure'); assert.equal((entry?.result?.markers as { status?: string } | undefined)?.status, 'not-scheduled'); + + // "Not scheduled" holds across the reconciliation delivery runs and + // across a restart: the batch was retired, so nothing can promote it. + writeFileSync(holdPath, '1'); + await framework.syncDiscordAwarenessMarkers(); + assert.deepEqual(ledger(), []); + await framework.stop(); + await start(true); + await framework.syncDiscordAwarenessMarkers(); + assert.deepEqual(ledger(), []); + assert.equal(jsonl(callsPath).length, 0, 'no reaction was ever sent'); }); - it('names a batch it could neither activate nor retire, which stays prepared', async () => { + it('reports unresolved bookkeeping when the batch can be neither activated nor retired', async () => { await start(false); const { tail } = seed(2); const activate = DiscordAwarenessOutbox.prototype.activate; @@ -251,11 +256,14 @@ describe('surgery marker receipt', () => { DiscordAwarenessOutbox.prototype.activate = activate; DiscordAwarenessOutbox.prototype.discard = discard; } - assert.equal(cm().currentBranch().name, result.targetBranch); - assert.equal(result.markers.status, 'not-scheduled'); + assert.equal(cm().currentBranch().name, result.targetBranch, 'the rollback stands'); + // The receipt promises nothing about this batch: it is still in the + // ledger, and a reconciliation may promote and deliver it. + assert.equal(result.markers.status, 'unresolved'); const batches = ledger(); assert.equal(batches.length, 1); - assert.equal(result.markers.status === 'not-scheduled' && result.markers.batchId, batches[0].id); + assert.equal(result.markers.status === 'unresolved' && result.markers.batchId, batches[0].id); + assert.match(result.markers.status === 'unresolved' ? result.markers.error : '', /injected ledger failure/); }); it('a suppression whose marker bookkeeping fails after the last redaction stands', async () => { From 0254342bd8f63f7907af517f82d9bdeb2cab3cd4 Mon Sep 17 00:00:00 2001 From: Ada-1017 Date: Wed, 7 Oct 2026 05:41:04 -0400 Subject: [PATCH 05/73] docs(surgery): describe delivery as started without awaiting, not run after return Review (June-1016, room-225 #41877): syncDiscordAwarenessMarkers() starts the drain synchronously through its first awaited tools/call, so the first reaction request can be written before release() and before the caller gets its receipt. Comments that said the drain runs after the call returns, or after the reservation is released, overstated the ordering. They now say delivery is started (behind the gate) and continues without the caller or the store reservation waiting on it. Comment-only change. Co-Authored-By: Claude Opus 5.5 --- src/framework.ts | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/src/framework.ts b/src/framework.ts index f296c4ebe..8385cc033 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -5957,8 +5957,9 @@ export class AgentFramework { * * Returns once the switch has landed and the marker batch is durably active * (`markers`), without waiting for Discord to accept the reactions: delivery - * runs after the store reservation is released, behind the MCPL data-plane - * gate, and its per-message outcomes stay in the outbox ledger. + * is started, behind the MCPL data-plane gate, and continues without the + * caller or the store reservation waiting on it; its per-message outcomes + * stay in the outbox ledger. * * Throws `OperatorActionError` (`agent-busy`, `unknown-message`, …) — the * agent must be idle; nothing is queued. @@ -6061,7 +6062,8 @@ export class AgentFramework { // apart from the body (`markers`). markers = this.activateSurgeryMarkers(markerBatch, 'rollback', agentName); // Installs the MCPL data-plane gate synchronously, before the store is - // released; the drain itself runs after this call returns. + // released, and starts delivery without awaiting it (the first + // reaction request may already be on the wire when this returns). this.deliverDiscordAwarenessInBackground('rollback', agentName); this.materializeConfigMountAfterBranchSwitch(); } finally { @@ -6109,7 +6111,8 @@ export class AgentFramework { * the batch is `explicit`-activated only after every removal succeeded, and * a crash mid-way is finished at next boot (resumePreparedDiscordSuppressions). * Like rollbackToMessage, it returns once the body change and the batch's - * activation are durable (`markers`); delivery runs in the background. + * activation are durable (`markers`); delivery continues without the caller + * waiting on it. * * Not retroactive over derived state: a message already folded into an * autobiographical summary stays in that summary — roll back to before it @@ -6250,7 +6253,8 @@ export class AgentFramework { // loses nothing.) markers = this.activateSurgeryMarkers(markerBatch, 'suppress', agentName); // Installs the MCPL data-plane gate synchronously, before the store is - // released; the drain itself runs after this call returns. + // released, and starts delivery without awaiting it (the first + // reaction request may already be on the wire when this returns). this.deliverDiscordAwarenessInBackground('suppress', agentName); this.materializeConfigMountAfterBranchSwitch(); } finally { @@ -6345,13 +6349,15 @@ export class AgentFramework { } /** - * Hand awareness delivery to the background, without holding the caller. + * Start awareness delivery without the caller awaiting it. * `syncDiscordAwarenessMarkers` installs the MCPL data-plane gate * synchronously, before its first await, so calling this while a surgery * still holds the store keeps MCPL-borne wakes behind the gate exactly as - * before. Only the surgery's response and its store reservation stop - * waiting on remote reactions. A failure is accounted by the gate itself - * (connections recycled); here it is logged and raised as an ops alert. + * before. The drain starts in the same synchronous run (the first reaction + * request can be written before this returns) and continues independently; + * only the surgery's response and its store reservation stop waiting on + * remote reactions. A failure is accounted by the gate itself (connections + * recycled); here it is logged and raised as an ops alert. */ private deliverDiscordAwarenessInBackground(verb: 'rollback' | 'suppress', agentName: string): void { if (!this.discordAwarenessOutbox) return; From 287eef2d695cfaad10e73787e2907d446bab00f4 Mon Sep 17 00:00:00 2001 From: Ada-1017 Date: Wed, 7 Oct 2026 05:45:45 -0400 Subject: [PATCH 06/73] docs(surgery): say what every receipt outcome guarantees, not 'durably active' Review (June-1016, room-225 #41978): the SurgeryMarkerReceipt doc said an unresolved batch is 'in its prepared state', and the surgery docs and reservation comments said the call returns once the batch is durably active. Neither holds for every outcome: June's real-server probe saw an unresolved batch already promoted and delivering. The docs now say the body is applied with a marker-scheduling receipt, the reservation is held until marker scheduling has been recorded or reported, and an unresolved receipt names a retained batch whose scheduling outcome is unresolved. The changelog fragment's opening sentence is corrected the same way. Comments only. Co-Authored-By: Claude Opus 5.5 --- .../fix-surgery-marks-receipt.fixed.md | 4 +-- src/framework.ts | 26 ++++++++++--------- src/operator-log.ts | 6 ++--- 3 files changed, 19 insertions(+), 17 deletions(-) diff --git a/changelog.d/fix-surgery-marks-receipt.fixed.md b/changelog.d/fix-surgery-marks-receipt.fixed.md index caef410eb..85e715a28 100644 --- a/changelog.d/fix-surgery-marks-receipt.fixed.md +++ b/changelog.d/fix-surgery-marks-receipt.fixed.md @@ -1,7 +1,7 @@ - Live `rollbackToMessage` and `suppressMessages` (and message-granular `host/command` undo, which rides on rollback) return once the body change - has landed and the Discord awareness batch is durably active, instead of - after every 💤 reaction has been delivered. Previously the store stayed + has landed, with a marker-scheduling receipt, instead of after every 💤 + reaction has been delivered. Previously the store stayed reserved, and the operator got no answer, for the whole serial drain: about 15 minutes for a rollback that removed 918 channel messages. Delivery now continues in the background, behind the same MCPL data-plane gate as diff --git a/src/framework.ts b/src/framework.ts index 8385cc033..13551a5ce 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -5955,11 +5955,11 @@ export class AgentFramework { * Discord messages that left the live context get awareness markers via the * durable outbox, exactly as message-granular `undo` does. * - * Returns once the switch has landed and the marker batch is durably active - * (`markers`), without waiting for Discord to accept the reactions: delivery - * is started, behind the MCPL data-plane gate, and continues without the - * caller or the store reservation waiting on it; its per-message outcomes - * stay in the outbox ledger. + * Returns once the switch has landed: the body is applied, with a + * marker-scheduling receipt (`markers`). It does not wait for Discord to + * accept the reactions: delivery is started, behind the MCPL data-plane + * gate, and continues without the caller or the store reservation waiting + * on it; its per-message outcomes stay in the outbox ledger. * * Throws `OperatorActionError` (`agent-busy`, `unknown-message`, …) — the * agent must be idle; nothing is queued. @@ -6022,8 +6022,9 @@ export class AgentFramework { } // Gate + reserve the whole store (see reserveStoreForSurgery); held - // until the switch has landed and its marker batch is durably active, - // or the switch has been rolled back. Remote delivery is not waited on. + // until the switch has landed and its marker scheduling has been + // recorded or reported, or the switch has been rolled back. Remote + // delivery is not waited on. const release = this.reserveStoreForSurgery('roll back', agentName); let branchName: string; let markers: SurgeryMarkerReceipt; @@ -6110,9 +6111,9 @@ export class AgentFramework { * removed together. Discord originals get awareness markers via the outbox; * the batch is `explicit`-activated only after every removal succeeded, and * a crash mid-way is finished at next boot (resumePreparedDiscordSuppressions). - * Like rollbackToMessage, it returns once the body change and the batch's - * activation are durable (`markers`); delivery continues without the caller - * waiting on it. + * Like rollbackToMessage, it returns once the body change has landed, with + * a marker-scheduling receipt (`markers`); delivery continues without the + * caller waiting on it. * * Not retroactive over derived state: a message already folded into an * autobiographical summary stays in that summary — roll back to before it @@ -6188,8 +6189,9 @@ export class AgentFramework { throw new OperatorActionError('invalid', 'Suppression branch name must differ from the active branch'); } // Gate + reserve the whole store (see reserveStoreForSurgery); held - // until the fork is fully redacted and its marker batch durably - // active, or the source is restored. Remote delivery is not waited on. + // until the fork is fully redacted and its marker scheduling has been + // recorded or reported, or the source is restored. Remote delivery is + // not waited on. const release = this.reserveStoreForSurgery('suppress', agentName); let createdBranch: string; let markers: SurgeryMarkerReceipt; diff --git a/src/operator-log.ts b/src/operator-log.ts index 2a50106c5..23f3dbaff 100644 --- a/src/operator-log.ts +++ b/src/operator-log.ts @@ -131,9 +131,9 @@ export function capIds(ids: string[]): { ids: string[]; count: number; truncated * active failed, so it was retired from the ledger: none of its marks will * be delivered. * - `unresolved` — the ledger could record neither the activation nor the - * retirement. The batch (`batchId`) is still in the ledger in its prepared - * state, and any later reconciliation or restart that can read it may - * promote and deliver it; nothing here promises otherwise. + * retirement. `batchId` names a retained batch whose scheduling outcome is + * unresolved: any later reconciliation or restart that can read it may + * promote and deliver it, and nothing here promises otherwise. */ export type SurgeryMarkerReceipt = | { status: 'none'; queued: 0 } From 80c74c813a75252deb874c9c61bb29eaca4eb2a1 Mon Sep 17 00:00:00 2001 From: Ada-1017 Date: Wed, 7 Oct 2026 06:02:34 -0400 Subject: [PATCH 07/73] docs(changelog): state the incident's drain duration as projected, not observed Review (June-1016, room-225 #42149): 'about 15 minutes for a rollback that removed 918 channel messages' read as an observed duration. The household stopped the host after 225 deliveries; fifteen minutes is the projection at the reported pace. Replaced with June's endorsed wording. Changelog only. Co-Authored-By: Claude Opus 5.5 --- changelog.d/fix-surgery-marks-receipt.fixed.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/changelog.d/fix-surgery-marks-receipt.fixed.md b/changelog.d/fix-surgery-marks-receipt.fixed.md index 85e715a28..dcaf321ea 100644 --- a/changelog.d/fix-surgery-marks-receipt.fixed.md +++ b/changelog.d/fix-surgery-marks-receipt.fixed.md @@ -1,9 +1,10 @@ - Live `rollbackToMessage` and `suppressMessages` (and message-granular `host/command` undo, which rides on rollback) return once the body change has landed, with a marker-scheduling receipt, instead of after every 💤 - reaction has been delivered. Previously the store stayed - reserved, and the operator got no answer, for the whole serial drain: about - 15 minutes for a rollback that removed 918 channel messages. Delivery now + reaction has been delivered. Previously the store stayed reserved, and the + operator got no answer, for the whole serial drain. At the reported pace + of about one reaction per second, 918 queued marks would take roughly 15 + minutes; the household's client timed out after 60 seconds. Delivery now continues in the background, behind the same MCPL data-plane gate as before. The result's new `markers` field (`none` | `queued` | `not-scheduled` | `unresolved`) says whether marks were scheduled, never From acd95b6b19f04eef2b9ded6be93df021b1987073 Mon Sep 17 00:00:00 2001 From: Fern-1209 Date: Wed, 7 Oct 2026 07:40:54 -0400 Subject: [PATCH 08/73] fix(awareness): new gates wait for running drains, not only pending marks Greptile's P1 on #246 (discussion_r4205867371). Now that rollbackToMessage returns while its marks are still being delivered, an operator can restore the source branch (api/server.ts cmdBranchSwitch: store.switchBranch, then syncDiscordAwarenessMarkers) while the first add_reaction still waits for its reply. Reconciliation makes the rollback's marks undesired, and the held add is not present yet, so pending() is empty. The new generation was then a no-work gate: releaseMcplDataPlaneGate completed it at once and it replaced the generation that was waiting on the running drain, opening every MCPL data plane while that drain went on sending its saved adds and then the removals they need. A tools/list_changed after such a restore reached the same hole. On main the same switch could only land as a concurrent request during a rollback's awaited drain (cmdBranchSwitch never consults surgeryHold); #246 makes it the ordinary sequence, so the fix belongs here. beginDiscordAwarenessBarrier now accounts for the server ids of every running drain (discordAwarenessDrains) as well as those with pending operations, and joins those drains; drainDiscordAwarenessOutbox already returns a running drain, which picks up the removals in its next pass. Every gate installer (startup, sync, quiesce resume, connect, reconnect, list change) goes through it. Only the case of a running drain with nothing pending changes: while a drain runs, its in-flight operation is otherwise still pending in the ledger. Out of scope, deliberately: the drain still sends the adds it saved before the restore and then removes them. That is how it already behaves, and the same as a restore on main after the rollback finished; the stacked marks contract (retract supersedes undispatched adds) replaces this drain. Test: restoring the source while delivery runs keeps traffic gated until the running drain is done. Red on 80c74c8 (the probe was answered while the first reaction was still held); green here. Focused file 7/7; awareness/surgery/quiesce/reconnect files 96/96. --- .../fix-surgery-marks-receipt.fixed.md | 18 ++++--- src/framework.ts | 29 ++++++++--- test/surgery-marks-receipt.test.ts | 48 +++++++++++++++++++ 3 files changed, 81 insertions(+), 14 deletions(-) diff --git a/changelog.d/fix-surgery-marks-receipt.fixed.md b/changelog.d/fix-surgery-marks-receipt.fixed.md index dcaf321ea..20abfe420 100644 --- a/changelog.d/fix-surgery-marks-receipt.fixed.md +++ b/changelog.d/fix-surgery-marks-receipt.fixed.md @@ -6,10 +6,14 @@ of about one reaction per second, 918 queued marks would take roughly 15 minutes; the household's client timed out after 60 seconds. Delivery now continues in the background, behind the same MCPL data-plane gate as - before. The result's new `markers` field (`none` | `queued` | - `not-scheduled` | `unresolved`) says whether marks were scheduled, never - that Discord accepted them. If the ledger write after the switch fails, the - rollback that already applied is no longer reported as failed: the batch is - retired (`not-scheduled`), or, if even that write fails, the result says - the batch may still be delivered (`unresolved`). A suppression whose - redactions all committed is no longer undone by that failure. + before. A gate installed while a drain is still running (restoring the + source branch mid-delivery, say) now waits for that drain even when the + switch leaves nothing pending, so traffic stays held through the drain's + remaining reactions and the removals they need. The result's new + `markers` field (`none` | `queued` | `not-scheduled` | `unresolved`) says + whether marks were scheduled, never that Discord accepted them. If the + ledger write after the switch fails, the rollback that already applied is + no longer reported as failed: the batch is retired (`not-scheduled`), or, + if even that write fails, the result says the batch may still be delivered + (`unresolved`). A suppression whose redactions all committed is no longer + undone by that failure. diff --git a/src/framework.ts b/src/framework.ts index 13551a5ce..9b80c3985 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -13295,7 +13295,10 @@ export class AgentFramework { await this.refreshMcplTools(); } - /** Reconcile the durable ledger with Chronicle, then deliver every server's work. */ + /** + * Reconcile the durable ledger with Chronicle, then deliver every server's + * work, joining any drain still running. + */ async syncDiscordAwarenessMarkers(_onlyServerId?: string): Promise { if (!this.discordAwarenessOutbox) return; // A targeted retry may discover pending work for other Discord servers @@ -13510,13 +13513,25 @@ export class AgentFramework { throw new DiscordAwarenessAccountingError('branch reconciliation', error); } const pending = this.readDiscordAwarenessPending(); - if (pending.length === 0) { + // A drain that is still running holds operations it read before this + // reconciliation, and it sends them whatever the ledger now says. A branch + // move can hide them from `pending`: restoring a rollback's source makes + // its marks undesired, and an add whose reply has not arrived is not yet + // present, so the ledger shows nothing to do while the drain goes on to + // send its saved adds and then the removals they need. So a generation + // accounts for every running drain as well as for every pending + // operation, and joins those drains instead of releasing the data planes + // beside them. + const serverIds = [...new Set([ + ...pending.map((operation) => operation.ref.serverId), + ...this.discordAwarenessDrains.keys(), + ])]; + if (serverIds.length === 0) { return { requiresBarrier: false, promise: Promise.resolve({ status: 'delivered', delivered: 0, failed: 0 }), }; } - const serverIds = [...new Set(pending.map((operation) => operation.ref.serverId))]; const promise = Promise.all( serverIds.map((serverId) => this.drainDiscordAwarenessOutbox(serverId)), ).then((outcomes): DiscordAwarenessDrainOutcome => { @@ -13546,10 +13561,10 @@ export class AgentFramework { /** * Install one framework-global Discord-awareness generation before releasing - * any inference-bearing event. With pending work, every MCPL data plane is - * paused while all control planes remain live. With no pending work, ready() - * runs synchronously so request responders preserve their historical - * same-stack behavior. + * any inference-bearing event. With pending work or a drain still running, + * every MCPL data plane is paused while all control planes remain live. + * With neither, ready() runs synchronously so request responders preserve + * their historical same-stack behavior. */ private installMcplDataPlaneGate(): DiscordAwarenessBarrier { for (const connection of this.mcplServerRegistry?.getAllServers() ?? []) { diff --git a/test/surgery-marks-receipt.test.ts b/test/surgery-marks-receipt.test.ts index c71ac0f31..cff83288b 100644 --- a/test/surgery-marks-receipt.test.ts +++ b/test/surgery-marks-receipt.test.ts @@ -204,6 +204,54 @@ describe('surgery marker receipt', () => { assert.ok(probe.at >= lastReaction, 'the probe is answered only after the held reactions'); }); + it('restoring the source while delivery runs keeps traffic gated until the running drain is done', async () => { + await start(true); + const { tail } = seed(3); + + const rollback = await settlesWithin( + framework.rollbackToMessage('resident', { messageId: tail }), + 3_000, + 'rollbackToMessage with reactions held', + ); + await waitFor('the first reaction call to reach the server', () => jsonl(callsPath).length >= 1); + + // The operator restores the source the way the branch-switch API does + // (store switch, then synchronization) while the first add_reaction still + // waits for its reply. Reconciliation makes every mark undesired, and the + // held one is not present yet, so the ledger shows nothing pending: only + // the running drain knows it will still send its saved adds. + framework.getStore().switchBranch(rollback.sourceBranch); + const restored = framework.syncDiscordAwarenessMarkers(); + + writeFileSync(probePath, '1'); + await waitFor('the probe to be sent', () => jsonl(eventsPath).some((e) => e.event === 'probe-sent')); + await new Promise((resolve) => setTimeout(resolve, 200)); + assert.equal( + jsonl(eventsPath).some((e) => e.event === 'probe-answered'), + false, + 'channel traffic waits for the running drain even though nothing appears pending', + ); + + writeFileSync(holdPath, '1'); + await settlesWithin(restored, 5_000, "the restore's synchronization"); + await waitFor('the probe to be answered after delivery', () => + jsonl(eventsPath).some((e) => e.event === 'probe-answered')); + + // Every reaction request the drain made, including the removals the + // restore needs, was answered before channel traffic was released, and + // the restore leaves no mark behind. + const events = jsonl<{ event: string }>(eventsPath); + const probeIndex = events.findIndex((e) => e.event === 'probe-answered'); + const answered = (from: number, to?: number) => + events.slice(from, to).filter((e) => e.event === 'reaction-answered').length; + assert.equal(answered(0, probeIndex), jsonl(callsPath).length, 'every reaction call answered before the probe'); + assert.equal(answered(probeIndex), 0, 'no reaction answered after the probe'); + assert.deepEqual( + ledger()[0].refs.map((ref) => [ref.messageId, ref.desired, ref.markerPresent, ref.deliveryStatus]), + [['amb-0', false, false, 'applied'], ['amb-1', false, false, 'applied'], ['amb-2', false, false, 'applied']], + ); + }); + it('a rollback whose marker bookkeeping fails after the switch reports the body as applied', async () => { await start(true); const { tail } = seed(2); From 8b22d1fbed4c1472b4018373b28e8fc4cd00925d Mon Sep 17 00:00:00 2001 From: Fern-1209 Date: Wed, 7 Oct 2026 07:42:08 -0400 Subject: [PATCH 09/73] docs(surgery): `none` also covers a disabled awareness outbox Greptile's P2 on #246 (discussion_r4205867381). The receipt documented `none` as "the change removed no addressable Discord message", but a framework without an awareness outbox (no discordAwarenessOutboxPath and no storePath, which the config doc calls disabled) has no batch to prepare, so activateSurgeryMarkers(null) returns `none` even when the surgery removed addressable Discord messages. A caller could read that as proof that none were affected. Broaden the documented meaning to what the code guarantees, no marks were scheduled, name both reasons, and point to the surgery's removedRefs (discordRefs in the operator log) as what says whether any addressable message was removed. The four statuses stay as they are: removedRefs already sits beside every receipt and tells the two `none` cases apart. Comment-only change. --- src/operator-log.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/operator-log.ts b/src/operator-log.ts index 23f3dbaff..726ee93ed 100644 --- a/src/operator-log.ts +++ b/src/operator-log.ts @@ -123,7 +123,11 @@ export function capIds(ids: string[]): { ids: string[]; count: number; truncated * the body change itself. A surgery that returns has applied its body change; * this says only whether marks were scheduled: * - * - `none` — the change removed no addressable Discord message. + * - `none` — no marks were scheduled: the change removed no addressable + * Discord message, or the awareness outbox is disabled (no + * `discordAwarenessOutboxPath` and no `storePath`). It is not proof that + * no addressable Discord message was removed: the surgery's `removedRefs` + * (`discordRefs` in the operator log) say whether any was. * - `queued` — the outbox batch is durably active and its delivery has been * handed to the background drain. It is not a delivery claim: per-message * outcomes stay in the outbox ledger. From 8b530ee4f40e3a20da7c973678998fdd2f5c0cfc Mon Sep 17 00:00:00 2001 From: Fern-1209 Date: Wed, 7 Oct 2026 07:47:06 -0400 Subject: [PATCH 10/73] test(surgery): require both surgeries' reactions before traffic, by event order Greptile's P2 on #246 (discussion_r4205867396). The second-batch test compared the probe's timestamp only with reactions that happened to be answered, with >=, so it passed if the second surgery's amb-0 were never delivered, and equal milliseconds let a probe answered first pass too. Assert instead, by the fake server's own event order, that exactly amb-0, amb-2 and amb-3 were answered before probe-answered (greptile's suggestion, with a comment saying why). Mutation check: with the drain cut to a single pass (a `break` after the first for loop in the built drainDiscordAwarenessOutbox), the second batch is never delivered (delivered=2). The old assertion still passed under it; the new one fails with actual [amb-2, amb-3]. Without the mutation it passes. Focused file 7/7; awareness/surgery/quiesce/reconnect files 96/96 after a clean rebuild. --- test/surgery-marks-receipt.test.ts | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/test/surgery-marks-receipt.test.ts b/test/surgery-marks-receipt.test.ts index cff83288b..f2a1b4c1b 100644 --- a/test/surgery-marks-receipt.test.ts +++ b/test/surgery-marks-receipt.test.ts @@ -198,10 +198,21 @@ describe('surgery marker receipt', () => { writeFileSync(holdPath, '1'); await waitFor('the probe to be answered after delivery', () => jsonl(eventsPath).some((e) => e.event === 'probe-answered')); - const events = jsonl<{ event: string; at: number }>(eventsPath); - const lastReaction = Math.max(...events.filter((e) => e.event === 'reaction-answered').map((e) => e.at)); - const probe = events.find((e) => e.event === 'probe-answered')!; - assert.ok(probe.at >= lastReaction, 'the probe is answered only after the held reactions'); + // The server's own event order, not timestamps (two events can share a + // millisecond), and every expected reaction by id: one compared only with + // the reactions that happened to be answered would pass if the second + // surgery's batch were never delivered. + const events = jsonl<{ event: string; messageId?: string }>(eventsPath); + const probeIndex = events.findIndex((e) => e.event === 'probe-answered'); + const answeredBeforeProbe = events.slice(0, probeIndex) + .filter((e) => e.event === 'reaction-answered') + .map((e) => e.messageId) + .sort(); + assert.deepEqual( + answeredBeforeProbe, + ['amb-0', 'amb-2', 'amb-3'], + 'both surgeries\' reactions are answered before channel traffic is released', + ); }); it('restoring the source while delivery runs keeps traffic gated until the running drain is done', async () => { From 7144970b3370b7b6f020a1e29a80df5f8d1562e6 Mon Sep 17 00:00:00 2001 From: Petra-1012 Date: Wed, 7 Oct 2026 05:45:51 -0400 Subject: [PATCH 11/73] feat(mcpl): type MCPL request failures by what reached the server MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A failed request to an MCPL server used to reject with a plain Error, so the only way to tell "never written" from "written and unanswered" from "answered with an error" was to match message text. Callers that act in the world need that difference as a fact: a publish or a reaction that was never sent left the world as it was, while one that was handed to the transport and never answered may have landed. (A rollback reaction drain met exactly this: its pending reactions failed at shutdown, and only their message text said whether each had been written.) McplRequestError extends Error with `outcome`: - `not-sent`: sendRequest refused before writing (connection already closed); - `error-response`: a JSON-RPC error response; `code` and `data` kept verbatim (a connector can describe a partial multi-part post there); - `no-response`: the request was handed to the transport and no answer came (per-request timeout, a deliberate close while awaiting, or an unexpected disconnect). It does not prove the request reached the server — the transport can decline or drop a write — only that the host can no longer say it didn't. Messages are byte-identical to before, and every rejection is still an Error. Exported from the mcpl index and the package root. Tests (test/mcpl-request-outcome.test.ts, a real stdio server): an error response keeps code and data; an unanswered request and a process exit while awaiting are no-response; a request on a closed connection is not-sent; the class keeps Error identity. 12/12 with the existing request-timeout tests. Reviewed by Tessa-974 (room-203 #43780), whose wording correction for no-response this carries. Shared with room-225's lane 1 (Ada-1017). Co-Authored-By: Claude Opus 5.5 --- changelog.d/mcpl-request-outcome.added.md | 10 +++ src/index.ts | 2 +- src/mcpl/index.ts | 2 +- src/mcpl/server-connection.ts | 45 +++++++++-- test/mcpl-request-outcome.test.ts | 92 +++++++++++++++++++++++ 5 files changed, 144 insertions(+), 7 deletions(-) create mode 100644 changelog.d/mcpl-request-outcome.added.md create mode 100644 test/mcpl-request-outcome.test.ts diff --git a/changelog.d/mcpl-request-outcome.added.md b/changelog.d/mcpl-request-outcome.added.md new file mode 100644 index 000000000..241ec226e --- /dev/null +++ b/changelog.d/mcpl-request-outcome.added.md @@ -0,0 +1,10 @@ +- **`McplRequestError`: why an MCPL request produced no result.** A request + to an MCPL server that fails now rejects with `McplRequestError` (exported), + whose `outcome` says which of three things happened: `not-sent` (refused + before anything was written: the connection was already closed), + `error-response` (the server answered with a JSON-RPC error; `code` and the + error's `data` are kept), or `no-response` (the request was handed to the + transport and no answer came back: a timeout, or the connection closed + while awaiting; it may or may not have reached the server). Only + `not-sent` proves the server never saw the request. Messages are unchanged, + and it is still an `Error`, so existing handling keeps working. diff --git a/src/index.ts b/src/index.ts index bd07d3547..e6bf2a6b3 100644 --- a/src/index.ts +++ b/src/index.ts @@ -90,7 +90,7 @@ export type { McplServerConfig } from './mcpl/index.js'; // MCPL client connection (exposed so external tooling — server playtests, // health probes, protocol harnesses — can dial an MCPL server exactly the // way the framework host does, over stdio or WebSocket) -export { McplServerConnection } from './mcpl/index.js'; +export { McplServerConnection, McplRequestError } from './mcpl/index.js'; export type { McplHostCapabilities } from './mcpl/index.js'; // MCPL tool lifecycle (RFC-007) and tool classes (RFC-008): the types a host diff --git a/src/mcpl/index.ts b/src/mcpl/index.ts index d82790c67..9033077ec 100644 --- a/src/mcpl/index.ts +++ b/src/mcpl/index.ts @@ -106,7 +106,7 @@ export { } from './errors.js'; // Server connection and registry -export { McplServerConnection } from './server-connection.js'; +export { McplServerConnection, McplRequestError } from './server-connection.js'; export { McplServerRegistry, type McplCapabilityQuery } from './server-registry.js'; // Feature set management (permission layer) diff --git a/src/mcpl/server-connection.ts b/src/mcpl/server-connection.ts index f29fea22d..18f82d096 100644 --- a/src/mcpl/server-connection.ts +++ b/src/mcpl/server-connection.ts @@ -62,6 +62,34 @@ const DEFAULT_REQUEST_TIMEOUT_MS = 60_000; /** MCP protocol version used in the initialize handshake. */ const MCP_PROTOCOL_VERSION = '2024-11-05'; +/** + * Why a request to an MCPL server produced no result, as far as the host can + * know it. Messages are unchanged from plain errors; the `outcome` makes the + * difference between "never left the host" and "left, and no answer came + * back" a fact a caller can act on instead of a string to match: + * - `not-sent`: the request was refused before anything was written (the + * connection was already closed); + * - `error-response`: the server answered with a JSON-RPC error (it may have + * acted before failing; the error alone does not say); + * - `no-response`: the request was handed to the transport and no answer + * came back (timeout, or the connection closed while awaiting). It does + * not prove the request reached the server — the transport can decline or + * drop a write — only that the host can no longer say it didn't. + */ +export class McplRequestError extends Error { + constructor( + message: string, + readonly outcome: 'not-sent' | 'error-response' | 'no-response', + readonly code?: number, + /** The JSON-RPC error's `data`, verbatim, when the server supplied one + * (e.g. which parts of a multi-part send were posted). */ + readonly data?: unknown, + ) { + super(message); + this.name = 'McplRequestError'; + } +} + /** * Represents a pending JSON-RPC request awaiting a response. */ @@ -846,7 +874,7 @@ export class McplServerConnection extends EventEmitter { // Reject all pending requests for (const [id, pending] of this.pendingRequests) { if (pending.timer) clearTimeout(pending.timer); - pending.reject(new Error(`Connection to MCPL server "${this.id}" closed while awaiting response for ${pending.method} (id=${id})`)); + pending.reject(new McplRequestError(`Connection to MCPL server "${this.id}" closed while awaiting response for ${pending.method} (id=${id})`, 'no-response')); } this.pendingRequests.clear(); this.orphanedRequests.clear(); @@ -973,7 +1001,7 @@ export class McplServerConnection extends EventEmitter { options: RequestOptions = {}, ): Promise { if (this.closed) { - return Promise.reject(new Error(`Cannot send request: connection to "${this.id}" is closed`)); + return Promise.reject(new McplRequestError(`Cannot send request: connection to "${this.id}" is closed`, 'not-sent')); } const id = this.nextRequestId++; @@ -994,13 +1022,14 @@ export class McplServerConnection extends EventEmitter { this.orphanedRequests.set(id, { method, ...(method === 'push/render' ? { renderParams: { featureSet: params.featureSet, key: params.key, eventId: params.eventId } } : {}) }); if (this.orphanedRequests.size > 4096) this.orphanedRequests.delete(this.orphanedRequests.keys().next().value!); } - reject(new Error( + reject(new McplRequestError( `MCPL server "${this.id}" did not respond to ${method} (id=${id}) ` + `within ${timeoutMs}ms — the server may be hung. The response ` + `outcome is unknown: the request was abandoned locally but was not ` + `cancelled, so the tool may still have completed server-side; verify ` + `state before retrying; a blind retry of a stateful/side-effecting ` + `tool may duplicate it.`, + 'no-response', )); }, timeoutMs); // Don't hold the event loop open for the watchdog alone. @@ -1215,7 +1244,12 @@ export class McplServerConnection extends EventEmitter { if (response.error) { pending.reject( - new Error(`MCPL server "${this.id}" returned error for ${pending.method}: [${response.error.code}] ${response.error.message}`), + new McplRequestError( + `MCPL server "${this.id}" returned error for ${pending.method}: [${response.error.code}] ${response.error.message}`, + 'error-response', + typeof response.error.code === 'number' ? response.error.code : undefined, + (response.error as { data?: unknown }).data, + ), ); } else { pending.resolve(response.result); @@ -1267,8 +1301,9 @@ export class McplServerConnection extends EventEmitter { for (const [id, pending] of this.pendingRequests) { if (pending.timer) clearTimeout(pending.timer); pending.reject( - new Error( + new McplRequestError( `MCPL server "${this.id}" disconnected unexpectedly (code=${info.code ?? 'n/a'}, signal=${info.signal ?? 'n/a'}, reason=${info.reason ?? 'unknown'}) while awaiting ${pending.method} (id=${id})`, + 'no-response', ), ); } diff --git a/test/mcpl-request-outcome.test.ts b/test/mcpl-request-outcome.test.ts new file mode 100644 index 000000000..1b0f45647 --- /dev/null +++ b/test/mcpl-request-outcome.test.ts @@ -0,0 +1,92 @@ +/** + * McplRequestError: a request that produced no result says which of three + * things happened, as a fact rather than a message to parse. + * not-sent refused before anything was written (connection closed) + * error-response the server answered with a JSON-RPC error (code, data kept) + * no-response the request was handed to the transport and no answer + * came back (timeout, or the connection closed while + * awaiting): it may or may not have reached the server + * Callers that act in the world (publishing speech, placing reactions) need + * the difference: only `not-sent` proves the server never saw the request. + */ +import { test, afterEach } from 'node:test'; +import assert from 'node:assert/strict'; + +import { McplServerConnection, McplRequestError } from '../src/mcpl/server-connection.js'; +import type { McplHostCapabilities, McplServerConfig } from '../src/mcpl/types.js'; + +// Stdio server: answers initialize; tools/call 'fail' gets a JSON-RPC error +// with data, 'stuck' gets nothing, 'exit' kills the process mid-request. +const SERVER = ` +let buf = ''; +process.stdin.on('data', (c) => { + buf += c.toString('utf8'); + let i; + while ((i = buf.indexOf('\\n')) >= 0) { + const line = buf.slice(0, i); buf = buf.slice(i + 1); + if (!line.trim()) continue; + let m; try { m = JSON.parse(line); } catch { continue; } + const out = (o) => process.stdout.write(JSON.stringify({ jsonrpc: '2.0', id: m.id, ...o }) + '\\n'); + if (m.method === 'initialize') out({ result: { capabilities: {} } }); + else if (m.method === 'tools/call' && m.params.name === 'fail') { + out({ error: { code: -32001, message: 'POSTED: part 1 of 2', data: { posted: ['111'] } } }); + } else if (m.method === 'tools/call' && m.params.name === 'exit') { + process.exit(3); + } + // 'stuck': deliberately no response. + } +}); +setInterval(() => {}, 1 << 30); +`; + +const HOST_CAPS: McplHostCapabilities = { version: '0.4', pushEvents: true, contextHooks: { beforeInference: true }, featureSets: true }; +const config = (overrides?: Partial): McplServerConfig => + ({ id: 'outcome', command: process.execPath, args: ['-e', SERVER], ...overrides }); + +let connection: McplServerConnection | null = null; +afterEach(async () => { + await connection?.close(); + connection = null; +}); + +const rejectsWith = async (promise: Promise, outcome: McplRequestError['outcome']): Promise => { + let caught: unknown; + await assert.rejects(promise, (err: unknown) => { caught = err; return true; }); + assert.ok(caught instanceof McplRequestError, `expected McplRequestError, got ${String(caught)}`); + assert.equal((caught as McplRequestError).outcome, outcome); + return caught as McplRequestError; +}; + +test('an error response is error-response, with its code and data kept', async () => { + connection = await McplServerConnection.connect(config(), HOST_CAPS); + const err = await rejectsWith(connection.sendToolsCall('fail', {}), 'error-response'); + assert.equal(err.code, -32001); + assert.deepEqual(err.data, { posted: ['111'] }); + assert.match(err.message, /returned error for tools\/call: \[-32001\] POSTED: part 1 of 2/); +}); + +test('a request nobody answers is no-response, and the message is unchanged', async () => { + connection = await McplServerConnection.connect(config({ requestTimeoutMs: 200 }), HOST_CAPS); + const err = await rejectsWith(connection.sendToolsCall('stuck', {}), 'no-response'); + assert.match(err.message, /did not respond to tools\/call/); +}); + +test('a connection lost while awaiting is no-response', async () => { + connection = await McplServerConnection.connect(config({ requestTimeoutMs: 10_000 }), HOST_CAPS); + await rejectsWith(connection.sendToolsCall('exit', {}), 'no-response'); +}); + +test('a request on a closed connection is not-sent', async () => { + connection = await McplServerConnection.connect(config(), HOST_CAPS); + await connection.close(); + const err = await rejectsWith(connection.sendToolsCall('fail', {}), 'not-sent'); + assert.match(err.message, /Cannot send request: connection to "outcome" is closed/); + connection = null; +}); + +test('McplRequestError is an Error that keeps its message and name', () => { + const err = new McplRequestError('boom', 'not-sent'); + assert.ok(err instanceof Error); + assert.equal(err.name, 'McplRequestError'); + assert.equal(err.message, 'boom'); +}); From 2cb501af62033293e0e375b181a7f69061f5f0b2 Mon Sep 17 00:00:00 2001 From: Ada-1017 Date: Wed, 7 Oct 2026 07:42:15 -0400 Subject: [PATCH 12/73] feat(awareness): make Discord awareness marks an explicit, one-shot choice that never gates MCPL MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Room-225's awareness-marks contract (Connectome incident: a web-UI rollback queued a 💤 reaction on each of 918 removed messages, mostly other people's ambient messages; the serial drain held the store and every MCPL data plane). Seven points, carried to nissa by Tessa (#41730) and designed with June-1016: 1. Marks are opt-in, default none. `marks: 'none' | { scope: 'addressed' | 'all', refs? }` on rollbackToMessage/suppressMessages; `marks` on host/command undo and hide (invalid values refused, never guessed); `--marks` on agent-framework-recover. `addressed` = MCPL tag chat:addressed. previewSurgeryMarks() shows per-scope counts/channels/refs; passing a scope's refs back freezes the choice (selection = refs ∩ removed, later arrivals removed locally but never marked; receipt counts unmarked and notRemoved). 2. One-shot. The ledger is rewritten as an append-only fsynced JSONL journal (recovery/discord-awareness-journal.jsonl): batch / activated / held / released / cancelled / retract / requested / op-cancelled / dispatching (write-ahead, with a dispatchId shared by ops riding one request) / outcome records. Branch switches, undo/redo and restarts derive nothing; reconcileForBranch and the API server's branch-switch syncs are removed. Startup: unanswered attempts become `unknown`; a prepared batch whose target is exactly the active branch is crash-completed; anything else prepared is held for explicit release. Suppression bodies still resume at startup (body integrity), but a held/cancelled batch's marks stay held. 3. Delivery is background and never gates: the DiscordAwarenessBarrier generation machinery, installMcplDataPlaneGate/release/complete/fail and the handlers' barrier awaits are gone; startup/connect/reconnect open planes via readyMcplPlanes (quiesce-aware) after the §5.3 grant, and kick delivery. A disconnected route keeps work queued; delivery failure is an ops alert, never a connection recycle. A journal that cannot be read still aborts startup (it carries the suppression resume record). 4. Outcomes come from McplRequestError.outcome (Petra's 023b439): not-sent stays requested; no-response is `unknown` forever (a later attempt's confirmation never resolves it); Discord errors are failed (permanent when the message/channel is gone). An unknown op is retried only while it is the newest live op for its key; one request per key on the wire at a time (including a cancelled op still awaiting its outcome). 5. Operator controls (framework methods, host/command `marks` verb, CLI --awareness): cancel stops all further sends/retries of a batch (prepared batches included; never removes); retract queues removal across batches by (route, channel, message, emoji), supersedes adds never sent, skips keys where nothing ever left the host, and discloses unresolved add requests; release queues a held batch. All operator-logged. 6. Multi-record appends are transactions (txn ids + commit record), so a torn append applies nothing; a torn final line is truncated (or a complete but unterminated record terminated) before the next append. 7. The v2 JSON ledger is imported once (deterministic ids, sha256 import-complete record): applied -> confirmed, permanent-failure -> failed, pending work held; the file is renamed .migrated-v2 (retried if needed). Receipts: SurgeryMarkerFacts (scope, unmarked, notRemoved) on every markers result; marks chosen without any journal path are `not-scheduled`. Offline recovery collects only addressing metadata + tags (never content). Tests: discord-awareness-outbox (18: selection, write-ahead, startup recovery, cancel/retract/release, retry-vs-unknown, txn/torn tail, v2 import, drain classification, suppression resume, host/command undo/marks/ hide), surgery-marks-receipt (10, real store + fake Discord MCPL server: default local, addressed, frozen refs, non-gating, not-scheduled/unresolved, cancel/retract with a timed-out add, no-journal), discord-awareness-delivery (5: startup not gated, mandatory deadline with requestTimeoutMs 0, route down then up, corrupt journal, branch switch inert), offline-recovery-branch (6). mcpl-awareness-barrier and its fixture are removed (behaviour gone). --- src/api/server.ts | 8 +- src/framework.ts | 1042 ++++++------ src/index.ts | 21 +- src/mcpl/server-connection.ts | 14 +- src/operator-log.ts | 60 +- src/recovery/discord-awareness-outbox.ts | 1434 +++++++++++++---- src/recovery/offline-branch.ts | 100 +- src/recovery/recover-cli.ts | 80 +- test/discord-awareness-delivery.test.ts | 207 +++ test/discord-awareness-outbox.test.ts | 763 ++++++--- .../awareness-barrier-mcpl-server.mjs | 276 ---- test/mcpl-awareness-barrier.test.ts | 798 --------- test/mcpl-reconnect-reregistration.test.ts | 7 +- test/offline-recovery-branch.test.ts | 65 +- test/operator-surgery.test.ts | 5 +- test/surgery-marks-receipt.test.ts | 337 ++-- 16 files changed, 2887 insertions(+), 2330 deletions(-) create mode 100644 test/discord-awareness-delivery.test.ts delete mode 100644 test/fixtures/awareness-barrier-mcpl-server.mjs delete mode 100644 test/mcpl-awareness-barrier.test.ts diff --git a/src/api/server.ts b/src/api/server.ts index 58664705b..51b7f0d9d 100644 --- a/src/api/server.ts +++ b/src/api/server.ts @@ -541,10 +541,8 @@ export class ApiServer { store.switchBranch(params.name); this.currentBranch = params.name; - - // Reversible Discord awareness markers are part of the branch projection. - // Do not acknowledge the switch until add/remove operations were attempted. - await this.framework.syncDiscordAwarenessMarkers(); + // Awareness marks are one-shot: a branch switch never adds or removes + // them (retract or release explicitly instead). // Materialize config files from the new branch const ws = this.framework.getModule('workspace'); @@ -598,7 +596,6 @@ export class ApiServer { if (result.undone) { this.currentBranch = result.toBranch!; - await this.framework.syncDiscordAwarenessMarkers(); this.broadcast('branch:switched', { from: result.fromBranch, to: result.toBranch, @@ -618,7 +615,6 @@ export class ApiServer { if (result.redone) { this.currentBranch = result.toBranch!; - await this.framework.syncDiscordAwarenessMarkers(); this.broadcast('branch:switched', { from: result.fromBranch, to: result.toBranch, diff --git a/src/framework.ts b/src/framework.ts index 9b80c3985..8ea76cbc8 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -387,7 +387,7 @@ import { isToolAllowed } from './mcpl/tool-policy.js'; import { EventGate, formatShadowWarning } from './gate/event-gate.js'; import { UsageTracker, type PersistedUsageState } from './usage/usage-tracker.js'; import type { SessionUsageSnapshot, UsageUpdatedEvent } from './usage/types.js'; -import type { McplServerConnection } from './mcpl/server-connection.js'; +import { McplRequestError, type McplServerConnection } from './mcpl/server-connection.js'; import type { McplServerConfig, McplHostCapabilities, @@ -406,14 +406,23 @@ import { DiscordAwarenessOutbox, defaultDiscordAwarenessOutboxPath, extractDiscordAwarenessRefs, + selectDiscordAwarenessRefs, type DiscordAwarenessBatch, + type DiscordAwarenessBatchView, + type DiscordAwarenessCancelReceipt, + type DiscordAwarenessMarks, + type DiscordAwarenessReleaseReceipt, + type DiscordAwarenessRetractReceipt, + type DiscordAwarenessScope, } from './recovery/discord-awareness-outbox.js'; import { OperatorActionError, OperatorLog, capIds, defaultOperatorLogPath, + type SurgeryMarkerFacts, type SurgeryMarkerReceipt, + type SurgeryMarksPreview, type OperatorLogEntry, type OperatorLogInput, type OperatorRequester, @@ -507,12 +516,6 @@ type DiscordAwarenessDrainOutcome = | { status: 'delivered'; delivered: number; failed: number } | { status: 'unavailable'; accounted: number }; -interface DiscordAwarenessBarrier { - generation: number; - requiresBarrier: boolean; - promise: Promise; -} - function normalizeDiscordAwarenessDeadline(value: number | undefined): number { if (value === undefined || !Number.isFinite(value) || value <= 0) { return DEFAULT_DISCORD_AWARENESS_DEADLINE_MS; @@ -719,8 +722,19 @@ function bodyGroupRun( }; } +/** The batch scope a publication choice records (`none` for local surgery). */ +function marksScope(marks: DiscordAwarenessMarks | undefined): 'none' | DiscordAwarenessScope { + return !marks || marks === 'none' ? 'none' : marks.scope; +} + +/** A publication choice, summarized for the operator log (refs counted, not listed). */ +function describeMarksChoice(marks: DiscordAwarenessMarks | undefined): Record | 'none' { + if (!marks || marks === 'none') return 'none'; + return { scope: marks.scope, ...(marks.refs ? { authorizedRefs: marks.refs.length } : {}) }; +} + function describeMarkers(m: SurgeryMarkerReceipt): string { - if (m.status === 'queued') return `queued:${m.queued}`; + if (m.status === 'queued') return `queued:${m.queued}(${m.scope}; unmarked ${m.unmarked})`; if (m.status === 'not-scheduled') return 'not-scheduled(batch retired)'; if (m.status === 'unresolved') return `unresolved(batch ${m.batchId} may still be delivered)`; return 'none'; @@ -883,6 +897,14 @@ interface HostCommandParams { * (regardless of participant) are no longer on the active branch. * Mutually exclusive with `turns`. */ messages?: number; + /** Message-granular undo and `hide`: awareness marks on the removed + * Discord messages, `none` (the default), `addressed` or `all`. Any other + * value is refused. */ + marks?: string; + /** For the `marks` command: `list`, `cancel`, `retract` or `release`. */ + action?: string; + /** For the `marks` command: the batch to act on (`all` for retract). */ + batchId?: string; /** For the `hide` command: Discord message id of the (first) message to * remove. With `toMessageId`, removes the inclusive range between them. */ fromMessageId?: string; @@ -1385,8 +1407,6 @@ export class AgentFramework { /** Serialize per-server drains so reconnect and an online undo cannot race. */ private discordAwarenessDrains: Map> = new Map(); /** Framework-global inference gate; older generations cannot release it. */ - private discordAwarenessBarrier: DiscordAwarenessBarrier | null = null; - private discordAwarenessBarrierGeneration = 0; // EventGate (null when FrameworkConfig.gate is omitted) private eventGate: EventGate | null = null; @@ -1565,24 +1585,24 @@ export class AgentFramework { ); framework.providerHoldHook = config.providerHold; - // If an offline recovery process crashed after switching Chronicle but - // before committing its prepared marker batch, the active branch is the - // commit record. Promote it now; no quarantined content is read. + // Startup reconciliation of the awareness journal, before any delivery. + // It derives nothing from branch ancestry: a surgery that crashed after + // switching to exactly the active branch completes its activation; any + // other prepared batch is held for an operator; an attempt the previous + // process left on the wire is recorded unknown. No content is read. if (discordAwarenessOutbox) { try { - const activated = discordAwarenessOutbox.activatePreparedForBranch( - store.currentBranch().name, - store.listBranches(), - ); - if (activated > 0) { + const recovered = discordAwarenessOutbox.recoverAtStartup(store.currentBranch().name); + if (recovered.activated.length > 0 || recovered.held.length > 0 || recovered.unknownAttempts > 0) { console.error( - `[discord-awareness] recovered ${activated} prepared batch(es) for active branch ${store.currentBranch().name}`, + `[discord-awareness] startup: activated ${recovered.activated.length} interrupted batch(es) ` + + `for ${store.currentBranch().name}, held ${recovered.held.length}, ` + + `recorded ${recovered.unknownAttempts} unanswered attempt(s) as unknown`, ); } } catch (error) { // The branch may be safe, but reporting the framework ready while its - // durable awareness projection is unreadable creates a half-ready - // host whose data plane can never be released safely. + // durable awareness journal is unreadable creates a half-ready host. throw new DiscordAwarenessAccountingError('startup reconciliation', error); } } @@ -1751,12 +1771,12 @@ export class AgentFramework { } // Restore persisted quiesce mode (issue #122) BEFORE initializeMcpl: the - // flag must be set before any data-plane barrier completion can run, or - // the startup funnel would open the data planes on a host that shut down - // mid-maintenance. Staged connections boot with both planes closed, so a - // quiesced boot needs no re-pause — completeMcplDataPlaneGate consults - // the flag and holds data planes (control planes come up normally). The - // gate already exists at this point, so the suppression is wired here too. + // flag must be set before startup opens any plane, or it would open the + // data planes on a host that shut down mid-maintenance. Staged + // connections boot with both planes closed, so a quiesced boot needs no + // re-pause — readyMcplPlanes consults the flag and holds data planes + // (control planes come up normally). The gate already exists at this + // point, so the suppression is wired here too. // // Both records live OUTSIDE branch history (recovery/ files next to the // store): a historical rollback must not be able to erase the marker of @@ -3265,26 +3285,10 @@ export class AgentFramework { await this.flushDeferredWrites('resume'); this.persistHostMode(null); } finally { - // Reopen MCPL data planes through the existing barrier funnel — NOT a - // bespoke ready() loop. The funnel inherits completeMcplDataPlaneGate's - // nested-install guard (a flushed tools-list-changed can install a - // newer barrier mid-flush), drains any awareness work accumulated - // during the window before opening, and replaces a stale failed - // barrier by identity. In `finally` so that nothing above can leave - // the host un-quiesced with every data plane still paused. - if (this.mcplServerRegistry) { - const barrier = this.installMcplDataPlaneGate(); - this.releaseMcplDataPlaneGate(barrier); - try { - await barrier.promise; - this.completeMcplDataPlaneGate(barrier); - } catch (error) { - // The host IS resumed — don't rethrow. failMcplDataPlaneGate - // recycles the connections, and their reconnect flows re-run the - // funnel with quiesced=false, self-healing the data planes. - await this.failMcplDataPlaneGate(barrier, 'quiesce resume', error); - } - } + // Reopen MCPL data planes (quiesced is already false). In `finally` + // so that nothing above can leave the host un-quiesced with every data + // plane still paused. + this.readyMcplPlanes(); this.emitTrace({ type: 'host:resume', @@ -5077,12 +5081,21 @@ export class AgentFramework { /** Discord addresses removed by message-granular undo. The durable outbox * owns eventual delivery; this is also returned for immediate surfaces. */ removedRefs?: Array<{ serverId: string; channelId: string; messageId: string }>; - /** For message-granular undo: whether awareness marks were scheduled - * (not whether Discord has accepted them). */ + /** For message-granular undo and `hide`: whether awareness marks were + * scheduled (not whether Discord has accepted them). */ markers?: SurgeryMarkerReceipt; + /** For the `marks` command: the journal's batches (`list`) or the + * action's receipt. */ + awareness?: + | DiscordAwarenessBatchView[] + | DiscordAwarenessCancelReceipt + | DiscordAwarenessRetractReceipt + | DiscordAwarenessReleaseReceipt; hidden?: number; /** For `hide`: the Discord (channelId, messageId) of each removed message - * that carried one — so the surface can mark them with a reaction. */ + * that carried one. Marking is the framework's (see `marks`/`markers`); + * a surface that also reacts itself should do so only when its operator + * chose marks and the framework returned no `markers`. */ hiddenRefs?: Array<{ channelId: string; messageId: string }>; lastVisible?: { participant?: string; role?: string; preview?: string } | null; /** For `unstick`: acknowledges the forced-rewind loop has started; the @@ -5104,13 +5117,48 @@ export class AgentFramework { params.command !== 'quiesce' && params.command !== 'resume' && params.command !== 'maintain' && - params.command !== 'host-status' + params.command !== 'host-status' && + params.command !== 'marks' ) { return { ok: false, error: `Unknown host command: ${String(params.command)}` }; } + if (params.marks !== undefined && params.marks !== 'none' && params.marks !== 'addressed' && params.marks !== 'all') { + // A publication choice is never guessed: a mistyped scope is refused + // rather than silently treated as none (or as anything else). + return { ok: false, error: `marks must be none, addressed or all (got ${JSON.stringify(params.marks)})` }; + } // Host-scoped verbs (issue #122) — no agent resolution. const requester = params.requesterName ?? params.requesterId ?? `mcpl:${serverId}`; + if (params.command === 'marks') { + // The awareness journal's operator controls, for surfaces that act + // through host/command (the web UI calls the framework methods). + const by = { requester: hostCommandRequester(serverId, params) }; + const batchId = params.batchId; + try { + switch (params.action) { + case 'list': + return { ok: true, awareness: this.listDiscordAwareness() }; + case 'cancel': + if (!batchId) return { ok: false, error: 'marks cancel needs batchId' }; + return { ok: true, awareness: this.cancelDiscordAwareness(batchId, by) }; + case 'retract': + if (!batchId) return { ok: false, error: 'marks retract needs batchId (or all)' }; + return { ok: true, awareness: this.retractDiscordAwareness(batchId, by) }; + case 'release': + if (!batchId) return { ok: false, error: 'marks release needs batchId' }; + return { ok: true, awareness: this.releaseDiscordAwareness(batchId, by) }; + default: + return { ok: false, error: 'marks needs action list, cancel, retract or release' }; + } + } catch (error) { + return { + ok: false, + error: error instanceof Error ? error.message : String(error), + ...(error instanceof OperatorActionError ? { code: error.code } : {}), + }; + } + } if (params.command === 'quiesce') { // quiesce() clamps timeoutMs to [1s, 10m] itself — one clamp for every ingress. console.error(`[host-command] quiesce by=${requester} (server=${serverId})`); @@ -5204,6 +5252,9 @@ export class AgentFramework { if (agent.state.status !== 'idle') { return { ok: false, error: `Cannot hide while agent is ${agent.state.status}` }; } + const hideMarks: DiscordAwarenessMarks = params.marks === 'addressed' || params.marks === 'all' + ? { scope: params.marks } + : 'none'; if (!params.fromMessageId) { return { ok: false, error: 'hide: fromMessageId is required' }; } @@ -5245,6 +5296,7 @@ export class AgentFramework { const refs = refsIn(lo, hi); const rangeIds = all.slice(lo, hi + 1).map((m) => String(m.id)); cm.removeMessages(all[lo].id, all[hi].id); + const markers = this.scheduleHideMarks(agentName, serverId, all.slice(lo, hi + 1), hideMarks); console.error( `[host-command] hide agent=${agentName} range removed=${hi - lo + 1} ` + `(${params.fromMessageId}..${params.toMessageId}) by=${params.requesterName ?? params.requesterId ?? 'unknown'} (server=${serverId})`, @@ -5254,24 +5306,31 @@ export class AgentFramework { kind: 'hide', agent: agentName, requester: hostCommandRequester(serverId, params), - params: { fromMessageId: params.fromMessageId, toMessageId: params.toMessageId }, + params: { + fromMessageId: params.fromMessageId, + toMessageId: params.toMessageId, + marks: describeMarksChoice(hideMarks), + }, result: { branch: this.store.currentBranch().name, hidden: hi - lo + 1, removedIds: loggedRange.ids, ...(loggedRange.truncated ? { removedIdsTruncated: true } : {}), + markers, }, }); return { ok: true, hidden: hi - lo + 1, hiddenRefs: refs, + markers, lastVisible: await this.lastVisiblePreview(agentName), }; } const refs = refsIn(fromIdx, fromIdx); const hiddenId = String(all[fromIdx].id); cm.removeMessage(all[fromIdx].id); + const markers = this.scheduleHideMarks(agentName, serverId, [all[fromIdx]], hideMarks); console.error( `[host-command] hide agent=${agentName} removed=1 (${params.fromMessageId}) ` + `by=${params.requesterName ?? params.requesterId ?? 'unknown'} (server=${serverId})`, @@ -5280,13 +5339,14 @@ export class AgentFramework { kind: 'hide', agent: agentName, requester: hostCommandRequester(serverId, params), - params: { fromMessageId: params.fromMessageId }, - result: { branch: this.store.currentBranch().name, hidden: 1, removedIds: [hiddenId] }, + params: { fromMessageId: params.fromMessageId, marks: describeMarksChoice(hideMarks) }, + result: { branch: this.store.currentBranch().name, hidden: 1, removedIds: [hiddenId], markers }, }); return { ok: true, hidden: 1, hiddenRefs: refs, + markers, lastVisible: await this.lastVisiblePreview(agentName), }; } catch (error) { @@ -5318,6 +5378,11 @@ export class AgentFramework { messageId: String(target.id), branchName: `undo-msgs/${agentName}/${Date.now()}`, requester: hostCommandRequester(serverId, params), + // An immediate act: the choice and its application are the same + // moment, so the scope is evaluated now (no frozen refs needed). + marks: params.marks === 'addressed' || params.marks === 'all' + ? { scope: params.marks } + : 'none', }); return { ok: true, @@ -5360,8 +5425,8 @@ export class AgentFramework { return { ok: true, undone: 0, requested, lastVisible: null }; } - await this.syncDiscordAwarenessMarkers(); - + // Turn undo is a branch move: awareness marks are one-shot and never + // follow branch state, so there is nothing to reconcile here. return { ok: true, undone, requested, lastVisible: await this.lastVisiblePreview(agentName) }; } @@ -5952,14 +6017,17 @@ export class AgentFramework { * Roll the active branch back so `messageId` becomes its tail: fork the * chronicle at that message (origin-sequence time-travel branch) and switch * to the fork. Everything after the message stays on the source branch. - * Discord messages that left the live context get awareness markers via the - * durable outbox, exactly as message-granular `undo` does. + * + * The rollback is local to the resident. Marking removed Discord messages + * with an awareness reaction is a separate, explicit publication choice + * (`marks`, default `none`); `previewSurgeryMarks` shows what each scope + * would cover, and passing its `refs` binds the choice to exactly that set. * * Returns once the switch has landed: the body is applied, with a * marker-scheduling receipt (`markers`). It does not wait for Discord to - * accept the reactions: delivery is started, behind the MCPL data-plane - * gate, and continues without the caller or the store reservation waiting - * on it; its per-message outcomes stay in the outbox ledger. + * accept the reactions: delivery is started and continues without the + * caller, the store reservation or MCPL traffic waiting on it; its + * per-request outcomes stay in the awareness journal. * * Throws `OperatorActionError` (`agent-busy`, `unknown-message`, …) — the * agent must be idle; nothing is queued. @@ -5972,6 +6040,8 @@ export class AgentFramework { note?: string; /** Branch name for the fork (default `rollback//`). */ branchName?: string; + /** Publication choice for awareness marks; default `none`. */ + marks?: DiscordAwarenessMarks; }, ): Promise<{ agentName: string; @@ -5992,29 +6062,12 @@ export class AgentFramework { agent: agentName, ...(opts.requester ? { requester: opts.requester } : {}), ...(opts.note ? { note: opts.note } : {}), - params: { messageId: opts.messageId }, + params: { messageId: opts.messageId, marks: describeMarksChoice(opts.marks) }, }; try { - const agent = this.agents.get(agentName); - if (!agent) throw new OperatorActionError('unknown-agent', `Unknown agent: ${agentName}`); - const cm = agent.getContextManager(); - const total = cm.getMessageCount(); - const requestedIndex = locateMessageIndex(cm, opts.messageId); - if (requestedIndex < 0) { - throw new OperatorActionError('unknown-message', `Message ${opts.messageId} is not on the active branch`); - } - // Never bisect a body group: chronicle refuses that for removals - // (byte-faithful reassembly needs the whole run) and branchAt has no - // such guard of its own. Snap to the run's last shard. - const run = bodyGroupRun(cm, requestedIndex); - const targetIndex = run.to; - const tailMessageId = run.toId; - const messagesRemoved = total - targetIndex - 1; - if (messagesRemoved <= 0) { - throw new OperatorActionError('invalid', `Message ${opts.messageId} is already the tail of the active branch`); - } - const discarded = cm.getMessageWindow(targetIndex + 1, messagesRemoved, { resolveBlobs: false }).messages; + const { cm, tailMessageId, messagesRemoved, discarded } = this.planRollback(agentName, opts.messageId); const removedRefs = extractDiscordAwarenessRefs(discarded); + const selection = selectDiscordAwarenessRefs(discarded, opts.marks ?? 'none'); const sourceBranch = this.store.currentBranch().name; const targetBranch = opts.branchName ?? `rollback/${agentName}/${Date.now()}`; if (targetBranch === sourceBranch) { @@ -6038,7 +6091,10 @@ export class AgentFramework { agentName, sourceBranch, targetBranch, - refs: removedRefs, + refs: selection.refs, + scope: marksScope(opts.marks), + unmarked: selection.unmarked, + notRemoved: selection.notRemoved, emoji: this.discordAwarenessEmoji, }) ?? null; @@ -6061,10 +6117,13 @@ export class AgentFramework { // The body change has landed. Nothing after this point turns the // rollback into a failure: marks are an outbox outcome, reported // apart from the body (`markers`). - markers = this.activateSurgeryMarkers(markerBatch, 'rollback', agentName); - // Installs the MCPL data-plane gate synchronously, before the store is - // released, and starts delivery without awaiting it (the first - // reaction request may already be on the wire when this returns). + markers = this.activateSurgeryMarkers(markerBatch, 'rollback', agentName, { + scope: marksScope(opts.marks) as SurgeryMarkerReceipt['scope'], + unmarked: selection.unmarked, + notRemoved: selection.notRemoved, + }, selection.refs.length); + // Starts delivery without awaiting it (the first reaction request may + // already be on the wire when this returns); it never gates traffic. this.deliverDiscordAwarenessInBackground('rollback', agentName); this.materializeConfigMountAfterBranchSwitch(); } finally { @@ -6127,6 +6186,8 @@ export class AgentFramework { note?: string; /** Branch name for the fork (default `suppress//`). */ branchName?: string; + /** Publication choice for awareness marks; default `none`. */ + marks?: DiscordAwarenessMarks; }, ): Promise<{ agentName: string; @@ -6145,43 +6206,13 @@ export class AgentFramework { agent: agentName, ...(opts.requester ? { requester: opts.requester } : {}), ...(opts.note ? { note: opts.note } : {}), - params: { messageIds: requestedIds }, + params: { messageIds: requestedIds, marks: describeMarksChoice(opts.marks) }, }; try { - const agent = this.agents.get(agentName); - if (!agent) throw new OperatorActionError('unknown-agent', `Unknown agent: ${agentName}`); - if (requestedIds.length === 0) throw new OperatorActionError('invalid', 'No message ids given'); - const cm = agent.getContextManager(); - const total = cm.getMessageCount(); - - const located = locateMessageIndices(cm, new Set(requestedIds)); - const missing = requestedIds.filter((id) => !located.has(id)); - if (missing.length > 0) { - throw new OperatorActionError( - 'unknown-message', - `Not on the active branch: ${missing.slice(0, 5).join(', ')}${missing.length > 5 ? ` (+${missing.length - 5})` : ''}`, - ); - } - - // Build removal intervals (index-ordered, newest first). A shard expands - // to its whole body group — chronicle refuses to bisect one — and a - // group interval is always removed as a RANGE: `removeMessage` refuses - // any sharded message even when the group is down to one shard. - const intervals = new Map>(); - for (const id of requestedIds) { - const run = bodyGroupRun(cm, located.get(id)!.index); - intervals.set(`${run.from}-${run.to}`, run); - } - const ordered = [...intervals.values()].sort((a, b) => b.from - a.from); - const messagesRemoved = ordered.reduce((n, iv) => n + (iv.to - iv.from + 1), 0); - if (messagesRemoved >= total) { - throw new OperatorActionError('invalid', `Cannot suppress all ${total} message(s) — at least one must remain`); - } - const targeted = ordered.flatMap((iv) => - cm.getMessageWindow(iv.from, iv.to - iv.from + 1, { resolveBlobs: false }).messages, - ); + const { cm, ordered, messagesRemoved, targeted } = this.planSuppression(agentName, requestedIds); const removedIds = targeted.map((m) => String(m.id)); const removedRefs = extractDiscordAwarenessRefs(targeted); + const selection = selectDiscordAwarenessRefs(targeted, opts.marks ?? 'none'); const sourceBranch = this.store.currentBranch().name; const targetBranch = opts.branchName ?? `suppress/${agentName}/${Date.now()}`; @@ -6200,7 +6231,10 @@ export class AgentFramework { agentName, sourceBranch, targetBranch, - refs: removedRefs, + refs: selection.refs, + scope: marksScope(opts.marks), + unmarked: selection.unmarked, + notRemoved: selection.notRemoved, emoji: this.discordAwarenessEmoji, // Seeing targetBranch active does not prove the removals finished; // only this operation activates the batch, after the last redaction. @@ -6208,10 +6242,9 @@ export class AgentFramework { suppressionIntervals: ordered.map((iv) => ({ fromId: iv.fromId, toId: iv.toId })), }) ?? null; // Anything short of activation retires the batch: a prepared explicit - // batch re-arms through preparedSuppressionsForBranch the moment its - // target branch (or a descendant) becomes active — e.g. an operator - // opening the failed fork to look — and an un-completable resume - // there aborts framework start. + // batch is resumed at startup whenever its exact target branch is + // active — e.g. after an operator opens the failed fork to look — and + // an un-completable resume there aborts framework start. const retireBatch = (): void => { if (markerBatch) this.discordAwarenessOutbox!.discard(markerBatch.id); }; @@ -6253,10 +6286,13 @@ export class AgentFramework { // stands and the receipt says the marks were not scheduled. (No // interval needs resuming any more, so retiring the batch there // loses nothing.) - markers = this.activateSurgeryMarkers(markerBatch, 'suppress', agentName); - // Installs the MCPL data-plane gate synchronously, before the store is - // released, and starts delivery without awaiting it (the first - // reaction request may already be on the wire when this returns). + markers = this.activateSurgeryMarkers(markerBatch, 'suppress', agentName, { + scope: marksScope(opts.marks) as SurgeryMarkerReceipt['scope'], + unmarked: selection.unmarked, + notRemoved: selection.notRemoved, + }, selection.refs.length); + // Starts delivery without awaiting it (the first reaction request may + // already be on the wire when this returns); it never gates traffic. this.deliverDiscordAwarenessInBackground('suppress', agentName); this.materializeConfigMountAfterBranchSwitch(); } finally { @@ -6296,6 +6332,177 @@ export class AgentFramework { } } + /** + * The messages a rollback to `messageId` would remove, validated, without + * changing anything. Never bisects a body group: chronicle refuses that + * for removals and branchAt has no such guard, so the target snaps to the + * group's last shard. + */ + private planRollback(agentName: string, messageId: string) { + const agent = this.agents.get(agentName); + if (!agent) throw new OperatorActionError('unknown-agent', `Unknown agent: ${agentName}`); + const cm = agent.getContextManager(); + const total = cm.getMessageCount(); + const requestedIndex = locateMessageIndex(cm, messageId); + if (requestedIndex < 0) { + throw new OperatorActionError('unknown-message', `Message ${messageId} is not on the active branch`); + } + const run = bodyGroupRun(cm, requestedIndex); + const targetIndex = run.to; + const tailMessageId = run.toId; + const messagesRemoved = total - targetIndex - 1; + if (messagesRemoved <= 0) { + throw new OperatorActionError('invalid', `Message ${messageId} is already the tail of the active branch`); + } + const discarded = cm.getMessageWindow(targetIndex + 1, messagesRemoved, { resolveBlobs: false }).messages; + return { cm, tailMessageId, messagesRemoved, discarded }; + } + + /** + * The removal intervals a suppression of `requestedIds` would apply + * (index-ordered, newest first), validated, without changing anything. A + * shard expands to its whole body group (chronicle refuses to bisect one), + * and a group interval is always removed as a range. + */ + private planSuppression(agentName: string, requestedIds: string[]) { + const agent = this.agents.get(agentName); + if (!agent) throw new OperatorActionError('unknown-agent', `Unknown agent: ${agentName}`); + if (requestedIds.length === 0) throw new OperatorActionError('invalid', 'No message ids given'); + const cm = agent.getContextManager(); + const total = cm.getMessageCount(); + const located = locateMessageIndices(cm, new Set(requestedIds)); + const missing = requestedIds.filter((id) => !located.has(id)); + if (missing.length > 0) { + throw new OperatorActionError( + 'unknown-message', + `Not on the active branch: ${missing.slice(0, 5).join(', ')}${missing.length > 5 ? ` (+${missing.length - 5})` : ''}`, + ); + } + const intervals = new Map>(); + for (const id of requestedIds) { + const run = bodyGroupRun(cm, located.get(id)!.index); + intervals.set(`${run.from}-${run.to}`, run); + } + const ordered = [...intervals.values()].sort((a, b) => b.from - a.from); + const messagesRemoved = ordered.reduce((n, iv) => n + (iv.to - iv.from + 1), 0); + if (messagesRemoved >= total) { + throw new OperatorActionError('invalid', `Cannot suppress all ${total} message(s) — at least one must remain`); + } + const targeted = ordered.flatMap((iv) => + cm.getMessageWindow(iv.from, iv.to - iv.from + 1, { resolveBlobs: false }).messages, + ); + return { cm, ordered, messagesRemoved, targeted }; + } + + /** + * What a rollback or suppression would remove, and which Discord messages + * each publication scope would mark, without changing anything or reading + * message content. A surface shows this before an operator chooses + * `marks`; passing a scope's `refs` back as `marks.refs` binds the choice + * to exactly that set, so a later application never widens it. + */ + previewSurgeryMarks( + agentName: string, + target: { rollbackTo: string } | { suppress: string[] }, + ): SurgeryMarksPreview { + const messages = 'rollbackTo' in target + ? this.planRollback(agentName, target.rollbackTo).discarded + : this.planSuppression(agentName, [...new Set(target.suppress.map(String))]).targeted; + const scopeView = (scope: DiscordAwarenessScope): SurgeryMarksPreview['scopes'][DiscordAwarenessScope] => { + const { refs } = selectDiscordAwarenessRefs(messages, { scope }); + const channels = new Map(); + for (const ref of refs) channels.set(ref.channelId, (channels.get(ref.channelId) ?? 0) + 1); + return { + count: refs.length, + channels: [...channels].map(([channelId, count]) => ({ channelId, count })), + refs, + }; + }; + return { + messagesRemoved: messages.length, + addressable: extractDiscordAwarenessRefs(messages).length, + emoji: this.discordAwarenessEmoji, + scopes: { addressed: scopeView('addressed'), all: scopeView('all') }, + }; + } + + /** The awareness journal's batches, for operator surfaces. */ + listDiscordAwareness(): DiscordAwarenessBatchView[] { + return this.discordAwarenessOutbox?.view() ?? []; + } + + /** + * Stop a batch's awareness marks that have not been sent. Never removes a + * reaction: requests already on the wire, or whose outcome is unknown, are + * reported and may still land; retract explicitly to remove them. + */ + cancelDiscordAwareness( + batchId: string, + opts: { requester?: OperatorRequester; note?: string } = {}, + ): DiscordAwarenessCancelReceipt { + return this.awarenessOperatorAction('awareness-cancel', { batchId }, opts, () => + this.requireDiscordAwarenessOutbox().cancel(batchId, describeRequester(opts.requester))); + } + + /** + * Remove this bot's awareness mark, through each ref's configured MCPL + * route, from a batch's refs, or from every ref any batch sent an add for + * (`all`). The receipt says how many removals were queued and how many + * earlier add attempts are unresolved, any of which may land afterwards. + */ + retractDiscordAwareness( + target: string | 'all', + opts: { requester?: OperatorRequester; note?: string } = {}, + ): DiscordAwarenessRetractReceipt { + const receipt = this.awarenessOperatorAction('awareness-retract', { target }, opts, () => + this.requireDiscordAwarenessOutbox().retract(target, describeRequester(opts.requester))); + this.deliverDiscordAwarenessInBackground('retract', 'operator'); + return receipt; + } + + /** Release a held batch's recorded operations, explicitly. */ + releaseDiscordAwareness( + batchId: string, + opts: { requester?: OperatorRequester; note?: string } = {}, + ): DiscordAwarenessReleaseReceipt { + const receipt = this.awarenessOperatorAction('awareness-release', { batchId }, opts, () => + this.requireDiscordAwarenessOutbox().release(batchId, describeRequester(opts.requester))); + this.deliverDiscordAwarenessInBackground('release', 'operator'); + return receipt; + } + + private requireDiscordAwarenessOutbox(): DiscordAwarenessOutbox { + if (!this.discordAwarenessOutbox) { + throw new OperatorActionError('invalid', 'This framework has no Discord awareness journal (no storePath)'); + } + return this.discordAwarenessOutbox; + } + + private awarenessOperatorAction( + kind: string, + params: Record, + opts: { requester?: OperatorRequester; note?: string }, + act: () => T, + ): T { + const logBase: OperatorLogInput = { + kind, + agent: '*', + ...(opts.requester ? { requester: opts.requester } : {}), + ...(opts.note ? { note: opts.note } : {}), + params, + }; + try { + const result = act(); + this.recordOperatorAction({ ...logBase, result: { ...result } as Record }); + console.error(`[discord-awareness] ${kind} ${JSON.stringify(params)} by=${describeRequester(opts.requester)}: ${JSON.stringify(result)}`); + return result; + } catch (error) { + this.recordOperatorAction({ ...logBase, error: error instanceof Error ? error.message : String(error) }); + if (error instanceof OperatorActionError) throw error; + throw new OperatorActionError('invalid', error instanceof Error ? error.message : String(error), { cause: error }); + } + } + /** * Record a surgery's prepared marker batch as active once its body change * has landed, and say what happened. Never throws: the body is already @@ -6311,10 +6518,27 @@ export class AgentFramework { */ private activateSurgeryMarkers( batch: DiscordAwarenessBatch | null, - verb: 'rollback' | 'suppress', + verb: 'rollback' | 'suppress' | 'hide', agentName: string, + facts: SurgeryMarkerFacts, + selected: number, ): SurgeryMarkerReceipt { - if (!batch) return { status: 'none', queued: 0 }; + if (!batch) { + if (selected > 0 && !this.discordAwarenessOutbox) { + // Chosen and in scope, but there is nowhere to record them: say so + // rather than report that none were wanted. + console.error( + `[discord-awareness] ${verb} agent=${agentName}: ${selected} mark(s) chosen, but no awareness journal is configured`, + ); + return { + ...facts, + status: 'not-scheduled', + queued: 0, + error: 'no awareness journal is configured (no storePath or discordAwarenessOutboxPath)', + }; + } + return { ...facts, status: 'none', queued: 0 }; + } try { this.discordAwarenessOutbox!.activate(batch.id); } catch (error) { @@ -6331,7 +6555,7 @@ export class AgentFramework { `could not be activated (${detail}); ` + (retired ? 'batch retired' : `batch left prepared${retireDetail ? ` (retire failed: ${retireDetail})` : ''}`), ); - if (batch.refs.length === 0) return { status: 'none', queued: 0 }; + if (batch.refs.length === 0) return { ...facts, status: 'none', queued: 0 }; this.opsAlert( retired ? 'discord-awareness-not-scheduled' : 'discord-awareness-unresolved', agentName, @@ -6343,33 +6567,83 @@ export class AgentFramework { { data: { batchId: batch.id, retired } }, ); return retired - ? { status: 'not-scheduled', queued: 0, error: detail } - : { status: 'unresolved', queued: 0, batchId: batch.id, error: detail }; + ? { ...facts, status: 'not-scheduled', queued: 0, error: detail } + : { ...facts, status: 'unresolved', queued: 0, batchId: batch.id, error: detail }; } - if (batch.refs.length === 0) return { status: 'none', queued: 0 }; - return { status: 'queued', queued: batch.refs.length, batchId: batch.id }; + if (batch.refs.length === 0) return { ...facts, status: 'none', queued: 0 }; + return { ...facts, status: 'queued', queued: batch.refs.length, batchId: batch.id }; } /** - * Start awareness delivery without the caller awaiting it. - * `syncDiscordAwarenessMarkers` installs the MCPL data-plane gate - * synchronously, before its first await, so calling this while a surgery - * still holds the store keeps MCPL-borne wakes behind the gate exactly as - * before. The drain starts in the same synchronous run (the first reaction - * request can be written before this returns) and continues independently; - * only the surgery's response and its store reservation stop waiting on - * remote reactions. A failure is accounted by the gate itself (connections - * recycled); here it is logged and raised as an ops alert. + * Marks for an in-place `hide`, scheduled after its redaction committed. + * The batch targets the current branch and is explicit (activated here); + * a crash between its two journal records leaves it prepared for exactly + * this branch, which startup completes. Old records without a serverId are + * routed through the server that issued the command. */ - private deliverDiscordAwarenessInBackground(verb: 'rollback' | 'suppress', agentName: string): void { + private scheduleHideMarks( + agentName: string, + serverId: string, + removed: Array<{ metadata?: Record }>, + marks: DiscordAwarenessMarks, + ): SurgeryMarkerReceipt { + const carriers = removed.map((message) => ({ + metadata: { ...(message.metadata ?? {}), serverId: message.metadata?.serverId ?? serverId }, + })); + const selection = selectDiscordAwarenessRefs(carriers, marks); + const facts: SurgeryMarkerFacts = { + scope: marksScope(marks) as SurgeryMarkerReceipt['scope'], + unmarked: selection.unmarked, + notRemoved: selection.notRemoved, + }; + let batch: DiscordAwarenessBatch | null; + try { + const branch = this.store.currentBranch().name; + batch = this.discordAwarenessOutbox?.prepare({ + agentName, + sourceBranch: branch, + targetBranch: branch, + refs: selection.refs, + scope: marksScope(marks), + unmarked: selection.unmarked, + notRemoved: selection.notRemoved, + emoji: this.discordAwarenessEmoji, + activationPolicy: 'explicit', + }) ?? null; + } catch (error) { + // The redaction stands; nothing was recorded, so nothing will be sent. + const detail = error instanceof Error ? error.message : String(error); + this.opsAlert( + 'discord-awareness-not-scheduled', + agentName, + `hide applied, but its ${selection.refs.length} awareness mark(s) were not scheduled: ${detail}`, + ); + return { ...facts, status: 'not-scheduled', queued: 0, error: detail }; + } + const receipt = this.activateSurgeryMarkers(batch, 'hide', agentName, facts, selection.refs.length); + if (receipt.status === 'queued') this.deliverDiscordAwarenessInBackground('hide', agentName); + return receipt; + } + + /** + * Start awareness delivery without the caller awaiting it. The drain starts + * in the same synchronous run (the first request can be written before + * this returns) and continues independently. Delivery never gates MCPL + * traffic or turns; a failure is logged and raised as an ops alert, and + * the queued work stays in the journal for the next pass. + */ + private deliverDiscordAwarenessInBackground( + occasion: 'rollback' | 'suppress' | 'hide' | 'retract' | 'release' | 'startup' | 'connect' | 'list-change' | 'reconnect', + agentName: string, + ): void { if (!this.discordAwarenessOutbox) return; this.syncDiscordAwarenessMarkers().catch((error) => { const detail = error instanceof Error ? error.message : String(error); - console.error(`[discord-awareness] delivery after ${verb} agent=${agentName} failed: ${detail}`); + console.error(`[discord-awareness] delivery after ${occasion} failed: ${detail}`); this.opsAlert( 'discord-awareness-delivery', agentName, - `Awareness mark delivery after ${verb} failed: ${detail}`, + `Awareness mark delivery after ${occasion} failed: ${detail}`, ); }); } @@ -13251,16 +13525,11 @@ export class AgentFramework { for (const config of serverConfigs) { try { - // Stage every startup connection with both planes closed. This removes - // server-order dependence: no early heartbeat/shell push can become - // model-visible before a later Discord connection has joined the same - // awareness generation. + // Stage every startup connection with both planes closed, and open + // them together once all are staged, so startup does not depend on + // server order. await this.connectMcplServerInternal(config, true); } catch (error) { - if (error instanceof DiscordAwarenessAccountingError) { - await this.mcplServerRegistry.closeAll(); - throw error; - } // Fail-open: log and continue with remaining servers const err = error instanceof Error ? error : new Error(String(error)); console.error(`Failed to connect MCPL server "${config.id}":`, err.message); @@ -13278,48 +13547,34 @@ export class AgentFramework { } } - // Start the durable drain only after all startup connections are staged. - // The awareness tools/call is written before control-plane registration is - // flushed, preserving the server-side queue -> registration -> call order. - const startupBarrier = this.installMcplDataPlaneGate(); - this.releaseMcplDataPlaneGate(startupBarrier); - try { - await startupBarrier.promise; - this.completeMcplDataPlaneGate(startupBarrier); - } catch (error) { - await this.mcplServerRegistry.closeAll(); - throw error; - } + // Every startup connection is staged with its grant established: open + // them together (a quiesced boot keeps data planes held). Awareness marks + // never gate this: queued marks are delivered in the background. + this.readyMcplPlanes(); + this.deliverDiscordAwarenessInBackground('startup', 'host'); // Discover tools from all connected servers await this.refreshMcplTools(); } /** - * Reconcile the durable ledger with Chronicle, then deliver every server's - * work, joining any drain still running. + * Deliver every server's queued awareness operations now and wait for this + * pass. Delivery never gates MCPL traffic or turns: callers that do not + * need to wait use the background form. A server that is not connected is + * skipped (its work stays queued until it connects). */ async syncDiscordAwarenessMarkers(_onlyServerId?: string): Promise { if (!this.discordAwarenessOutbox) return; - // A targeted retry may discover pending work for other Discord servers - // during reconciliation. Safety is global, so one generation accounts for - // all pending operations before releasing any MCPL data plane. - const barrier = this.installMcplDataPlaneGate(); - this.releaseMcplDataPlaneGate(barrier); - try { - await barrier.promise; - this.completeMcplDataPlaneGate(barrier); - } catch (error) { - await this.failMcplDataPlaneGate(barrier, 'explicit synchronization', error); - throw error; - } + const serverIds = [...new Set( + this.readDiscordAwarenessDispatches().map((dispatch) => dispatch.key.serverId), + )]; + await Promise.all(serverIds.map((serverId) => this.drainDiscordAwarenessOutbox(serverId))); } private async resumePreparedDiscordSuppressions(): Promise { if (!this.discordAwarenessOutbox) return; const batches = this.discordAwarenessOutbox.preparedSuppressionsForBranch( this.store.currentBranch().name, - this.store.listBranches(), ); for (const batch of batches) { const agent = this.agents.get(batch.agentName); @@ -13342,15 +13597,28 @@ export class AgentFramework { if (interval.fromId === interval.toId) cm.removeMessage(from.id); else cm.removeMessages(from.id, to.id); } - this.discordAwarenessOutbox.activate(batch.id); - console.error(`[discord-awareness] resumed suppression batch ${batch.id}`); + if (batch.status === 'held' || batch.cancelled) { + // Body integrity only: its marks were held at an earlier startup (and + // are released explicitly, never by a branch becoming active), or an + // operator cancelled them. + this.discordAwarenessOutbox.recordSuppressionComplete(batch.id); + console.error(`[discord-awareness] completed held suppression ${batch.id} (marks stay held)`); + } else { + this.discordAwarenessOutbox.activate(batch.id); + console.error(`[discord-awareness] resumed suppression batch ${batch.id}`); + } } } /** - * Deliver every due ref independently. Permanent Discord failures are kept - * in the ledger for audit but do not block later operations; retryable - * failures remain pending for the next reconnect/list-change attempt. + * Deliver one server's due awareness operations, serially, in request + * order per reaction key. Every dispatch is written ahead to the journal + * before the request can leave, and its outcome is recorded from what the + * connection can actually know (McplRequestError.outcome): a request that + * was never written stays queued, one written and never answered is + * `unknown` for good, and a Discord answer is definitive (permanent when + * Discord says the message or channel is gone or inaccessible). A server + * that is not connected is skipped: its work simply stays queued. */ private drainDiscordAwarenessOutbox( serverId: string, @@ -13362,84 +13630,99 @@ export class AgentFramework { if (existing) return existing; const drain = (async () => { + const outbox = this.discordAwarenessOutbox!; const connection = this.mcplServerRegistry?.getServer(serverId); if (!connection?.isConnected) { - const operations = this.readDiscordAwarenessPending(serverId); - for (const operation of operations) { - this.writeDiscordAwarenessFailure( - operation.batchId, - operation.ref, - operation.action, - 'Awareness delivery not attempted: MCPL connection unavailable', - false, - ); - } - if (operations.length > 0) { + const waiting = this.readDiscordAwarenessDispatches(serverId).length; + if (waiting > 0) { console.error( - `[discord-awareness] ${serverId}: connection unavailable; ` + - `durably deferred=${operations.length}`, + `[discord-awareness] ${serverId}: connection unavailable; ${waiting} operation(s) stay queued`, ); } - return { status: 'unavailable' as const, accounted: operations.length }; + return { status: 'unavailable' as const, accounted: waiting }; } let delivered = 0; let failed = 0; const attempted = new Set(); + let announced = false; while (true) { - const operations = this.readDiscordAwarenessPending(serverId).filter((operation) => { - const key = `${operation.batchId}\0${operation.ref.channelId}\0${operation.ref.messageId}\0${operation.action}`; + const dispatches = this.readDiscordAwarenessDispatches(serverId).filter((dispatch) => { + const key = `${dispatch.opIds.join(',')}\0${dispatch.action}`; if (attempted.has(key)) return false; attempted.add(key); return true; }); - if (operations.length === 0) break; - for (const operation of operations) { - const ref = operation.ref; + if (dispatches.length === 0) break; + if (!announced) { + announced = true; + console.error( + `[discord-awareness] ${serverId}: delivering ${dispatches.length} queued mark operation(s)`, + ); + } + for (const dispatch of dispatches) { + const ref = dispatch.key; const channelId = ref.channelId.startsWith('discord:') ? ref.channelId.split(':').at(-1)! : ref.channelId; - let deliveryError: string | undefined; + const tool = dispatch.action === 'add' ? 'add_reaction' : 'remove_reaction'; + let attempts: Array<{ opId: string; attempt: number }>; + try { + attempts = outbox.recordDispatching(dispatch); + } catch (error) { + throw new DiscordAwarenessAccountingError('dispatch write-ahead', error); + } + let outcome: 'confirmed' | 'failed' | 'not-sent' | 'unknown'; + let detail: string | undefined; try { - const tool = operation.action === 'add' ? 'add_reaction' : 'remove_reaction'; const result = await connection.sendToolsCallWithDeadline(tool, { channelId, messageId: ref.messageId, - emoji: operation.emoji, + emoji: ref.emoji, }, this.discordAwarenessDeadlineMs); if (result.isError) { - deliveryError = result.content + outcome = 'failed'; + detail = result.content .map((content) => content.text ?? '') .filter(Boolean) .join('; ') || `Discord ${tool} returned an error`; + } else { + outcome = 'confirmed'; } } catch (error) { - deliveryError = error instanceof Error ? error.message : String(error); + detail = error instanceof Error ? error.message : String(error); + if (error instanceof McplRequestError) { + outcome = error.outcome === 'not-sent' + ? 'not-sent' + : error.outcome === 'error-response' ? 'failed' : 'unknown'; + } else { + // Not classified by the connection: it may have been written. + outcome = 'unknown'; + } } - - if (deliveryError !== undefined) { - const permanent = isPermanentDiscordReactionFailure(deliveryError); - this.writeDiscordAwarenessFailure( - operation.batchId, - ref, - operation.action, - deliveryError, - permanent, - ); + const permanent = outcome === 'failed' && isPermanentDiscordReactionFailure(detail ?? ''); + try { + outbox.recordOutcome(attempts, outcome, { + ...(permanent ? { permanent: true } : {}), + ...(detail !== undefined ? { error: detail } : {}), + }); + } catch (error) { + throw new DiscordAwarenessAccountingError('outcome write', error); + } + if (outcome === 'confirmed') { + delivered++; + } else { failed++; console.error( - `[discord-awareness] ${operation.action} failed for ${ref.channelId}/${ref.messageId}` + - ` (${permanent ? 'permanent' : 'retryable'}): ${deliveryError}`, + `[discord-awareness] ${dispatch.action} ${ref.channelId}/${ref.messageId}: ${outcome}` + + `${outcome === 'failed' ? (permanent ? ' (permanent)' : ' (retryable)') : ''}: ${detail ?? ''}`, ); - } else { - this.writeDiscordAwarenessSuccess(operation.batchId, ref, operation.action); - delivered++; } } } if (delivered > 0 || failed > 0) { console.error( - `[discord-awareness] ${serverId}: delivered=${delivered} failed=${failed}`, + `[discord-awareness] ${serverId}: confirmed=${delivered} not-confirmed=${failed}`, ); } return { status: 'delivered' as const, delivered, failed }; @@ -13451,206 +13734,29 @@ export class AgentFramework { return drain; } - private readDiscordAwarenessPending(serverId?: string) { + private readDiscordAwarenessDispatches(serverId?: string) { try { - return this.discordAwarenessOutbox!.pending(serverId); + return this.discordAwarenessOutbox!.pendingDispatches(serverId); } catch (error) { throw new DiscordAwarenessAccountingError( - `ledger read${serverId ? ` for ${serverId}` : ''}`, + `journal read${serverId ? ` for ${serverId}` : ''}`, error, ); } } - private writeDiscordAwarenessSuccess( - batchId: string, - ref: import('./recovery/discord-awareness-outbox.js').DiscordAwarenessRef, - action: import('./recovery/discord-awareness-outbox.js').DiscordAwarenessAction, - ): void { - try { - this.discordAwarenessOutbox!.recordSuccess(batchId, ref, action); - } catch (error) { - throw new DiscordAwarenessAccountingError('recordSuccess ledger write', error); - } - } - - private writeDiscordAwarenessFailure( - batchId: string, - ref: import('./recovery/discord-awareness-outbox.js').DiscordAwarenessRef, - action: import('./recovery/discord-awareness-outbox.js').DiscordAwarenessAction, - errorMessage: string, - permanent: boolean, - ): void { - try { - this.discordAwarenessOutbox!.recordFailure( - batchId, - ref, - action, - errorMessage, - permanent, - ); - } catch (error) { - throw new DiscordAwarenessAccountingError('recordFailure ledger write', error); - } - } - - private beginDiscordAwarenessBarrier(): { - requiresBarrier: boolean; - promise: Promise; - } { - if (!this.discordAwarenessOutbox) { - return { - requiresBarrier: false, - promise: Promise.resolve({ status: 'delivered', delivered: 0, failed: 0 }), - }; - } - try { - this.discordAwarenessOutbox.reconcileForBranch( - this.store.currentBranch().name, - this.store.listBranches(), - ); - } catch (error) { - throw new DiscordAwarenessAccountingError('branch reconciliation', error); - } - const pending = this.readDiscordAwarenessPending(); - // A drain that is still running holds operations it read before this - // reconciliation, and it sends them whatever the ledger now says. A branch - // move can hide them from `pending`: restoring a rollback's source makes - // its marks undesired, and an add whose reply has not arrived is not yet - // present, so the ledger shows nothing to do while the drain goes on to - // send its saved adds and then the removals they need. So a generation - // accounts for every running drain as well as for every pending - // operation, and joins those drains instead of releasing the data planes - // beside them. - const serverIds = [...new Set([ - ...pending.map((operation) => operation.ref.serverId), - ...this.discordAwarenessDrains.keys(), - ])]; - if (serverIds.length === 0) { - return { - requiresBarrier: false, - promise: Promise.resolve({ status: 'delivered', delivered: 0, failed: 0 }), - }; - } - const promise = Promise.all( - serverIds.map((serverId) => this.drainDiscordAwarenessOutbox(serverId)), - ).then((outcomes): DiscordAwarenessDrainOutcome => { - if (outcomes.every((outcome) => outcome.status === 'unavailable')) { - return { - status: 'unavailable', - accounted: outcomes.reduce( - (count, outcome) => count + (outcome.status === 'unavailable' ? outcome.accounted : 0), - 0, - ), - }; - } - return { - status: 'delivered', - delivered: outcomes.reduce( - (count, outcome) => count + (outcome.status === 'delivered' ? outcome.delivered : 0), - 0, - ), - failed: outcomes.reduce( - (count, outcome) => count + (outcome.status === 'delivered' ? outcome.failed : 0), - 0, - ), - }; - }); - return { requiresBarrier: true, promise }; - } - /** - * Install one framework-global Discord-awareness generation before releasing - * any inference-bearing event. With pending work or a drain still running, - * every MCPL data plane is paused while all control planes remain live. - * With neither, ready() runs synchronously so request responders preserve - * their historical same-stack behavior. + * Open MCPL connections' planes (every connection when none is given). A + * quiesced host (issue #122) opens only control planes, so data planes stay + * held for the whole maintenance window; resume() calls this again once + * the flag is cleared. Control planes always come up, since host/command + * (and with it resume) rides the control plane. */ - private installMcplDataPlaneGate(): DiscordAwarenessBarrier { - for (const connection of this.mcplServerRegistry?.getAllServers() ?? []) { - connection.pauseDataPlane(); - } - const generation = ++this.discordAwarenessBarrierGeneration; - - let begun: ReturnType; - try { - begun = this.beginDiscordAwarenessBarrier(); - } catch (error) { - begun = { - requiresBarrier: true, - promise: Promise.reject(error), - }; + private readyMcplPlanes(connections?: McplServerConnection[]): void { + for (const connection of connections ?? this.mcplServerRegistry?.getAllServers() ?? []) { + if (this.quiesced) connection.readyControlPlane(); + else connection.ready(); } - const barrier: DiscordAwarenessBarrier = { generation, ...begun }; - this.discordAwarenessBarrier = barrier; - return barrier; - } - - private releaseMcplDataPlaneGate( - barrier: DiscordAwarenessBarrier, - ): void { - if (this.discordAwarenessBarrier !== barrier) return; - if (barrier.requiresBarrier) { - for (const connection of this.mcplServerRegistry?.getAllServers() ?? []) { - connection.readyControlPlane(); - } - return; - } - this.completeMcplDataPlaneGate(barrier); - } - - private completeMcplDataPlaneGate( - barrier: DiscordAwarenessBarrier, - ): boolean { - if (this.discordAwarenessBarrier !== barrier) return false; - this.discordAwarenessBarrier = null; - for (const connection of this.mcplServerRegistry?.getAllServers() ?? []) { - // ready() can synchronously flush a nested list-change notification that - // installs a newer global generation. Never let this older completion - // release any remaining server behind that newer gate. (The quiesce - // check sits AFTER this guard: readyControlPlane can also synchronously - // flush a control event that installs a newer barrier.) - if (this.discordAwarenessBarrier !== null) return false; - // Host quiesce (issue #122): this is the single point every barrier - // completion and reconnect flow funnels through, so holding here keeps - // data planes paused across awareness drains and reconnects for the - // whole window. Control planes come up normally (host/command rides the - // control plane, so resume stays deliverable); resume() performs the - // real ready() flush through this same funnel once its feasibility gate - // passes and the flag is cleared. - if (this.quiesced) { - connection.readyControlPlane(); - continue; - } - connection.ready(); - } - return this.discordAwarenessBarrier === null; - } - - private async failMcplDataPlaneGate( - barrier: DiscordAwarenessBarrier, - context: string, - error: unknown, - ): Promise { - if (this.discordAwarenessBarrier !== barrier) return; - const err = error instanceof Error ? error : new Error(String(error)); - console.error(`[discord-awareness] ${context} failed globally:`, err.message); - const connections = this.mcplServerRegistry?.getAllServers() ?? []; - for (const connection of connections) { - this.emitTrace({ - type: 'mcpl:server-error', - serverId: connection.id, - error: `Discord awareness accounting unhealthy: ${err.message}`, - }); - } - await Promise.all(connections.map(async (connection) => { - await connection.reconnectAfterFailure().catch((closeError) => { - console.error( - `[discord-awareness] could not recycle unhealthy connection ${connection.id}:`, - closeError instanceof Error ? closeError.message : closeError, - ); - }); - })); } /** @@ -13661,7 +13767,7 @@ export class AgentFramework { */ private async connectMcplServerInternal( config: import('./mcpl/types.js').McplServerConfig, - deferAwareness = false, + staged = false, ): Promise { if (!this.mcplServerRegistry || !this.mcplHostCapabilities) { throw new Error('MCPL subsystem is not initialized'); @@ -13684,49 +13790,22 @@ export class AgentFramework { const connection = await this.mcplServerRegistry.addServer(config, this.mcplHostCapabilities); - // Wire listeners before either startup staging or the runtime global gate - // releases control traffic needed for registration and marker service. + // Wire listeners before any plane opens. this.wireMcplEvents(connection); - if (deferAwareness) { - // Staged startup: both planes are closed, so the §5.3 policy - // round-trip cannot let any buffered traffic slip — establish the - // grant now, before staging ever releases. - await this.registerMcplServerFeatures(config, connection); - this.emitTrace({ type: 'module:added', moduleName: `mcpl:${config.id}` }); - return; - } - - const awarenessBarrier = this.installMcplDataPlaneGate(); - // NO early release: both planes stay closed through accounting AND the - // §5.3 policy round-trip below (PR #79 review blocker 3 — the early - // control flush released channels-register/changed before any grant - // existed). The awareness drain rides tools/call RESPONSES, which are - // never event-buffered, so it needs no plane release to make progress. - try { - await awarenessBarrier.promise; - } catch (error) { - // Startup cannot fail open on a broken awareness ledger. Disable the - // reconnecting stub/connection before propagating the distinct error to - // initializeMcpl, which tears down any other servers and aborts create(). - // Ordering matters: awareness accounting settles BEFORE the §5.3 - // policy round-trip below, so an accounting failure aborts with the - // grant still empty and every buffered event still behind the gate — - // a failed startup never reports a released data plane. (The reverse - // order let a push arrive during the policy await and get flushed by - // teardown.) - await connection.close().catch(() => {}); - throw error; - } - - // §5.3 initial policy handshake — after accounting, before the gate - // completes: the grant is established (or knowingly left empty) before - // ANY buffered traffic (control or data) flows. complete() then - // ready()s both planes through the admission gate. + // §5.3 initial policy handshake with both planes still closed (PR #79 + // review blocker 3: an early control flush released channels-register/ + // changed before any grant existed). The grant is established (or + // knowingly left empty) before ANY buffered traffic flows. await this.registerMcplServerFeatures(config, connection); - this.completeMcplDataPlaneGate(awarenessBarrier); - this.emitTrace({ type: 'module:added', moduleName: `mcpl:${config.id}` }); + // Staged startup opens every connection together (initializeMcpl). + if (staged) return; + + this.readyMcplPlanes([connection]); + // A connection can make queued awareness work deliverable; it never + // waits on it. + this.deliverDiscordAwarenessInBackground('connect', 'host'); } /** Per-connection §17.8 fetch limiter: at most one in-flight fetch, at @@ -14229,8 +14308,6 @@ export class AgentFramework { params: PushEventParams, responder?: { respond: (result: unknown) => void; respondError: (code: number, message: string) => void }, ) => { - const barrier = this.discordAwarenessBarrier; - if (barrier) await barrier.promise; await this.pushHandler?.handlePushEvent(connection.id, params, responder as never); }); @@ -14245,8 +14322,6 @@ export class AgentFramework { return; } if (this.inferenceRouter && responder) { - const barrier = this.discordAwarenessBarrier; - if (barrier) await barrier.promise; await this.inferenceRouter.handleInferenceRequest(connection.id, params, { respond: responder.respond, respondError: responder.respondError, @@ -14332,8 +14407,6 @@ export class AgentFramework { params: ChannelsIncomingParams, responder?: { respond: (result: unknown) => void }, ) => { - const barrier = this.discordAwarenessBarrier; - if (barrier) await barrier.promise; await this.channelRegistry?.handleIncoming(connection.id, params, responder as never); }); @@ -14344,8 +14417,6 @@ export class AgentFramework { ) => { if (!responder) return; try { - const barrier = this.discordAwarenessBarrier; - if (barrier) await barrier.promise; const result = await this.handleHostCommand(connection.id, params ?? {}); responder.respond(result); } catch (error) { @@ -14356,19 +14427,10 @@ export class AgentFramework { // Handle dynamic tool list changes (notifications/tools/list_changed) connection.on('tools-list-changed', () => { - // Pause before starting any async refresh. This listener runs - // synchronously in the transport line callback, so a following inbound - // data event cannot pass before the new barrier exists. - const awarenessBarrier = this.installMcplDataPlaneGate(); - this.releaseMcplDataPlaneGate(awarenessBarrier); this.handleToolsListChanged(connection.id); - void awarenessBarrier.promise.then(() => { - this.completeMcplDataPlaneGate(awarenessBarrier); - }).catch((error) => this.failMcplDataPlaneGate( - awarenessBarrier, - 'tools-list reconciliation', - error, - )); + // A server re-listing its tools is a natural moment to retry queued + // awareness work for it; delivery never holds its traffic. + this.deliverDiscordAwarenessInBackground('list-change', 'host'); }); // Re-establish full server registration on reconnect. The 'close' handler @@ -14379,14 +14441,13 @@ export class AgentFramework { // preserved across the transient close are resumed by idempotent // registration. Then refresh tools (server may have different tools). connection.on('reconnect', (info?: { attempts?: number }) => { - // Install the barrier synchronously so any inbound event emitted after - // reconnect observes it before doing work that could wake an agent. The - // connection paused its data plane before wiring the fresh transport. - const awarenessBarrier = this.installMcplDataPlaneGate(); - // Async: the §5.3 policy Request must be answered (re-establishing the - // grant) before the data-plane barrier releases — otherwise the first + // The connection paused its data plane before wiring the fresh + // transport. Let control traffic through for registration, then + // re-establish the grant: the §5.3 policy Request must be answered + // before this connection's data plane opens, otherwise the first // post-reconnect events land while the grant is still empty and are // rejected fail-closed instead of delivered. + connection.readyControlPlane(); void (async () => { try { const config = this.mcplServerConfigs.get(connection.id); @@ -14399,14 +14460,8 @@ export class AgentFramework { error instanceof Error ? error.message : error, ); } - this.releaseMcplDataPlaneGate(awarenessBarrier); + this.readyMcplPlanes([connection]); this.handleToolsListChanged(connection.id); - })(); - void awarenessBarrier.promise.then(() => { - if ( - awarenessBarrier.requiresBarrier - && !this.completeMcplDataPlaneGate(awarenessBarrier) - ) return; this.emitTrace({ type: 'mcpl:server-reconnected', serverId: connection.id, @@ -14415,11 +14470,10 @@ export class AgentFramework { // Mirror the module:removed emitted on 'close' so module-lifecycle // consumers see the server come back, not just vanish. this.emitTrace({ type: 'module:added', moduleName: `mcpl:${connection.id}` }); - }).catch((error) => this.failMcplDataPlaneGate( - awarenessBarrier, - 'reconnect reconciliation', - error, - )); + // Work queued while the route was down can be sent now; it never + // holds this connection's traffic. + this.deliverDiscordAwarenessInBackground('reconnect', 'host'); + })(); }); // Surface connect/reconnect failures. Before these traces existed the diff --git a/src/index.ts b/src/index.ts index 9f7efcbc5..fe873b1ce 100644 --- a/src/index.ts +++ b/src/index.ts @@ -126,18 +126,31 @@ export { PassthroughStrategy, AutobiographicalStrategy, KnowledgeStrategy } from export type { KnowledgeConfig, PhaseType } from '@animalabs/context-manager'; export type { Membrane, NormalizedMessage, NormalizedRequest, ContentBlock } from '@animalabs/membrane'; -// Offline outage recovery and branch-independent Discord awareness markers +// Offline outage recovery and the Discord awareness-mark journal export { DiscordAwarenessOutbox, DEFAULT_DISCORD_AWARENESS_EMOJI, defaultDiscordAwarenessOutboxPath, extractDiscordAwarenessRefs, + selectDiscordAwarenessRefs, } from './recovery/discord-awareness-outbox.js'; export type { + DiscordAwarenessAction, + DiscordAwarenessAttempt, DiscordAwarenessBatch, - DiscordAwarenessEntry, - DiscordAwarenessOperation, + DiscordAwarenessBatchRecord, + DiscordAwarenessBatchView, + DiscordAwarenessCancelReceipt, + DiscordAwarenessDispatch, + DiscordAwarenessMarks, + DiscordAwarenessOp, + DiscordAwarenessOpStatus, + DiscordAwarenessOutcome, DiscordAwarenessRef, + DiscordAwarenessReleaseAction, + DiscordAwarenessReleaseReceipt, + DiscordAwarenessRetractReceipt, + DiscordAwarenessScope, DiscordSuppressionInterval, } from './recovery/discord-awareness-outbox.js'; // Live operator surgery (rollback / suppress) and its durable action log @@ -146,7 +159,9 @@ export type { OperatorLogEntry, OperatorLogInput, OperatorRequester, + SurgeryMarkerFacts, SurgeryMarkerReceipt, + SurgeryMarksPreview, } from './operator-log.js'; export { createOfflineRecoveryBranch } from './recovery/offline-branch.js'; export type { diff --git a/src/mcpl/server-connection.ts b/src/mcpl/server-connection.ts index 18f82d096..273f7b61a 100644 --- a/src/mcpl/server-connection.ts +++ b/src/mcpl/server-connection.ts @@ -354,8 +354,8 @@ export class McplServerConnection extends EventEmitter { /** * Close the data plane synchronously while leaving an already-released - * control plane live. Used before reconnect adoption and list-change - * reconciliation so no new inference event can beat barrier installation. + * control plane live. Used before reconnect adoption, so no data event can + * arrive before the server's grant is re-established, and by quiesce. */ pauseDataPlane(): void { this.dataPlaneReady = false; @@ -363,8 +363,8 @@ export class McplServerConnection extends EventEmitter { /** * Override emit to buffer server→host events until the corresponding plane - * is ready. Lifecycle events always pass through so reconnect can install a - * new data-plane gate before the fresh transport is exposed. + * is ready. Lifecycle events always pass through so the reconnect listener + * runs before the fresh transport's buffered traffic is released. */ override emit(event: string | symbol, ...args: unknown[]): boolean { const name = typeof event === 'string' ? event : ''; @@ -954,9 +954,9 @@ export class McplServerConnection extends EventEmitter { ); // Close the data plane before the new transport can emit. The reconnect - // lifecycle event bypasses buffering; its framework listener installs the - // awareness barrier synchronously, then releases control traffic needed - // to establish the server while data remains held. + // lifecycle event bypasses buffering; its framework listener releases + // the control traffic needed to re-establish the server's grant while + // data stays held until the grant is in place. this.pauseDataPlane(); this.resetPolicyForTransportBoundary(); this.transport = transport; diff --git a/src/operator-log.ts b/src/operator-log.ts index 726ee93ed..173192c26 100644 --- a/src/operator-log.ts +++ b/src/operator-log.ts @@ -118,32 +118,62 @@ export function capIds(ids: string[]): { ids: string[]; count: number; truncated : { ids, count: ids.length }; } +/** + * Facts every surgery marker receipt carries, whatever happened: the + * publication scope chosen (`none` unless the operator chose marks), how many + * removed addressable Discord messages the surgery left unmarked, and how + * many authorized refs it did not remove (a delayed application whose anchor + * no longer covered them; their marks are never widened to anything else). + */ +export interface SurgeryMarkerFacts { + scope: 'none' | 'addressed' | 'all'; + unmarked: number; + notRemoved: number; +} + /** * What a live surgery did about Discord awareness marks, reported apart from * the body change itself. A surgery that returns has applied its body change; * this says only whether marks were scheduled: * - * - `none` — no marks were scheduled: the change removed no addressable - * Discord message, or the awareness outbox is disabled (no - * `discordAwarenessOutboxPath` and no `storePath`). It is not proof that - * no addressable Discord message was removed: the surgery's `removedRefs` - * (`discordRefs` in the operator log) say whether any was. - * - `queued` — the outbox batch is durably active and its delivery has been - * handed to the background drain. It is not a delivery claim: per-message - * outcomes stay in the outbox ledger. - * - `not-scheduled` — the body change landed, but recording the batch as - * active failed, so it was retired from the ledger: none of its marks will - * be delivered. - * - `unresolved` — the ledger could record neither the activation nor the + * - `none` — no marks were requested (the default), or none of the removed + * messages fell within the chosen scope and authorized refs. It says + * nothing about whether Discord messages were removed: `unmarked` counts + * the addressable ones left unmarked, and the surgery's `removedRefs` + * (`discordRefs` in the operator log) list every one. + * - `queued` — one add request per mark is durably recorded and delivery has + * started. It is not a delivery claim: per-request outcomes stay in the + * awareness journal. + * - `not-scheduled` — marks were chosen and in scope, but none will be + * delivered: either this framework has no awareness journal (no storePath + * or discordAwarenessOutboxPath), or the body change landed and recording + * the batch as active failed, so it was retired from the journal. `error` + * says which. + * - `unresolved` — the journal could record neither the activation nor the * retirement. `batchId` names a retained batch whose scheduling outcome is * unresolved: any later reconciliation or restart that can read it may - * promote and deliver it, and nothing here promises otherwise. + * activate and deliver it, and nothing here promises otherwise. */ -export type SurgeryMarkerReceipt = +export type SurgeryMarkerReceipt = SurgeryMarkerFacts & ( | { status: 'none'; queued: 0 } | { status: 'queued'; queued: number; batchId: string } | { status: 'not-scheduled'; queued: 0; error: string } - | { status: 'unresolved'; queued: 0; batchId: string; error: string }; + | { status: 'unresolved'; queued: 0; batchId: string; error: string } +); + +/** What a surgery would remove and which messages each marks scope covers. */ +export interface SurgeryMarksPreview { + messagesRemoved: number; + /** Removed messages that carry a Discord address at all. */ + addressable: number; + emoji: string; + scopes: Record<'addressed' | 'all', { + count: number; + channels: Array<{ channelId: string; count: number }>; + /** Pass back as `marks.refs` to bind the choice to exactly these. */ + refs: Array<{ serverId: string; channelId: string; messageId: string }>; + }>; +} /** Thrown by live surgery methods when a request cannot be honored. `code` * lets surfaces distinguish "agent busy — quiesce first" from bad input. */ diff --git a/src/recovery/discord-awareness-outbox.ts b/src/recovery/discord-awareness-outbox.ts index 5ea0b4109..cde3dfc3a 100644 --- a/src/recovery/discord-awareness-outbox.ts +++ b/src/recovery/discord-awareness-outbox.ts @@ -1,478 +1,1266 @@ import { + closeSync, + existsSync, + fsyncSync, mkdirSync, + openSync, readFileSync, renameSync, - writeFileSync, + truncateSync, + writeSync, } from 'node:fs'; -import { dirname, join } from 'node:path'; -import { randomUUID } from 'node:crypto'; +import { basename, dirname, join } from 'node:path'; +import { createHash, randomUUID } from 'node:crypto'; + +/** + * Discord awareness marks: a reaction (💤 by default) that an operator may + * choose to place on Discord messages a surgery removed from a resident's + * context, so the people who wrote them can see the resident no longer has + * them in view. + * + * Marks are an explicit publication choice, separate from the surgery itself + * (`DiscordAwarenessMarks`, default `none`). Once requested, they are + * one-shot: branch moves and restarts never derive new operations from branch + * state; an operator retracts, cancels or releases explicitly. + * + * The ledger is an append-only journal of what was asked and what was + * answered, never a model of what is on Discord. Each operation is recorded + * as `requested`, then `dispatching` (written before the request can leave), + * then an outcome. A request that was dispatched and never answered stays + * `unknown` for good: a later retry's confirmation does not resolve it, and a + * later opposite operation discloses it. + * + * Operations are keyed by (MCPL server id, channel, message, emoji). The + * server id is the configured route to a bot, not the bot's identity: if the + * route is re-pointed at another bot, retained operations act as that bot, + * and nothing here can detect it. + */ export const DEFAULT_DISCORD_AWARENESS_EMOJI = '💤'; +/** The journal file under a store, next to the other recovery state. */ export function defaultDiscordAwarenessOutboxPath(storePath: string): string { - return join(storePath, 'recovery', 'discord-awareness-outbox.json'); + return join(storePath, 'recovery', 'discord-awareness-journal.jsonl'); } +/** The pre-journal (version 1 and 2) ledger file name, imported once. */ +const LEGACY_LEDGER_NAME = 'discord-awareness-outbox.json'; + export interface DiscordAwarenessRef { serverId: string; channelId: string; messageId: string; } +/** Which removed messages a publication choice covers. */ +export type DiscordAwarenessScope = 'addressed' | 'all'; + +/** + * An operator's publication choice for one surgery. `none`, the default, + * keeps the surgery local. With `refs`, the choice authorizes exactly those + * refs (as a preview showed them): the surgery marks only the ones it + * actually removes, and never widens the set. Without `refs`, the scope is + * evaluated when the surgery applies; that form is for an immediate act whose + * choice and application are the same moment. + */ +export type DiscordAwarenessMarks = + | 'none' + | { scope: DiscordAwarenessScope; refs?: DiscordAwarenessRef[] }; + export type DiscordAwarenessAction = 'add' | 'remove'; -export type DiscordAwarenessDeliveryStatus = 'pending' | 'applied' | 'permanent-failure'; -export interface DiscordAwarenessEntry extends DiscordAwarenessRef { - desired: boolean; - markerPresent: boolean; - deliveryStatus: DiscordAwarenessDeliveryStatus; - attempts: number; - lastAction?: DiscordAwarenessAction; - lastAttemptAt?: number; - lastError?: string; +export interface DiscordSuppressionInterval { + fromId: string; + toId: string; } -export interface DiscordAwarenessBatch { +export interface DiscordAwarenessReleaseAction extends DiscordAwarenessRef { + action: DiscordAwarenessAction; + /** Earlier attempts recorded for this ref before it was held (legacy import). */ + priorAttempts?: number; + /** The last error recorded for this ref before it was held, verbatim. */ + priorError?: string; +} + +/** One surgery's request, as recorded when the surgery prepared it. */ +export interface DiscordAwarenessBatchRecord { id: string; - status: 'prepared' | 'active'; agentName: string; sourceBranch: string; targetBranch: string; emoji: string; createdAt: number; - refs: DiscordAwarenessEntry[]; + /** `none` for a suppression journal with no marks; `legacy` for imports. */ + scope: 'none' | DiscordAwarenessScope | 'legacy'; + /** The refs this batch requests marks on (authorized ∩ removed). */ + refs: DiscordAwarenessRef[]; + /** Removed addressable messages this batch leaves unmarked. */ + unmarked?: number; + /** Authorized refs the surgery did not remove. */ + notRemoved?: number; /** - * `target-branch` may be promoted when its target branch (or a descendant) - * is active. `explicit` requires activate() after a second mutation. + * `target-branch`: startup activates the batch only if its target is + * exactly the active branch (the surgery crashed after switching). + * `explicit`: only the surgery (or startup's resume of its intervals) does. */ activationPolicy?: 'target-branch' | 'explicit'; /** Idempotent interval operations used to resume an interrupted suppression. */ suppressionIntervals?: DiscordSuppressionInterval[]; } -export interface DiscordSuppressionInterval { - fromId: string; - toId: string; +export type DiscordAwarenessBatchStatus = 'prepared' | 'active' | 'held' | 'discarded'; + +export interface DiscordAwarenessBatch extends DiscordAwarenessBatchRecord { + status: DiscordAwarenessBatchStatus; + /** A held suppression whose body was completed at a later startup. */ + suppressionComplete?: boolean; + held?: { reason: string; at: number; releaseActions: DiscordAwarenessReleaseAction[] }; + cancelled?: { at: number; by?: string }; + released?: { at: number; by?: string }; } -export interface DiscordAwarenessOperation { - batchId: string; - emoji: string; +export type DiscordAwarenessOutcome = 'confirmed' | 'failed' | 'not-sent' | 'unknown'; + +export interface DiscordAwarenessAttempt { + attempt: number; + /** One id per request on the wire; operations sharing a dispatch share it. */ + dispatchId?: string; + dispatchedAt: number; + /** Absent while the request is on the wire in this process. */ + outcome?: DiscordAwarenessOutcome; + /** For `failed`: Discord's refusal is final for this request. */ + permanent?: boolean; + outcomeAt?: number; + error?: string; +} + +export interface DiscordAwarenessOpCause { + kind: 'batch' | 'retract' | 'release' | 'import'; + batchId?: string; + requestId?: string; +} + +export type DiscordAwarenessOpStatus = + | 'requested' + | 'dispatching' + | 'confirmed' + | 'failed' + | 'unknown' + | 'cancelled'; + +export interface DiscordAwarenessOp { + opId: string; + key: DiscordAwarenessRef & { emoji: string }; + action: DiscordAwarenessAction; + cause: DiscordAwarenessOpCause; + requestedAt: number; + attempts: DiscordAwarenessAttempt[]; + cancelled?: { at: number; reason: 'batch-cancelled' | 'superseded' }; +} + +/** One dispatch: the head operation for a reaction key, plus any consecutive + * pending operations with the same action that share it. */ +export interface DiscordAwarenessDispatch { + key: DiscordAwarenessRef & { emoji: string }; action: DiscordAwarenessAction; - ref: DiscordAwarenessEntry; + opIds: string[]; } -interface DiscordAwarenessOutboxDocument { - version: 2; - batches: DiscordAwarenessBatch[]; +/** Receipt for cancel(). Nothing is ever removed from Discord by a cancel. */ +export interface DiscordAwarenessCancelReceipt { + batchId: string; + /** Requests that will now never be sent (for a batch not yet activated, + * every mark it would have requested). */ + cancelled: number; + /** Held release actions dropped (a held batch was cancelled). */ + heldDropped: number; + /** Operations on the wire right now: they may still land. */ + inFlight: number; + /** Operations whose outcome is unknown: they may have landed. */ + unknown: number; + /** Operations Discord already confirmed (cancel does not undo them). */ + confirmed: number; +} + +/** Receipt for retract(). */ +export interface DiscordAwarenessRetractReceipt { + requestId: string; + /** Removals queued (one per reaction key). */ + removalsQueued: number; + /** Add requests for those keys that had not been sent, now superseded. */ + addsSuperseded: number; + /** Keys whose earlier add attempts are unresolved (on the wire or unknown): + * such an add may land after this removal. */ + keysWithUnresolvedAdds: number; + /** Unresolved earlier add requests on those keys (one per request sent, + * however many batches' operations it carried). */ + unresolvedAddAttempts: number; } -interface BranchDescriptor { +export interface DiscordAwarenessReleaseReceipt { + batchId: string; + addsQueued: number; + removalsQueued: number; +} + +/** Per-batch view for operators: what was asked and what is known. */ +export interface DiscordAwarenessBatchView { id: string; - name: string; - parentId?: string; + status: DiscordAwarenessBatchStatus; + scope: DiscordAwarenessBatchRecord['scope']; + agentName: string; + sourceBranch: string; + targetBranch: string; + emoji: string; + createdAt: number; + refs: number; + unmarked?: number; + notRemoved?: number; + held?: { reason: string; at: number; releaseActions: number }; + cancelled?: { at: number; by?: string }; + released?: { at: number; by?: string }; + /** Counts over the add operations this batch requested, by status. */ + adds: Record; + /** Removals requested (by any retract) on this batch's refs, by status. */ + removals: Record; + /** Attempts on this batch's refs whose outcome is unknown or on the wire. */ + unresolvedAttempts: number; } +/** Extract Discord addressing metadata without reading message content. */ export interface DiscordMessageMetadataCarrier { metadata?: Record; } -/** Extract Discord addressing metadata without reading message content. */ export function extractDiscordAwarenessRefs( messages: DiscordMessageMetadataCarrier[], forcedServerId?: string, ): DiscordAwarenessRef[] { const refs = new Map(); + for (const message of messages) { + const ref = discordRef(message, forcedServerId); + if (ref) refs.set(refKey(ref), ref); + } + return [...refs.values()]; +} +/** + * Select the refs a publication choice covers among removed messages. + * `addressed` means the message carried the MCPL `chat:addressed` tag (a + * mention, a reply to the bot, or a DM, as the server classified it); the + * same signal the framework uses for loci and wakes. With `marks.refs`, only + * those refs are eligible as well. + */ +export function selectDiscordAwarenessRefs( + messages: DiscordMessageMetadataCarrier[], + marks: DiscordAwarenessMarks, + forcedServerId?: string, +): { refs: DiscordAwarenessRef[]; addressable: number; unmarked: number; notRemoved: number } { + const addressable = new Map(); for (const message of messages) { - const metadata = message.metadata ?? {}; - const channelId = typeof metadata.channelId === 'string' ? metadata.channelId : ''; - const messageId = typeof metadata.messageId === 'string' ? metadata.messageId : ''; - const metadataServerId = typeof metadata.serverId === 'string' ? metadata.serverId : ''; - const serverId = forcedServerId ?? metadataServerId; - if (!channelId || !messageId || !serverId) continue; + const ref = discordRef(message, forcedServerId); + if (!ref) continue; + const key = refKey(ref); + const addressed = isAddressed(message); + const existing = addressable.get(key); + addressable.set(key, { ref, addressed: addressed || existing?.addressed === true }); + } + if (marks === 'none') { + return { refs: [], addressable: addressable.size, unmarked: addressable.size, notRemoved: 0 }; + } + const authorized = marks.refs ? new Set(marks.refs.map(refKey)) : null; + const refs: DiscordAwarenessRef[] = []; + for (const [key, candidate] of addressable) { + if (marks.scope === 'addressed' && !candidate.addressed) continue; + if (authorized && !authorized.has(key)) continue; + refs.push(candidate.ref); + } + const notRemoved = authorized + ? [...authorized].filter((key) => !addressable.has(key)).length + : 0; + return { refs, addressable: addressable.size, unmarked: addressable.size - refs.length, notRemoved }; +} - const isDiscord = serverId.toLowerCase().includes('discord') || channelId.startsWith('discord:'); - if (!isDiscord) continue; +function discordRef(message: DiscordMessageMetadataCarrier, forcedServerId?: string): DiscordAwarenessRef | null { + const metadata = message.metadata ?? {}; + const channelId = typeof metadata.channelId === 'string' ? metadata.channelId : ''; + const messageId = typeof metadata.messageId === 'string' ? metadata.messageId : ''; + const metadataServerId = typeof metadata.serverId === 'string' ? metadata.serverId : ''; + const serverId = forcedServerId ?? metadataServerId; + if (!channelId || !messageId || !serverId) return null; + const isDiscord = serverId.toLowerCase().includes('discord') || channelId.startsWith('discord:'); + if (!isDiscord) return null; + return { serverId, channelId, messageId }; +} - const ref = { serverId, channelId, messageId }; - refs.set(refKey(ref), ref); - } +function isAddressed(message: DiscordMessageMetadataCarrier): boolean { + const tags = message.metadata?.tags; + return Array.isArray(tags) && tags.includes('chat:addressed'); +} - return [...refs.values()]; +// --------------------------------------------------------------------------- +// Journal records +// --------------------------------------------------------------------------- + +type JournalRecord = + | { t: 'batch'; at: number; batch: DiscordAwarenessBatchRecord } + | { t: 'activated'; at: number; batchId: string } + | { t: 'discarded'; at: number; batchId: string } + | { t: 'held'; at: number; batchId: string; reason: string; releaseActions: DiscordAwarenessReleaseAction[] } + | { t: 'released'; at: number; batchId: string; by?: string } + | { t: 'cancelled'; at: number; batchId: string; by?: string } + | { + t: 'requested'; + at: number; + opId: string; + key: DiscordAwarenessRef & { emoji: string }; + action: DiscordAwarenessAction; + cause: DiscordAwarenessOpCause; + } + | { t: 'op-cancelled'; at: number; opId: string; reason: 'batch-cancelled' | 'superseded' } + | { t: 'dispatching'; at: number; opId: string; attempt: number; dispatchId?: string } + | { + t: 'outcome'; + at: number; + opId: string; + attempt: number; + outcome: DiscordAwarenessOutcome; + permanent?: boolean; + error?: string; + } + | { t: 'retract'; at: number; requestId: string; target: string; by?: string } + | { t: 'suppression-complete'; at: number; batchId: string } + | { t: 'import-complete'; at: number; source: 'v2'; sha256: string } + | { t: 'commit'; at: number; txn: string }; + +/** + * Every multi-record append is one transaction: its records carry a `txn` + * id and only take effect when the closing `commit` record is present. A + * torn append (disk full, crash mid-write) therefore applies none of its + * records, never a prefix, so an activation can't half-queue its marks and a + * retract can't supersede adds without queuing its removals. + */ +type StoredRecord = JournalRecord & { txn?: string }; + +interface JournalState { + batches: Map; + /** Operations in request order. */ + ops: Map; + importedSources: Set; } /** - * A branch-independent, durable operation ledger for Discord awareness marks. - * - * Completed entries are retained. Reconciliation derives whether their emoji - * should currently exist from Chronicle branch ancestry, queuing `remove` - * after switching away from a recovery branch and `add` when returning to it. + * The durable awareness-mark ledger: an append-only journal (one JSON record + * per line, each append fsynced) and the operations derived from it. */ export class DiscordAwarenessOutbox { - constructor(readonly path: string) {} + readonly path: string; + private readonly legacyPath: string; + /** Operations dispatched by this process and still awaiting an outcome. */ + private readonly inFlight = new Set(); + + /** + * @param path the journal file. A path ending in `.json` names a pre-journal + * ledger (as an older `discordAwarenessOutboxPath` would): the journal then + * lives beside it as `.journal.jsonl` and the file itself is imported. + */ + constructor(path: string) { + if (path.endsWith('.json')) { + this.legacyPath = path; + this.path = `${path.slice(0, -'.json'.length)}.journal.jsonl`; + } else { + this.path = path; + this.legacyPath = join(dirname(path), LEGACY_LEDGER_NAME); + } + } + // -- surgery side ----------------------------------------------------------- + + /** + * Record a surgery's request before its branch switch. Returns null when it + * carries neither marks nor a suppression journal. A prepared batch never + * delivers: activate() (or startup's crash completion) does. + */ prepare(input: { agentName: string; sourceBranch: string; targetBranch: string; refs: DiscordAwarenessRef[]; + scope: DiscordAwarenessBatchRecord['scope']; emoji?: string; activationPolicy?: 'target-branch' | 'explicit'; suppressionIntervals?: DiscordSuppressionInterval[]; + unmarked?: number; + notRemoved?: number; }): DiscordAwarenessBatch | null { const refs = dedupeRefs(input.refs); if (refs.length === 0 && !input.suppressionIntervals?.length) return null; - - const document = this.read(); - const batch: DiscordAwarenessBatch = { + const batch: DiscordAwarenessBatchRecord = { id: randomUUID(), - status: 'prepared', agentName: input.agentName, sourceBranch: input.sourceBranch, targetBranch: input.targetBranch, emoji: input.emoji ?? DEFAULT_DISCORD_AWARENESS_EMOJI, createdAt: Date.now(), - refs: refs.map((ref) => ({ - ...ref, - desired: false, - markerPresent: false, - deliveryStatus: 'applied', - attempts: 0, - })), + scope: input.scope, + refs, + ...(input.unmarked ? { unmarked: input.unmarked } : {}), + ...(input.notRemoved ? { notRemoved: input.notRemoved } : {}), activationPolicy: input.activationPolicy ?? 'target-branch', - suppressionIntervals: input.suppressionIntervals?.map((interval) => ({ ...interval })), + ...(input.suppressionIntervals?.length + ? { suppressionIntervals: input.suppressionIntervals.map((interval) => ({ ...interval })) } + : {}), }; - document.batches.push(batch); - this.write(document); - return structuredClone(batch); + this.append([{ t: 'batch', at: batch.createdAt, batch }]); + return { ...structuredClone(batch), status: 'prepared' }; } - /** Activate after Chronicle has safely switched to the target branch. */ - activate(batchId: string): void { - const document = this.read(); - const batch = document.batches.find((candidate) => candidate.id === batchId); - if (!batch) throw new Error(`Discord awareness outbox batch not found: ${batchId}`); - batch.status = 'active'; - for (const ref of batch.refs) setDesired(ref, true); - this.write(document); + /** + * Activate a prepared batch after its body change landed: one `add` request + * per ref, durable before this returns. Returns how many were queued. + */ + activate(batchId: string): number { + const state = this.load(); + const batch = state.batches.get(batchId); + if (!batch) throw new Error(`Discord awareness batch not found: ${batchId}`); + if (batch.status === 'active') return 0; + if (batch.status !== 'prepared') { + throw new Error(`Discord awareness batch ${batchId} is ${batch.status}, not prepared`); + } + if (batch.cancelled) { + // An operator cancelled it while its surgery was still running. + throw new Error(`Discord awareness batch ${batchId} was cancelled before activation`); + } + const at = Date.now(); + const records: JournalRecord[] = [{ t: 'activated', at, batchId }]; + records.push(...this.requestRecords(state, batch.refs.map((ref) => ({ ...ref, action: 'add' as const })), batch.emoji, { + kind: 'batch', + batchId, + }, at)); + this.append(records); + return batch.refs.length; } /** - * Retire a batch that was prepared but whose operation did not complete - * (fork failed, a removal threw and the source branch was restored). A - * prepared batch is armed: it re-fires at boot or on branch switch through - * `activatePreparedForBranch` / `preparedSuppressionsForBranch`, and an - * un-completable suppression there aborts framework start. Only prepared - * batches can be discarded — an active batch owns delivered markers and - * must be reconciled, not dropped. Returns false when nothing was removed. + * Retire a batch whose surgery did not complete. Only a prepared batch can + * be discarded; returns false when nothing was retired. */ discard(batchId: string): boolean { - const document = this.read(); - const index = document.batches.findIndex((candidate) => candidate.id === batchId); - if (index < 0) return false; - if (document.batches[index].status !== 'prepared') return false; - document.batches.splice(index, 1); - this.write(document); + const batch = this.load().batches.get(batchId); + if (!batch || batch.status !== 'prepared') return false; + this.append([{ t: 'discarded', at: Date.now(), batchId }]); return true; } /** - * Reconcile every retained ledger entry against the active branch. A marker - * is desired when any active batch for the same ref+emoji has a target branch - * in the active branch's ancestry. + * Suppressions whose body may still need resuming on the active branch: + * explicit batches with intervals whose target is exactly that branch, + * prepared (crash completion: resume, then activate) or held at an earlier + * startup (resume the body only; marks stay held). Branch ancestry is never + * consulted. + */ + preparedSuppressionsForBranch(branchName: string): DiscordAwarenessBatch[] { + return [...this.load().batches.values()] + .filter((batch) => (batch.status === 'prepared' || batch.status === 'held') + && !batch.suppressionComplete + && batch.activationPolicy === 'explicit' + && !!batch.suppressionIntervals?.length + && batch.targetBranch === branchName) + .map((batch) => structuredClone(batch)); + } + + /** Record that a held suppression's body was completed (its marks stay held). */ + recordSuppressionComplete(batchId: string): void { + this.append([{ t: 'suppression-complete', at: Date.now(), batchId }]); + } + + /** + * Startup reconciliation, run once before any delivery. It derives nothing + * from branch ancestry: + * - an attempt left `dispatching` by a previous process is recorded + * `unknown` (it may have reached Discord); + * - a prepared batch whose target is exactly the active branch completes + * its crash window (activated), except a suppression, whose resume + * completes its body first; + * - any other prepared batch is held for an operator; + * - a batch an operator cancelled while it was prepared stays as it is. */ - reconcileForBranch( - branchName: string, - branches: BranchDescriptor[] = [], - ): { activated: number; queued: number } { - const document = this.read(); - let activated = 0; - - for (const batch of document.batches) { - if ( - batch.status === 'prepared' - && (batch.activationPolicy ?? 'target-branch') === 'target-branch' - && isTargetActive(branchName, batch.targetBranch, branches) - ) { - batch.status = 'active'; - activated++; + recoverAtStartup(activeBranchName: string): { activated: string[]; held: string[]; unknownAttempts: number } { + const state = this.load(); + const at = Date.now(); + const records: JournalRecord[] = []; + let unknownAttempts = 0; + for (const op of state.ops.values()) { + for (const attempt of op.attempts) { + if (attempt.outcome !== undefined) continue; + records.push({ + t: 'outcome', + at, + opId: op.opId, + attempt: attempt.attempt, + outcome: 'unknown', + error: 'the previous process ended before an outcome was recorded', + }); + unknownAttempts++; } } + const activated: string[] = []; + const held: string[] = []; + for (const batch of state.batches.values()) { + if (batch.status !== 'prepared' || batch.cancelled) continue; + const suppression = (batch.activationPolicy ?? 'target-branch') === 'explicit' + && !!batch.suppressionIntervals?.length; + if (batch.targetBranch === activeBranchName) { + if (suppression) continue; // resumePreparedDiscordSuppressions completes it - const desiredKeys = new Set(); - for (const batch of document.batches) { - if (batch.status !== 'active') continue; - if (!isTargetActive(branchName, batch.targetBranch, branches)) continue; - for (const ref of batch.refs) desiredKeys.add(markerKey(batch.emoji, ref)); + records.push({ t: 'activated', at, batchId: batch.id }); + records.push(...this.requestRecords( + state, + batch.refs.map((ref) => ({ ...ref, action: 'add' as const })), + batch.emoji, + { kind: 'batch', batchId: batch.id }, + at, + )); + activated.push(batch.id); + } else { + records.push({ + t: 'held', + at, + batchId: batch.id, + reason: suppression + ? `its suppression on ${batch.targetBranch} was interrupted; the body resumes when that branch is active at startup, the marks only on release` + : `its surgery was interrupted before the switch to ${batch.targetBranch} was confirmed`, + releaseActions: batch.refs.map((ref) => ({ ...ref, action: 'add' })), + }); + held.push(batch.id); + } } + if (records.length > 0) this.append(records); + return { activated, held, unknownAttempts }; + } + + // -- operator controls ------------------------------------------------------ - let queued = 0; - for (const batch of document.batches) { - if (batch.status !== 'active') continue; - for (const ref of batch.refs) { - setDesired(ref, desiredKeys.has(markerKey(batch.emoji, ref))); - if (ref.deliveryStatus === 'pending') queued++; + /** + * Stop all future dispatch and retry of a batch's requests: unsent ones, + * ones on the wire (their outcome is still recorded when it comes), and + * ones whose outcome is unknown. A held batch's release actions are + * dropped. Nothing is ever removed from Discord by a cancel: the receipt + * reports what may already have landed, and retract removes it. + */ + cancel(batchId: string, by?: string): DiscordAwarenessCancelReceipt { + const state = this.load(); + const batch = state.batches.get(batchId); + if (!batch) throw new Error(`Discord awareness batch not found: ${batchId}`); + const receipt: DiscordAwarenessCancelReceipt = { + batchId, + // A prepared batch (its surgery still running, or its bookkeeping + // unresolved) has no requests yet: every one of its marks is stopped. + cancelled: batch.status === 'prepared' && !batch.cancelled ? batch.refs.length : 0, + heldDropped: batch.held?.releaseActions.length ?? 0, + inFlight: 0, + unknown: 0, + confirmed: 0, + }; + if (batch.cancelled) return receipt; + const at = Date.now(); + const records: JournalRecord[] = [{ t: 'cancelled', at, batchId, ...(by ? { by } : {}) }]; + for (const op of state.ops.values()) { + if (op.cause.batchId !== batchId) continue; + const status = this.opStatus(op); + if (status === 'confirmed') { + receipt.confirmed++; + continue; } + if (status === 'requested') receipt.cancelled++; + else if (status === 'dispatching') receipt.inFlight++; + else if (status === 'unknown') receipt.unknown++; + else continue; // failed or already cancelled + records.push({ t: 'op-cancelled', at, opId: op.opId, reason: 'batch-cancelled' }); } + this.append(records); + return receipt; + } - this.write(document); - return { activated, queued }; + /** + * Remove this bot's mark (through the configured route) from a batch's refs, + * or from every ref any batch ever sent an add for. Acts across batches: a + * reaction is one per (route, message, emoji), whichever batches asked for + * it. Keys where no add attempt ever left the host are skipped. Add + * requests not yet sent for those keys are superseded. The removal is + * always sent, whatever earlier outcomes say. + */ + retract(target: string | 'all', by?: string): DiscordAwarenessRetractReceipt { + const state = this.load(); + let candidateKeys: Array; + if (target === 'all') { + candidateKeys = uniqueKeys([...state.ops.values()].filter((op) => op.action === 'add').map((op) => op.key)); + } else { + const batch = state.batches.get(target); + if (!batch) throw new Error(`Discord awareness batch not found: ${target}`); + candidateKeys = batch.refs.map((ref) => ({ ...ref, emoji: batch.emoji })); + } + const requestId = randomUUID(); + const at = Date.now(); + const receipt: DiscordAwarenessRetractReceipt = { + requestId, + removalsQueued: 0, + addsSuperseded: 0, + keysWithUnresolvedAdds: 0, + unresolvedAddAttempts: 0, + }; + const toRemove: Array = []; + const records: JournalRecord[] = [{ t: 'retract', at, requestId, target, ...(by ? { by } : {}) }]; + for (const key of candidateKeys) { + const adds = this.opsForKey(state, key).filter((op) => op.action === 'add'); + const left = adds.some((op) => op.attempts.some((attempt) => attempt.outcome !== 'not-sent')); + if (!left) { + // Nothing was ever sent for this key: stop its unsent adds, and + // there is no reaction of ours to remove. + for (const op of adds) { + if (this.opStatus(op) !== 'requested') continue; + records.push({ t: 'op-cancelled', at, opId: op.opId, reason: 'superseded' }); + receipt.addsSuperseded++; + } + continue; + } + const unresolved = unresolvedDispatches(adds, (op, attempt) => this.attemptUnresolved(op, attempt)); + if (unresolved > 0) { + receipt.keysWithUnresolvedAdds++; + receipt.unresolvedAddAttempts += unresolved; + } + toRemove.push({ ...key, action: 'remove' }); + } + const requested = this.requestRecords(state, toRemove, undefined, { kind: 'retract', requestId }, at); + receipt.addsSuperseded += requested.filter((record) => record.t === 'op-cancelled').length; + receipt.removalsQueued = requested.filter((record) => record.t === 'requested').length; + records.push(...requested); + this.append(records); + return receipt; } - /** Backward-compatible startup helper. */ - activatePreparedForBranch(branchName: string, branches: BranchDescriptor[] = []): number { - return this.reconcileForBranch(branchName, branches).activated; + /** Release a held batch: queue its recorded release actions, explicitly. */ + release(batchId: string, by?: string): DiscordAwarenessReleaseReceipt { + const state = this.load(); + const batch = state.batches.get(batchId); + if (!batch) throw new Error(`Discord awareness batch not found: ${batchId}`); + if (batch.status !== 'held' || !batch.held) { + throw new Error(`Discord awareness batch ${batchId} is ${batch.status}, not held`); + } + const at = Date.now(); + const actions = batch.held.releaseActions; + const records: JournalRecord[] = [{ t: 'released', at, batchId, ...(by ? { by } : {}) }]; + records.push(...this.requestRecords(state, actions.map((action) => ({ ...action })), batch.emoji, { + kind: 'release', + batchId, + }, at)); + this.append(records); + return { + batchId, + addsQueued: actions.filter((action) => action.action === 'add').length, + removalsQueued: actions.filter((action) => action.action === 'remove').length, + }; } - pending(serverId?: string): DiscordAwarenessOperation[] { - const operations: DiscordAwarenessOperation[] = []; - for (const batch of this.read().batches) { - if (batch.status !== 'active') continue; - for (const ref of batch.refs) { - if (ref.deliveryStatus !== 'pending') continue; - if (serverId !== undefined && ref.serverId !== serverId) continue; - operations.push({ - batchId: batch.id, - emoji: batch.emoji, - action: ref.desired ? 'add' : 'remove', - ref: { ...ref }, - }); + // -- delivery --------------------------------------------------------------- + + /** + * The next dispatch for each reaction key that has work and nothing on the + * wire. A key's operations go out in request order; consecutive requested + * operations with the same action share one dispatch. An `unknown` + * operation is retried only while it is still the newest live operation + * for its key. + */ + pendingDispatches(serverId?: string): DiscordAwarenessDispatch[] { + const state = this.load(); + const byKey = new Map(); + for (const op of state.ops.values()) { + if (serverId !== undefined && op.key.serverId !== serverId) continue; + const key = markKey(op.key); + const list = byKey.get(key); + if (list) list.push(op); + else byKey.set(key, [op]); + } + const dispatches: Array = []; + for (const ops of byKey.values()) { + // One request per reaction key on the wire at a time, including a + // cancelled one still awaiting its outcome: opposite actions must not + // overtake each other. + if (ops.some((op) => op.attempts.at(-1)?.outcome === undefined && op.attempts.length > 0 + && this.inFlight.has(op.opId))) continue; + const live = ops.filter((op) => !op.cancelled); + let head: DiscordAwarenessOp | undefined; + for (let index = 0; index < live.length; index++) { + const op = live[index]; + const status = this.opStatus(op); + if (status === 'requested') { head = op; break; } + if (status === 'unknown' && index === live.length - 1) { head = op; break; } } + if (!head) continue; + const opIds = [head.opId]; + const start = live.indexOf(head); + for (let index = start + 1; index < live.length; index++) { + const next = live[index]; + if (next.action !== head.action || this.opStatus(next) !== 'requested') break; + opIds.push(next.opId); + } + dispatches.push({ key: head.key, action: head.action, opIds, order: head.requestedAt }); } - return operations; - } - - recordSuccess(batchId: string, ref: DiscordAwarenessRef, action: DiscordAwarenessAction): void { - const document = this.read(); - const entry = findEntry(document, batchId, ref); - if (!entry) return; - entry.markerPresent = action === 'add'; - entry.deliveryStatus = entry.markerPresent === entry.desired ? 'applied' : 'pending'; - entry.attempts++; - entry.lastAction = action; - entry.lastAttemptAt = Date.now(); - delete entry.lastError; - this.write(document); - } - - recordFailure( - batchId: string, - ref: DiscordAwarenessRef, - action: DiscordAwarenessAction, - error: string, - permanent: boolean, - ): void { - const document = this.read(); - const entry = findEntry(document, batchId, ref); - if (!entry) return; - entry.deliveryStatus = permanent ? 'permanent-failure' : 'pending'; - entry.attempts++; - entry.lastAction = action; - entry.lastAttemptAt = Date.now(); - entry.lastError = error; - this.write(document); + return dispatches + .sort((a, b) => a.order - b.order) + .map(({ order: _order, ...dispatch }) => dispatch); + } + + /** Write-ahead: record that a dispatch is about to leave the host. */ + recordDispatching(dispatch: DiscordAwarenessDispatch): Array<{ opId: string; attempt: number }> { + const state = this.load(); + const at = Date.now(); + const dispatchId = randomUUID(); + const attempts = dispatch.opIds.map((opId) => { + const op = state.ops.get(opId); + if (!op) throw new Error(`Discord awareness operation not found: ${opId}`); + return { opId, attempt: op.attempts.length + 1 }; + }); + this.append(attempts.map(({ opId, attempt }) => ({ t: 'dispatching' as const, at, opId, attempt, dispatchId }))); + for (const { opId } of attempts) this.inFlight.add(opId); + return attempts; } - /** Compatibility alias for older callers that only acknowledged adds. */ - acknowledge(batchId: string, ref: DiscordAwarenessRef): void { - this.recordSuccess(batchId, ref, 'add'); + /** Record the outcome of a dispatch's attempts. */ + recordOutcome( + attempts: Array<{ opId: string; attempt: number }>, + outcome: DiscordAwarenessOutcome, + detail: { permanent?: boolean; error?: string } = {}, + ): void { + const at = Date.now(); + try { + this.append(attempts.map(({ opId, attempt }) => ({ + t: 'outcome' as const, + at, + opId, + attempt, + outcome, + ...(outcome === 'failed' && detail.permanent ? { permanent: true } : {}), + ...(detail.error !== undefined ? { error: detail.error } : {}), + }))); + } finally { + for (const { opId } of attempts) this.inFlight.delete(opId); + } } + // -- views ------------------------------------------------------------------ + batches(): DiscordAwarenessBatch[] { - return structuredClone(this.read().batches); + return [...this.load().batches.values()].map((batch) => structuredClone(batch)); } - preparedSuppressionsForBranch( - branchName: string, - branches: BranchDescriptor[] = [], - ): DiscordAwarenessBatch[] { - return structuredClone(this.read().batches.filter((batch) => - batch.status === 'prepared' - && batch.activationPolicy === 'explicit' - && !!batch.suppressionIntervals?.length - && isTargetActive(branchName, batch.targetBranch, branches))); + operations(): DiscordAwarenessOp[] { + const state = this.load(); + return [...state.ops.values()].map((op) => structuredClone(op)); } - private read(): DiscordAwarenessOutboxDocument { - let parsed: unknown; + /** Status of an operation as this process can know it. */ + operationStatus(op: DiscordAwarenessOp): DiscordAwarenessOpStatus { + return this.opStatus(op); + } + + view(): DiscordAwarenessBatchView[] { + const state = this.load(); + const views: DiscordAwarenessBatchView[] = []; + for (const batch of state.batches.values()) { + if (batch.status === 'discarded') continue; + const keys = new Set(batch.refs.map((ref) => markKey({ ...ref, emoji: batch.emoji }))); + const adds = emptyCounts(); + const removals = emptyCounts(); + const onKeys: DiscordAwarenessOp[] = []; + for (const op of state.ops.values()) { + if (!keys.has(markKey(op.key))) continue; + onKeys.push(op); + if (op.action === 'add' && op.cause.batchId === batch.id) adds[this.opStatus(op)]++; + if (op.action === 'remove') removals[this.opStatus(op)]++; + } + const unresolvedAttempts = unresolvedDispatches(onKeys, (op, attempt) => this.attemptUnresolved(op, attempt)); + views.push({ + id: batch.id, + status: batch.status, + scope: batch.scope, + agentName: batch.agentName, + sourceBranch: batch.sourceBranch, + targetBranch: batch.targetBranch, + emoji: batch.emoji, + createdAt: batch.createdAt, + refs: batch.refs.length, + ...(batch.unmarked !== undefined ? { unmarked: batch.unmarked } : {}), + ...(batch.notRemoved !== undefined ? { notRemoved: batch.notRemoved } : {}), + ...(batch.held + ? { held: { reason: batch.held.reason, at: batch.held.at, releaseActions: batch.held.releaseActions.length } } + : {}), + ...(batch.cancelled ? { cancelled: batch.cancelled } : {}), + ...(batch.released ? { released: batch.released } : {}), + adds, + removals, + unresolvedAttempts, + }); + } + return views; + } + + // -- internals -------------------------------------------------------------- + + private opStatus(op: DiscordAwarenessOp): DiscordAwarenessOpStatus { + const last = op.attempts.at(-1); + // A confirmation is a fact about Discord, even for a request cancelled + // while it was on the wire. + if (last?.outcome === 'confirmed') return 'confirmed'; + if (op.cancelled) return 'cancelled'; + if (!last) return 'requested'; + if (last.outcome === undefined) return this.inFlight.has(op.opId) ? 'dispatching' : 'unknown'; + if (last.outcome === 'unknown') return 'unknown'; + if (last.outcome === 'not-sent') return 'requested'; + return last.permanent ? 'failed' : 'requested'; + } + + /** + * An attempt that may have reached Discord without an answer: `unknown`, + * or still without an outcome (on the wire in this process, or left by a + * previous process before recoverAtStartup recorded it unknown). + */ + private attemptUnresolved(_op: DiscordAwarenessOp, attempt: DiscordAwarenessAttempt): boolean { + return attempt.outcome === 'unknown' || attempt.outcome === undefined; + } + + private opsForKey(state: JournalState, key: DiscordAwarenessRef & { emoji: string }): DiscordAwarenessOp[] { + const wanted = markKey(key); + return [...state.ops.values()].filter((op) => markKey(op.key) === wanted); + } + + /** + * Records for new requests. A request supersedes requests for the same + * key with the opposite action that were never sent (the newest explicit + * intent wins without churning the reaction); operations already sent are + * never touched. + */ + private requestRecords( + state: JournalState, + requests: Array, + emoji: string | undefined, + cause: DiscordAwarenessOpCause, + at: number, + ): JournalRecord[] { + const records: JournalRecord[] = []; + const superseded = new Set(); + for (const request of requests) { + const key = { + serverId: request.serverId, + channelId: request.channelId, + messageId: request.messageId, + emoji: request.emoji ?? emoji ?? DEFAULT_DISCORD_AWARENESS_EMOJI, + }; + for (const op of this.opsForKey(state, key)) { + if (op.action === request.action || superseded.has(op.opId)) continue; + if (this.opStatus(op) !== 'requested' || op.attempts.some((attempt) => attempt.outcome !== 'not-sent')) continue; + records.push({ t: 'op-cancelled', at, opId: op.opId, reason: 'superseded' }); + superseded.add(op.opId); + } + records.push({ t: 'requested', at, opId: randomUUID(), key, action: request.action, cause }); + } + return records; + } + + private load(): JournalState { + const { records } = this.readJournal(); + const state = fold(records); + const legacy = this.pendingLegacyImport(state); + if (legacy?.records.length) return fold([...records, ...legacy.records]); + return state; + } + + /** + * The journal's records, and how to repair an unterminated final line + * before the next append: a torn fragment (never applied) is truncated + * away; a complete record that only lost its newline is terminated. + * Without the repair, the next append would share that line and turn a + * harmless torn tail into corruption. + */ + private readJournal(): { records: StoredRecord[]; repair?: { truncateTo: number } | { terminate: true } } { + let raw: Buffer; try { - parsed = JSON.parse(readFileSync(this.path, 'utf8')); + raw = readFileSync(this.path); } catch (error) { - if ((error as NodeJS.ErrnoException).code === 'ENOENT') { - return { version: 2, batches: [] }; - } + if ((error as NodeJS.ErrnoException).code === 'ENOENT') return { records: [] }; throw new Error( - `Could not read Discord awareness outbox ${this.path}: ${error instanceof Error ? error.message : String(error)}`, + `Could not read Discord awareness journal ${this.path}: ${error instanceof Error ? error.message : String(error)}`, ); } + const terminated = raw.length === 0 || raw[raw.length - 1] === 0x0a; + const lastNewline = raw.lastIndexOf(0x0a); + const body = terminated ? raw : raw.subarray(0, lastNewline + 1); + const lines = body.toString('utf8').split('\n'); + const records: StoredRecord[] = []; + for (let index = 0; index < lines.length; index++) { + const line = lines[index]; + if (!line.trim()) continue; + try { + records.push(JSON.parse(line) as StoredRecord); + } catch { + throw new Error(`Corrupt Discord awareness journal ${this.path} at line ${index + 1}`); + } + } + if (terminated) return { records }; + const tail = raw.subarray(lastNewline + 1).toString('utf8'); + try { + records.push(JSON.parse(tail) as StoredRecord); + return { records, repair: { terminate: true } }; + } catch { + // A torn append: none of it ever applied. + return { records, repair: { truncateTo: lastNewline + 1 } }; + } + } - if (isDocumentV2(parsed)) return parsed; - if (isDocumentV1(parsed)) return migrateV1(parsed); - throw new Error(`Invalid Discord awareness outbox document: ${this.path}`); + /** + * A pre-journal ledger beside the journal: null when there is none; + * otherwise its import records, empty when its hash is already imported + * (a crash between the import and the rename), so the file is only renamed. + */ + private pendingLegacyImport(state: JournalState): { records: JournalRecord[]; sha256: string } | null { + let raw: string; + try { + raw = readFileSync(this.legacyPath, 'utf8'); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') return null; + throw new Error( + `Could not read Discord awareness ledger ${this.legacyPath}: ${error instanceof Error ? error.message : String(error)}`, + ); + } + const sha256 = createHash('sha256').update(raw).digest('hex'); + if (state.importedSources.has(sha256)) return { records: [], sha256 }; + let parsed: unknown; + try { + parsed = JSON.parse(raw); + } catch { + throw new Error(`Invalid Discord awareness ledger ${this.legacyPath}`); + } + return { records: importLegacy(parsed, this.legacyPath, sha256), sha256 }; } - private write(document: DiscordAwarenessOutboxDocument): void { + private append(records: JournalRecord[]): void { + if (records.length === 0) return; + const journal = this.readJournal(); + const state = fold(journal.records); + const legacy = this.pendingLegacyImport(state); + const all: StoredRecord[] = legacy ? [...legacy.records, ...records] : records; + let lines: StoredRecord[] = all; + if (all.length > 1) { + const txn = randomUUID(); + lines = [ + ...all.map((record) => ({ ...record, txn })), + { t: 'commit', at: Date.now(), txn }, + ]; + } mkdirSync(dirname(this.path), { recursive: true }); - const temporary = `${this.path}.${process.pid}.${randomUUID()}.tmp`; - writeFileSync(temporary, `${JSON.stringify(document, null, 2)}\n`, { mode: 0o600 }); - renameSync(temporary, this.path); + if (journal.repair && 'truncateTo' in journal.repair) truncateSync(this.path, journal.repair.truncateTo); + const prefix = journal.repair && 'terminate' in journal.repair ? '\n' : ''; + const fd = openSync(this.path, 'a', 0o600); + try { + writeSync(fd, prefix + lines.map((record) => JSON.stringify(record)).join('\n') + '\n'); + fsyncSync(fd); + } finally { + closeSync(fd); + } + if (legacy) this.retireLegacyFile(); + } + + private retireLegacyFile(): void { + if (!existsSync(this.legacyPath)) return; + try { + renameSync(this.legacyPath, `${this.legacyPath}.migrated-v2`); + } catch (error) { + // The import is already recorded (by hash), so the file is inert; the + // next write retries the rename. + console.error( + `[discord-awareness] imported ${basename(this.legacyPath)} but could not rename it: ` + + `${error instanceof Error ? error.message : String(error)}`, + ); + } } } -function setDesired(ref: DiscordAwarenessEntry, desired: boolean): void { - const changed = ref.desired !== desired; - ref.desired = desired; - if (ref.markerPresent === desired) { - ref.deliveryStatus = 'applied'; - return; +// --------------------------------------------------------------------------- +// Fold and legacy import +// --------------------------------------------------------------------------- + +function fold(stored: StoredRecord[]): JournalState { + const state: JournalState = { batches: new Map(), ops: new Map(), importedSources: new Set() }; + // Records of a transaction apply, in order, at its commit; an uncommitted + // transaction (a torn or interrupted append) never applies. + const open = new Map(); + const records: JournalRecord[] = []; + for (const record of stored) { + if (record.t === 'commit') { + records.push(...(open.get(record.txn) ?? [])); + open.delete(record.txn); + } else if (record.txn) { + const buffered = open.get(record.txn); + if (buffered) buffered.push(record); + else open.set(record.txn, [record]); + } else { + records.push(record); + } } - const action: DiscordAwarenessAction = desired ? 'add' : 'remove'; - if (changed || ref.lastAction !== action || ref.deliveryStatus !== 'permanent-failure') { - ref.deliveryStatus = 'pending'; + for (const record of records) { + switch (record.t) { + case 'batch': + if (!state.batches.has(record.batch.id)) { + state.batches.set(record.batch.id, { ...structuredClone(record.batch), status: 'prepared' }); + } + break; + case 'activated': { + const batch = state.batches.get(record.batchId); + if (batch && batch.status !== 'discarded') batch.status = 'active'; + break; + } + case 'discarded': { + const batch = state.batches.get(record.batchId); + if (batch && batch.status === 'prepared') batch.status = 'discarded'; + break; + } + case 'held': { + const batch = state.batches.get(record.batchId); + if (batch && !batch.cancelled) { + batch.status = 'held'; + batch.held = { reason: record.reason, at: record.at, releaseActions: structuredClone(record.releaseActions) }; + } + break; + } + case 'released': { + const batch = state.batches.get(record.batchId); + if (batch && batch.status === 'held') { + batch.status = 'active'; + delete batch.held; + batch.released = { at: record.at, ...(record.by ? { by: record.by } : {}) }; + } + break; + } + case 'cancelled': { + const batch = state.batches.get(record.batchId); + if (batch && !batch.cancelled) { + if (batch.status === 'held') { + batch.status = 'active'; + delete batch.held; + } + batch.cancelled = { at: record.at, ...(record.by ? { by: record.by } : {}) }; + } + break; + } + case 'requested': + if (!state.ops.has(record.opId)) { + state.ops.set(record.opId, { + opId: record.opId, + key: { ...record.key }, + action: record.action, + cause: { ...record.cause }, + requestedAt: record.at, + attempts: [], + }); + } + break; + case 'op-cancelled': { + const op = state.ops.get(record.opId); + if (op && !op.cancelled) op.cancelled = { at: record.at, reason: record.reason }; + break; + } + case 'dispatching': { + const op = state.ops.get(record.opId); + if (op && !op.attempts.some((attempt) => attempt.attempt === record.attempt)) { + op.attempts.push({ + attempt: record.attempt, + ...(record.dispatchId ? { dispatchId: record.dispatchId } : {}), + dispatchedAt: record.at, + }); + } + break; + } + case 'outcome': { + const op = state.ops.get(record.opId); + const attempt = op?.attempts.find((candidate) => candidate.attempt === record.attempt); + if (attempt && attempt.outcome === undefined) { + attempt.outcome = record.outcome; + attempt.outcomeAt = record.at; + if (record.permanent) attempt.permanent = true; + if (record.error !== undefined) attempt.error = record.error; + } + break; + } + case 'suppression-complete': { + const batch = state.batches.get(record.batchId); + if (batch) batch.suppressionComplete = true; + break; + } + case 'import-complete': + state.importedSources.add(record.sha256); + break; + default: + break; + } } + return state; +} + +interface LegacyEntry extends DiscordAwarenessRef { + desired?: boolean; + markerPresent?: boolean; + deliveryStatus?: 'pending' | 'applied' | 'permanent-failure'; + attempts?: number; + lastAction?: DiscordAwarenessAction; + lastAttemptAt?: number; + lastError?: string; } -function isTargetActive( - activeName: string, - targetName: string, - branches: BranchDescriptor[], -): boolean { - if (activeName === targetName) return true; - const byName = new Map(branches.map((branch) => [branch.name, branch])); - const byId = new Map(branches.map((branch) => [branch.id, branch])); - let cursor = byName.get(activeName); - const visited = new Set(); - while (cursor?.parentId && !visited.has(cursor.id)) { - visited.add(cursor.id); - cursor = byId.get(cursor.parentId); - if (cursor?.name === targetName) return true; - } - return false; +interface LegacyBatch { + id: string; + status: 'prepared' | 'active' | 'pending'; + agentName: string; + sourceBranch: string; + targetBranch: string; + emoji: string; + createdAt: number; + refs: LegacyEntry[]; + activationPolicy?: 'target-branch' | 'explicit'; + suppressionIntervals?: DiscordSuppressionInterval[]; } -function findEntry( - document: DiscordAwarenessOutboxDocument, - batchId: string, - ref: DiscordAwarenessRef, -): DiscordAwarenessEntry | undefined { - return document.batches - .find((batch) => batch.id === batchId) - ?.refs.find((candidate) => refKey(candidate) === refKey(ref)); +/** + * Translate a version 1 or 2 ledger into journal records with deterministic + * identities, ending with an `import-complete` record carrying the file's + * hash. What the old ledger recorded is kept as it was recorded; nothing is + * inferred about Discord. Pending work, and batches still prepared, are held + * for an operator: they were queued before marks were an explicit choice. + */ +function importLegacy(parsed: unknown, path: string, sha256: string): JournalRecord[] { + const document = parsed as { version?: unknown; batches?: unknown }; + if (!document || (document.version !== 1 && document.version !== 2) || !Array.isArray(document.batches)) { + throw new Error(`Invalid Discord awareness ledger ${path}`); + } + const at = Date.now(); + const records: JournalRecord[] = []; + for (const raw of document.batches as LegacyBatch[]) { + if (!raw || typeof raw.id !== 'string' || !Array.isArray(raw.refs)) { + throw new Error(`Invalid Discord awareness ledger ${path}`); + } + const legacyActive = raw.status === 'active' || raw.status === 'pending'; + const refs = raw.refs.map((entry) => ({ + serverId: entry.serverId, + channelId: entry.channelId, + messageId: entry.messageId, + })); + records.push({ + t: 'batch', + at: raw.createdAt, + batch: { + id: raw.id, + agentName: raw.agentName, + sourceBranch: raw.sourceBranch, + targetBranch: raw.targetBranch, + emoji: raw.emoji, + createdAt: raw.createdAt, + scope: 'legacy', + refs, + ...(raw.activationPolicy ? { activationPolicy: raw.activationPolicy } : {}), + ...(raw.suppressionIntervals?.length ? { suppressionIntervals: raw.suppressionIntervals } : {}), + }, + }); + if (legacyActive) records.push({ t: 'activated', at: raw.createdAt, batchId: raw.id }); + const releaseActions: DiscordAwarenessReleaseAction[] = []; + for (const entry of raw.refs) { + const key = { serverId: entry.serverId, channelId: entry.channelId, messageId: entry.messageId, emoji: raw.emoji }; + // v1 entries carry no delivery state: an active v1 batch was all pending. + const status = entry.deliveryStatus ?? (raw.status === 'pending' ? 'pending' : 'applied'); + const desired = entry.desired ?? raw.status === 'pending'; + if (!legacyActive || status === 'pending') { + if (!legacyActive || desired || entry.markerPresent) { + releaseActions.push({ + ...key, + action: desired || !legacyActive ? 'add' : 'remove', + ...(entry.attempts ? { priorAttempts: entry.attempts } : {}), + ...(entry.lastError ? { priorError: entry.lastError } : {}), + }); + } + continue; + } + if (!entry.attempts || !entry.lastAction) continue; + const opId = `v2:${raw.id}:${markKey(key)}`; + const attemptAt = entry.lastAttemptAt ?? raw.createdAt; + records.push({ t: 'requested', at: raw.createdAt, opId, key, action: entry.lastAction, cause: { kind: 'import', batchId: raw.id } }); + records.push({ t: 'dispatching', at: attemptAt, opId, attempt: 1 }); + records.push(status === 'applied' + ? { t: 'outcome', at: attemptAt, opId, attempt: 1, outcome: 'confirmed' } + : { + t: 'outcome', + at: attemptAt, + opId, + attempt: 1, + outcome: 'failed', + permanent: true, + ...(entry.lastError ? { error: `recorded by an earlier ledger: ${entry.lastError}` } : {}), + }); + } + if (releaseActions.length > 0) { + records.push({ + t: 'held', + at, + batchId: raw.id, + reason: legacyActive + ? 'queued by an earlier release, before marks were an explicit choice' + : 'prepared by an earlier release, before marks were an explicit choice', + releaseActions, + }); + } + } + records.push({ t: 'import-complete', at, source: 'v2', sha256 }); + return records; } +// --------------------------------------------------------------------------- +// Keys +// --------------------------------------------------------------------------- + function dedupeRefs(refs: DiscordAwarenessRef[]): DiscordAwarenessRef[] { const deduped = new Map(); for (const ref of refs) { if (!ref.serverId || !ref.channelId || !ref.messageId) continue; - deduped.set(refKey(ref), { ...ref }); + deduped.set(refKey(ref), { serverId: ref.serverId, channelId: ref.channelId, messageId: ref.messageId }); } return [...deduped.values()]; } -function refKey(ref: DiscordAwarenessRef): string { - return `${ref.serverId}\0${ref.channelId}\0${ref.messageId}`; -} - -function markerKey(emoji: string, ref: DiscordAwarenessRef): string { - return `${emoji}\0${refKey(ref)}`; +function uniqueKeys(keys: Array): Array { + const unique = new Map(); + for (const key of keys) unique.set(markKey(key), key); + return [...unique.values()]; } -function isEntry(value: unknown): value is DiscordAwarenessEntry { - if (!value || typeof value !== 'object') return false; - const entry = value as Partial; - return typeof entry.serverId === 'string' - && typeof entry.channelId === 'string' - && typeof entry.messageId === 'string' - && typeof entry.desired === 'boolean' - && typeof entry.markerPresent === 'boolean' - && (entry.deliveryStatus === 'pending' - || entry.deliveryStatus === 'applied' - || entry.deliveryStatus === 'permanent-failure') - && typeof entry.attempts === 'number' - && (entry.lastAction === undefined || entry.lastAction === 'add' || entry.lastAction === 'remove') - && (entry.lastAttemptAt === undefined || typeof entry.lastAttemptAt === 'number') - && (entry.lastError === undefined || typeof entry.lastError === 'string'); -} - -function isDocumentV2(value: unknown): value is DiscordAwarenessOutboxDocument { - if (!value || typeof value !== 'object') return false; - const document = value as { version?: unknown; batches?: unknown }; - if (document.version !== 2 || !Array.isArray(document.batches)) return false; - return document.batches.every((batch) => { - if (!batch || typeof batch !== 'object') return false; - const candidate = batch as Partial; - return typeof candidate.id === 'string' - && (candidate.status === 'prepared' || candidate.status === 'active') - && typeof candidate.agentName === 'string' - && typeof candidate.sourceBranch === 'string' - && typeof candidate.targetBranch === 'string' - && typeof candidate.emoji === 'string' - && typeof candidate.createdAt === 'number' - && Array.isArray(candidate.refs) - && (candidate.activationPolicy === undefined - || candidate.activationPolicy === 'target-branch' - || candidate.activationPolicy === 'explicit') - && (candidate.suppressionIntervals === undefined - || (Array.isArray(candidate.suppressionIntervals) - && candidate.suppressionIntervals.every((interval) => - !!interval - && typeof interval.fromId === 'string' - && typeof interval.toId === 'string'))) - && candidate.refs.every(isEntry); - }); +function refKey(ref: DiscordAwarenessRef): string { + return `${ref.serverId}\0${ref.channelId}\0${ref.messageId}`; } -interface LegacyDocument { - version: 1; - batches: Array<{ - id: string; - status: 'prepared' | 'pending'; - agentName: string; - sourceBranch: string; - targetBranch: string; - emoji: string; - createdAt: number; - refs: DiscordAwarenessRef[]; - activationPolicy?: 'target-branch' | 'explicit'; - }>; +function markKey(key: DiscordAwarenessRef & { emoji: string }): string { + return `${key.emoji}\0${refKey(key)}`; } -function isDocumentV1(value: unknown): value is LegacyDocument { - if (!value || typeof value !== 'object') return false; - const document = value as { version?: unknown; batches?: unknown }; - if (document.version !== 1 || !Array.isArray(document.batches)) return false; - return document.batches.every((batch) => { - if (!batch || typeof batch !== 'object') return false; - const candidate = batch as LegacyDocument['batches'][number]; - return typeof candidate.id === 'string' - && (candidate.status === 'prepared' || candidate.status === 'pending') - && typeof candidate.agentName === 'string' - && typeof candidate.sourceBranch === 'string' - && typeof candidate.targetBranch === 'string' - && typeof candidate.emoji === 'string' - && typeof candidate.createdAt === 'number' - && Array.isArray(candidate.refs) - && candidate.refs.every((ref) => !!ref - && typeof ref.serverId === 'string' - && typeof ref.channelId === 'string' - && typeof ref.messageId === 'string'); - }); +/** Unresolved requests among these operations' attempts, one per dispatch. */ +function unresolvedDispatches( + ops: DiscordAwarenessOp[], + unresolved: (op: DiscordAwarenessOp, attempt: DiscordAwarenessAttempt) => boolean, +): number { + const dispatches = new Set(); + for (const op of ops) { + for (const attempt of op.attempts) { + if (unresolved(op, attempt)) dispatches.add(attempt.dispatchId ?? `${op.opId}#${attempt.attempt}`); + } + } + return dispatches.size; } -function migrateV1(document: LegacyDocument): DiscordAwarenessOutboxDocument { - return { - version: 2, - batches: document.batches.map((batch) => ({ - ...batch, - status: batch.status === 'pending' ? 'active' : 'prepared', - refs: batch.refs.map((ref) => ({ - ...ref, - desired: batch.status === 'pending', - markerPresent: false, - deliveryStatus: batch.status === 'pending' ? 'pending' : 'applied', - attempts: 0, - })), - })), - }; +function emptyCounts(): Record { + return { requested: 0, dispatching: 0, confirmed: 0, failed: 0, unknown: 0, cancelled: 0 }; } diff --git a/src/recovery/offline-branch.ts b/src/recovery/offline-branch.ts index 6c2a202c8..7045d4392 100644 --- a/src/recovery/offline-branch.ts +++ b/src/recovery/offline-branch.ts @@ -5,7 +5,10 @@ import { DiscordAwarenessOutbox, defaultDiscordAwarenessOutboxPath, extractDiscordAwarenessRefs, + selectDiscordAwarenessRefs, + type DiscordAwarenessMarks, type DiscordAwarenessRef, + type DiscordMessageMetadataCarrier, } from './discord-awareness-outbox.js'; export interface OfflineRecoveryBranchOptions { @@ -28,6 +31,11 @@ export interface OfflineRecoveryBranchOptions { discordServerId?: string; outboxPath?: string; emoji?: string; + /** + * Awareness marks on the removed Discord messages: an explicit publication + * choice, `none` by default. The recovery itself stays local either way. + */ + marks?: DiscordAwarenessMarks; dryRun?: boolean; } @@ -37,8 +45,18 @@ export interface OfflineRecoveryBranchResult { targetBranch: string; messagesRemoved: number; messagesSuppressed: number; + /** Removed messages that carry a Discord address. */ + discordAddressable: number; + /** The publication scope recorded for this recovery. */ + marksScope: 'none' | 'addressed' | 'all'; + /** Marks requested (queued once the branch is activated); not delivered. */ discordMarkersQueued: number; + /** The refs that will be marked (empty unless marks were chosen). */ refs: DiscordAwarenessRef[]; + /** Removed addressable messages left unmarked. */ + unmarked: number; + /** Authorized refs (marks.refs) this recovery does not remove. */ + notRemoved: number; outboxPath: string; } @@ -81,7 +99,7 @@ export async function createOfflineRecoveryBranch( let target: StoredWindowMessage; let targetIndex: number; let removedCount: number; - let refs: DiscordAwarenessRef[]; + let carriers: DiscordMessageMetadataCarrier[]; if (options.messageId) { // Search backward in bounded windows: recovery anchors are normally near @@ -98,12 +116,7 @@ export async function createOfflineRecoveryBranch( target = located.message; targetIndex = located.index; removedCount = total - located.index - 1; - refs = collectDiscordRefs( - contextManager, - located.index + 1, - total, - options.discordServerId, - ); + carriers = collectDiscordCarriers(contextManager, located.index + 1, total); } else if (options.contextId) { const located = findInternalMessageFromTail(contextManager, total, options.contextId.trim()); if (!located) { @@ -112,12 +125,7 @@ export async function createOfflineRecoveryBranch( target = located.message; targetIndex = located.index; removedCount = total - located.index - 1; - refs = collectDiscordRefs( - contextManager, - located.index + 1, - total, - options.discordServerId, - ); + carriers = collectDiscordCarriers(contextManager, located.index + 1, total); } else { // Count mode remains for compatibility. Read only the target plus suffix, // avoiding full-history materialization and attachment blob inflation. @@ -132,7 +140,7 @@ export async function createOfflineRecoveryBranch( target = countTarget; targetIndex = total - count! - 1; removedCount = count!; - refs = extractDiscordAwarenessRefs(discarded, options.discordServerId); + carriers = discarded.map(addressingOnly); } validateAnchorBoundary(contextManager, targetIndex, total, target); @@ -142,7 +150,6 @@ export async function createOfflineRecoveryBranch( targetIndex, options.suppressMessageIds ?? [], options.suppressRanges ?? [], - options.discordServerId, ); validateRemovalIntegrity( contextManager, @@ -157,7 +164,11 @@ export async function createOfflineRecoveryBranch( ...suppression.intervals, ], ); - refs = dedupeDiscordRefs([...refs, ...suppression.refs]); + carriers = [...carriers, ...suppression.carriers]; + const addressable = extractDiscordAwarenessRefs(carriers, options.discordServerId).length; + const marks = options.marks ?? 'none'; + const selection = selectDiscordAwarenessRefs(carriers, marks, options.discordServerId); + const refs = selection.refs; const sourceBranch = store.currentBranch().name; const targetBranch = options.branchName @@ -174,8 +185,12 @@ export async function createOfflineRecoveryBranch( targetBranch, messagesRemoved: removedCount, messagesSuppressed: suppression.messageCount, + discordAddressable: addressable, + marksScope: marks === 'none' ? 'none' : marks.scope, discordMarkersQueued: refs.length, refs, + unmarked: selection.unmarked, + notRemoved: selection.notRemoved, outboxPath, }; if (options.dryRun) return result; @@ -186,6 +201,9 @@ export async function createOfflineRecoveryBranch( sourceBranch, targetBranch, refs, + scope: marks === 'none' ? 'none' : marks.scope, + unmarked: selection.unmarked, + notRemoved: selection.notRemoved, emoji: options.emoji ?? DEFAULT_DISCORD_AWARENESS_EMOJI, // Merely seeing targetBranch active does not prove branch-local // suppressions finished. Only the recovery operation may activate this @@ -368,8 +386,7 @@ function buildSuppressionPlan( targetIndex: number, rawMessageIds: string[], rawRanges: Array<{ fromMessageId: string; toMessageId: string }>, - forcedServerId?: string, -): { intervals: SuppressionInterval[]; refs: DiscordAwarenessRef[]; messageCount: number } { +): { intervals: SuppressionInterval[]; carriers: DiscordMessageMetadataCarrier[]; messageCount: number } { const messageIds = rawMessageIds.map(normalizeDiscordMessageId); const ranges = rawRanges.map((range) => ({ fromMessageId: normalizeDiscordMessageId(range.fromMessageId), @@ -379,7 +396,7 @@ function buildSuppressionPlan( ...messageIds, ...ranges.flatMap((range) => [range.fromMessageId, range.toMessageId]), ]); - if (wanted.size === 0) return { intervals: [], refs: [], messageCount: 0 }; + if (wanted.size === 0) return { intervals: [], carriers: [], messageCount: 0 }; const locations = new Map(); for (let offset = 0; offset <= targetIndex; offset += RECOVERY_SCAN_WINDOW) { @@ -466,45 +483,48 @@ function buildSuppressionPlan( } } - const refs: DiscordAwarenessRef[] = []; + const carriers: DiscordMessageMetadataCarrier[] = []; for (const interval of merged) { - refs.push(...collectDiscordRefs( - contextManager, - interval.start, - interval.end + 1, - forcedServerId, - )); + carriers.push(...collectDiscordCarriers(contextManager, interval.start, interval.end + 1)); } return { intervals: merged, - refs: dedupeDiscordRefs(refs), + carriers, messageCount: merged.reduce((sum, interval) => sum + interval.end - interval.start + 1, 0), }; } -function collectDiscordRefs( +/** + * The addressing metadata of removed messages, read in bounded windows: only + * the Discord address and the MCPL tags (for `addressed` selection) are kept, + * never content. A sharded message may repeat its metadata across adjacent + * records; selection dedupes by address. + */ +function collectDiscordCarriers( contextManager: ContextManager, start: number, end: number, - forcedServerId?: string, -): DiscordAwarenessRef[] { - const refs: DiscordAwarenessRef[] = []; +): DiscordMessageMetadataCarrier[] { + const carriers: DiscordMessageMetadataCarrier[] = []; for (let offset = start; offset < end; offset += RECOVERY_SCAN_WINDOW) { const messages = contextManager.getMessageWindow( offset, Math.min(RECOVERY_SCAN_WINDOW, end - offset), { resolveBlobs: false }, ).messages; - refs.push(...extractDiscordAwarenessRefs(messages, forcedServerId)); + carriers.push(...messages.map(addressingOnly)); } - // A sharded message may repeat Discord metadata across adjacent records. - return dedupeDiscordRefs(refs); + return carriers; } -function dedupeDiscordRefs(refs: DiscordAwarenessRef[]): DiscordAwarenessRef[] { - const deduped = new Map(); - for (const ref of refs) { - deduped.set(`${ref.serverId}\0${ref.channelId}\0${ref.messageId}`, ref); - } - return [...deduped.values()]; +function addressingOnly(message: { metadata?: Record }): DiscordMessageMetadataCarrier { + const metadata = message.metadata ?? {}; + return { + metadata: { + serverId: metadata.serverId, + channelId: metadata.channelId, + messageId: metadata.messageId, + tags: metadata.tags, + }, + }; } diff --git a/src/recovery/recover-cli.ts b/src/recovery/recover-cli.ts index e77556a15..99c642c0e 100644 --- a/src/recovery/recover-cli.ts +++ b/src/recovery/recover-cli.ts @@ -1,6 +1,11 @@ #!/usr/bin/env node import { createOfflineRecoveryBranch } from './offline-branch.js'; +import { + DiscordAwarenessOutbox, + defaultDiscordAwarenessOutboxPath, + type DiscordAwarenessMarks, +} from './discord-awareness-outbox.js'; interface CliOptions { storePath?: string; @@ -15,16 +20,21 @@ interface CliOptions { discordServerId?: string; outboxPath?: string; emoji?: string; + marks: 'none' | 'addressed' | 'all'; + awareness?: { verb: 'list' | 'cancel' | 'retract' | 'release'; target?: string }; dryRun: boolean; } function usage(): string { return `Usage: agent-framework-recover --store --agent [options] + agent-framework-recover --store --awareness -Creates and activates a Chronicle recovery branch while the normal host is down, -then queues Discord reactions for addressable messages removed from awareness. -Message content is never compiled or written to the recovery outbox. +Creates and activates a Chronicle recovery branch while the normal host is +down. The recovery is local to the resident: by default no Discord reaction is +placed. --marks chooses to mark removed Discord messages with an awareness +reaction (queued here, delivered when the host and its Discord MCPL server +next connect). Message content is never compiled or written to the journal. Options: --message-id Last Discord message the agent should still see @@ -35,16 +45,30 @@ Options: --branch Recovery branch name (generated by default) --namespace Context namespace (default: agents/) --discord-server Override/inject the Discord MCPL server id - --outbox Override the branch-independent outbox path + --marks + Awareness marks on removed Discord messages + (default none; addressed = mentions, replies to + the bot and DMs; all = every addressable message) + --outbox Override the awareness journal path --emoji Marker reaction (default: 💤) --dry-run Inspect counts/addresses without writing --help Show this help +Awareness journal (host stopped): + --awareness list Each batch: scope, counts by status, holds + --awareness cancel Stop a batch's unsent marks (never removes any) + --awareness retract + Queue removal of this bot's marks (through each + ref's configured MCPL route) on a batch's refs, + or on every ref an add was ever sent for + --awareness release Queue a held batch's recorded operations + +Queued work is delivered when the host next connects to the Discord server. The agent host must be stopped before running this command.`; } function parseArgs(args: string[]): CliOptions { - const options: CliOptions = { dryRun: false, suppressMessageIds: [], suppressRanges: [] }; + const options: CliOptions = { dryRun: false, suppressMessageIds: [], suppressRanges: [], marks: 'none' }; for (let i = 0; i < args.length; i++) { const arg = args[i]; const value = () => { @@ -80,6 +104,25 @@ function parseArgs(args: string[]): CliOptions { case '--discord-server': options.discordServerId = value(); break; case '--outbox': options.outboxPath = value(); break; case '--emoji': options.emoji = value(); break; + case '--marks': { + const marks = value(); + if (marks !== 'none' && marks !== 'addressed' && marks !== 'all') { + throw new Error('--marks must be none, addressed or all'); + } + options.marks = marks; + break; + } + case '--awareness': { + const verb = value(); + if (verb === 'list') { + options.awareness = { verb }; + } else if (verb === 'cancel' || verb === 'retract' || verb === 'release') { + options.awareness = { verb, target: value() }; + } else { + throw new Error('--awareness must be list, cancel , retract or release '); + } + break; + } case '--dry-run': options.dryRun = true; break; case '--help': case '-h': @@ -91,8 +134,31 @@ function parseArgs(args: string[]): CliOptions { return options; } +function awareness(options: CliOptions): void { + const outbox = new DiscordAwarenessOutbox( + options.outboxPath ?? defaultDiscordAwarenessOutboxPath(options.storePath!), + ); + const { verb, target } = options.awareness!; + const by = 'agent-framework-recover'; + let result: unknown; + switch (verb) { + case 'list': result = { journal: outbox.path, batches: outbox.view() }; break; + case 'cancel': result = outbox.cancel(target!, by); break; + case 'retract': result = outbox.retract(target!, by); break; + case 'release': result = outbox.release(target!, by); break; + } + console.log(JSON.stringify(result, null, 2)); +} + async function main(): Promise { const options = parseArgs(process.argv.slice(2)); + if (options.awareness) { + if (!options.storePath && !options.outboxPath) { + throw new Error(`--awareness needs --store (or --outbox)\n\n${usage()}`); + } + awareness(options); + return; + } if ( !options.storePath || !options.agentName @@ -101,6 +167,9 @@ async function main(): Promise { throw new Error(`--store, --agent, and one anchor (--message-id, --context-id, or --messages) are required\n\n${usage()}`); } + // A CLI choice and its application are the same moment (the host is + // stopped), so the scope is evaluated now rather than frozen to refs. + const marks: DiscordAwarenessMarks = options.marks === 'none' ? 'none' : { scope: options.marks }; const result = await createOfflineRecoveryBranch({ storePath: options.storePath, agentName: options.agentName, @@ -114,6 +183,7 @@ async function main(): Promise { discordServerId: options.discordServerId, outboxPath: options.outboxPath, emoji: options.emoji, + marks, dryRun: options.dryRun, }); diff --git a/test/discord-awareness-delivery.test.ts b/test/discord-awareness-delivery.test.ts new file mode 100644 index 000000000..137afb045 --- /dev/null +++ b/test/discord-awareness-delivery.test.ts @@ -0,0 +1,207 @@ +/** + * Awareness-mark delivery never gates the resident. + * + * Before: queued marks held every MCPL data plane, at startup, reconnect and + * after surgery, until each reaction had been attempted; at Discord's pace a + * large batch left the resident deaf for many minutes, and an unreadable + * ledger aborted startup or recycled connections at runtime. Now delivery is + * a background drain over the awareness journal: traffic flows while marks + * are delivered; a route that is down leaves its work queued; the mandatory + * per-request deadline still turns a hung reaction into an unknown outcome. + * The journal is still read before startup, because it also carries the + * suppression resume journal that protects the resident's body. + */ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { fileURLToPath } from 'node:url'; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { AgentFramework } from '../src/framework.js'; +import { DiscordAwarenessOutbox } from '../src/recovery/discord-awareness-outbox.js'; +import { MockMembrane } from './helpers/mock-membrane.js'; + +const FIXTURE = fileURLToPath(new URL('./fixtures/surgery-marks-mcpl-server.mjs', import.meta.url)); + +function jsonl(path: string): Array> { + if (!existsSync(path)) return []; + return readFileSync(path, 'utf8').split('\n').filter(Boolean).map((line) => JSON.parse(line)); +} + +async function waitFor(description: string, predicate: () => boolean, timeoutMs = 5_000): Promise { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + if (predicate()) return; + await new Promise((resolve) => setTimeout(resolve, 10)); + } + assert.fail(`timed out waiting for ${description}`); +} + +function setup() { + const dir = mkdtempSync(join(tmpdir(), 'awareness-delivery-')); + const storePath = join(dir, 'store'); + // Outside the store: the store directory must not exist before Chronicle + // creates it, and these tests seed the journal before the first start. + const journal = join(dir, 'awareness', 'discord-awareness-journal.jsonl'); + const paths = { + calls: join(dir, 'calls.jsonl'), + events: join(dir, 'events.jsonl'), + hold: join(dir, 'release'), + probe: join(dir, 'probe'), + }; + const queueMarks = (messageIds: string[]) => { + const outbox = new DiscordAwarenessOutbox(journal); + const batch = outbox.prepare({ + agentName: 'resident', + sourceBranch: 'source', + targetBranch: 'main', + refs: messageIds.map((messageId) => ({ serverId: 'discord', channelId: 'discord:g1:c1', messageId })), + scope: 'all', + activationPolicy: 'explicit', + })!; + outbox.activate(batch.id); + return outbox; + }; + const config = (opts: { discord?: boolean; requestTimeoutMs?: number; awarenessDeadlineMs?: number } = {}) => ({ + storePath, + membrane: new MockMembrane().asMembrane(), + agents: [{ name: 'resident', model: 'test', systemPrompt: 'test' }], + modules: [], + maintenanceIntervalMs: 0, + discordAwarenessOutboxPath: journal, + ...(opts.awarenessDeadlineMs ? { discordAwarenessDeadlineMs: opts.awarenessDeadlineMs } : {}), + ...(opts.discord === false ? {} : { + mcplServers: [{ + id: 'discord', + command: process.execPath, + args: [FIXTURE], + ...(opts.requestTimeoutMs !== undefined ? { requestTimeoutMs: opts.requestTimeoutMs } : {}), + env: { CALLS_PATH: paths.calls, EVENTS_PATH: paths.events, HOLD_PATH: paths.hold, PROBE_PATH: paths.probe }, + enabledFeatureSets: ['chat'], + }], + }), + }); + return { dir, storePath, journal, paths, queueMarks, config }; +} + +test('startup does not wait for queued marks, and channel traffic flows while they are held', async () => { + const { dir, journal, paths, queueMarks, config } = setup(); + let framework: AgentFramework | undefined; + try { + queueMarks(['m1', 'm2']); + writeFileSync(paths.probe, '1'); // the server sends channel traffic as soon as it can + framework = await AgentFramework.create(config()); + await waitFor('the queued add to reach the server', () => jsonl(paths.calls).length >= 1); + await waitFor('the channel traffic to be answered', () => + jsonl(paths.events).some((event) => event.event === 'probe-answered')); + assert.equal( + jsonl(paths.events).filter((event) => event.event === 'reaction-answered').length, + 0, + 'answered while every reaction was still withheld', + ); + writeFileSync(paths.hold, '1'); + await waitFor('both adds confirmed', () => { + const outbox = new DiscordAwarenessOutbox(journal); + return outbox.operations().filter((op) => outbox.operationStatus(op) === 'confirmed').length === 2; + }); + } finally { + writeFileSync(paths.hold, '1'); + await framework?.stop(); + rmSync(dir, { recursive: true, force: true }); + } +}); + +test('the mandatory awareness deadline still bounds a hung reaction with requestTimeoutMs: 0', async () => { + const { dir, journal, paths, queueMarks, config } = setup(); + let framework: AgentFramework | undefined; + try { + queueMarks(['m1']); + framework = await AgentFramework.create(config({ requestTimeoutMs: 0, awarenessDeadlineMs: 150 })); + // Another reader sees an unanswered dispatch as unknown at once; wait + // for the drain's own outcome record. + await waitFor('the hung add to be recorded unknown', () => + new DiscordAwarenessOutbox(journal).operations().some((op) => op.attempts[0]?.outcome === 'unknown')); + const outbox = new DiscordAwarenessOutbox(journal); + const [op] = outbox.operations(); + assert.equal(op.attempts[0].outcome, 'unknown'); + assert.match(op.attempts[0].error ?? '', /did not respond/); + } finally { + writeFileSync(paths.hold, '1'); + await framework?.stop(); + rmSync(dir, { recursive: true, force: true }); + } +}); + +test('a route that is not connected leaves marks queued; they go out once it is', async () => { + const { dir, journal, paths, queueMarks, config } = setup(); + let framework: AgentFramework | undefined; + try { + queueMarks(['m1']); + const before = readFileSync(journal, 'utf8'); + framework = await AgentFramework.create(config({ discord: false })); + await framework.syncDiscordAwarenessMarkers(); + assert.equal(readFileSync(journal, 'utf8'), before, 'nothing recorded for a route that is down'); + await framework.stop(); + framework = undefined; + + writeFileSync(paths.hold, '1'); + framework = await AgentFramework.create(config()); + await waitFor('the queued add confirmed', () => { + const outbox = new DiscordAwarenessOutbox(journal); + return outbox.operations().every((op) => outbox.operationStatus(op) === 'confirmed'); + }); + assert.deepEqual(jsonl(paths.calls).map((call) => call.name), ['add_reaction']); + } finally { + writeFileSync(paths.hold, '1'); + await framework?.stop(); + rmSync(dir, { recursive: true, force: true }); + } +}); + +test('a corrupt journal stops startup, since it may hold a suppression the body needs resumed', async () => { + const { dir, journal, config } = setup(); + try { + mkdirSync(join(journal, '..'), { recursive: true }); + writeFileSync(journal, '{"t":"batch",\nnot json at all\n{"t":"commit","at":1,"txn":"x"}\n'); + await assert.rejects( + AgentFramework.create(config({ discord: false })), + /Discord awareness accounting failed during startup reconciliation: Corrupt Discord awareness journal/, + ); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); + +test('a branch switch neither adds nor removes marks', async () => { + const { dir, journal, paths, config } = setup(); + let framework: AgentFramework | undefined; + try { + writeFileSync(paths.hold, '1'); + framework = await AgentFramework.create(config()); + const cm = framework.getAgent('resident')!.getContextManager(); + cm.addMessage('Operator', [{ type: 'text', text: 'kept' }], { serverId: 'discord', channelId: 'discord:g1:c1', messageId: 'kept' }); + const tail = String(cm.addMessage('Operator', [{ type: 'text', text: 'tail' }], { + serverId: 'discord', channelId: 'discord:g1:c1', messageId: 'tail', + })); + cm.addMessage('Member', [{ type: 'text', text: 'hi' }], { + serverId: 'discord', channelId: 'discord:g1:c1', messageId: 'm1', tags: ['chat:addressed'], + }); + const result = await framework.rollbackToMessage('resident', { messageId: tail, marks: { scope: 'addressed' } }); + await waitFor('the add confirmed', () => + new DiscordAwarenessOutbox(journal).operations().some((op) => op.attempts[0]?.outcome === 'confirmed')); + // Switch back to the source and forward again: nothing is sent. + await cm.switchBranch(result.sourceBranch); + await framework.syncDiscordAwarenessMarkers(); + await cm.switchBranch(result.targetBranch); + await framework.syncDiscordAwarenessMarkers(); + await framework.stop(); + framework = await AgentFramework.create(config()); + await framework.syncDiscordAwarenessMarkers(); + assert.deepEqual(jsonl(paths.calls).map((call) => call.name), ['add_reaction']); + assert.equal(new DiscordAwarenessOutbox(journal).operations().length, 1); + } finally { + writeFileSync(paths.hold, '1'); + await framework?.stop(); + rmSync(dir, { recursive: true, force: true }); + } +}); diff --git a/test/discord-awareness-outbox.test.ts b/test/discord-awareness-outbox.test.ts index 4a35a21f5..ba8e1b9b6 100644 --- a/test/discord-awareness-outbox.test.ts +++ b/test/discord-awareness-outbox.test.ts @@ -1,15 +1,75 @@ +/** + * The Discord awareness-mark journal: an append-only record of what was + * asked and what was answered, never a model of what is on Discord. + * + * Contract (room-225's proposed upstream change, after a field incident in + * which a web-UI rollback queued 918 💤 reactions on other people's messages): + * marks are an explicit publication choice (default none); one-shot, so + * branch moves and restarts never derive operations from branch state; and + * cancel / retract / release are explicit operator acts whose receipts say + * what is known and what is not. + */ import { test } from 'node:test'; import assert from 'node:assert/strict'; -import { mkdtempSync, readFileSync, rmSync } from 'node:fs'; +import { appendFileSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; import { join } from 'node:path'; import { tmpdir } from 'node:os'; import { DiscordAwarenessOutbox, extractDiscordAwarenessRefs, + selectDiscordAwarenessRefs, + type DiscordAwarenessRef, } from '../src/recovery/discord-awareness-outbox.js'; import { AgentFramework } from '../src/framework.js'; +import { McplRequestError } from '../src/mcpl/server-connection.js'; import { OperatorLog } from '../src/operator-log.js'; +const ref = (messageId: string, channel = 'c1'): DiscordAwarenessRef => + ({ serverId: 'discord', channelId: `discord:g1:${channel}`, messageId }); + +function withJournal(fn: (path: string, dir: string) => void | Promise): () => Promise { + return async () => { + const dir = mkdtempSync(join(tmpdir(), 'discord-awareness-')); + try { + await fn(join(dir, 'recovery', 'discord-awareness-journal.jsonl'), dir); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }; +} + +function activeBatch(outbox: DiscordAwarenessOutbox, refs: DiscordAwarenessRef[], extra: Record = {}) { + const batch = outbox.prepare({ + agentName: 'cairn', + sourceBranch: 'main', + targetBranch: 'rollback/cairn/1', + refs, + scope: 'all', + ...extra, + })!; + outbox.activate(batch.id); + return batch; +} + +/** Dispatch everything due, answering each dispatch with `answer`. */ +function drain( + outbox: DiscordAwarenessOutbox, + answer: (dispatch: { action: string; messageId: string }) => 'confirmed' | 'failed' | 'not-sent' | 'unknown', +): string[] { + const seen: string[] = []; + for (let pass = 0; pass < 10; pass++) { + const dispatches = outbox.pendingDispatches('discord'); + if (dispatches.length === 0) break; + for (const dispatch of dispatches) { + const attempts = outbox.recordDispatching(dispatch); + const outcome = answer({ action: dispatch.action, messageId: dispatch.key.messageId }); + seen.push(`${dispatch.action}:${dispatch.key.messageId}:${outcome}`); + outbox.recordOutcome(attempts, outcome, outcome === 'failed' ? { permanent: true, error: 'Unknown Message' } : {}); + } + } + return seen; +} + test('extractDiscordAwarenessRefs keeps only direct Discord addressing metadata', () => { const refs = extractDiscordAwarenessRefs([ { metadata: { serverId: 'discord', channelId: 'discord:g1:c1', messageId: 'm1' } }, @@ -17,104 +77,423 @@ test('extractDiscordAwarenessRefs keeps only direct Discord addressing metadata' { metadata: { serverId: 'discord', channelId: 'discord:g1:c1', messageId: 'm1' } }, { metadata: { serverId: 'discord', channelId: 'discord:g1:c1' } }, ]); - - assert.deepEqual(refs, [ - { serverId: 'discord', channelId: 'discord:g1:c1', messageId: 'm1' }, - ]); + assert.deepEqual(refs, [ref('m1')]); }); -test('ledger survives restart and reverses completed markers on source branch', () => { - const dir = mkdtempSync(join(tmpdir(), 'discord-awareness-outbox-')); - const path = join(dir, 'outbox.json'); - try { - const first = new DiscordAwarenessOutbox(path); - const batch = first.prepare({ - agentName: 'cairn', - sourceBranch: 'main', - targetBranch: 'recovery/cairn/1', - refs: [ - { serverId: 'discord', channelId: 'discord:g1:c1', messageId: 'm1' }, - { serverId: 'discord', channelId: 'discord:g1:c1', messageId: 'm2' }, - ], - }); - assert.ok(batch); - assert.equal(first.pending('discord').length, 0, 'prepared work is not delivered early'); - - // Simulate the host restarting after Chronicle switched branches but - // before the recovery process activated its batch. - const reopened = new DiscordAwarenessOutbox(path); - assert.equal(reopened.activatePreparedForBranch('some-other-branch'), 0); - assert.equal(reopened.activatePreparedForBranch('recovery/cairn/1'), 1); - assert.deepEqual( - reopened.pending('discord').map((operation) => operation.ref.messageId), - ['m1', 'm2'], - ); - - reopened.recordSuccess(batch.id, batch.refs[0], 'add'); - assert.deepEqual(reopened.pending('discord').map((op) => op.ref.messageId), ['m2']); - reopened.recordSuccess(batch.id, batch.refs[1], 'add'); - assert.equal(reopened.pending('discord').length, 0); - - // Completed work remains durable and switching back queues removals. - reopened.reconcileForBranch('main'); - const removals = reopened.pending('discord'); - assert.deepEqual(removals.map((op) => op.action), ['remove', 'remove']); - for (const operation of removals) { - reopened.recordSuccess(operation.batchId, operation.ref, operation.action); - } - assert.equal(reopened.pending('discord').length, 0); - assert.equal(reopened.batches().length, 1); - assert.ok(reopened.batches()[0].refs.every((ref) => !ref.markerPresent)); +test('selection: none marks nothing; addressed means the chat:addressed tag; refs bind the set', () => { + const messages = [ + { metadata: { ...ref('a1'), tags: ['chat:addressed', 'chat:mention'] } }, + { metadata: { ...ref('b1'), tags: ['chat:ambient'] } }, + { metadata: { ...ref('b2') } }, + { metadata: { ...ref('a2', 'dm'), tags: ['chat:addressed', 'chat:dm'] } }, + ]; + const none = selectDiscordAwarenessRefs(messages, 'none'); + assert.deepEqual(none, { refs: [], addressable: 4, unmarked: 4, notRemoved: 0 }); - const stored = JSON.parse(readFileSync(path, 'utf8')) as { version: number; batches: unknown[] }; - assert.equal(stored.version, 2); - assert.equal(stored.batches.length, 1); - } finally { - rmSync(dir, { recursive: true, force: true }); - } + const addressed = selectDiscordAwarenessRefs(messages, { scope: 'addressed' }); + assert.deepEqual(addressed.refs.map((r) => r.messageId), ['a1', 'a2']); + assert.equal(addressed.unmarked, 2); + + const all = selectDiscordAwarenessRefs(messages, { scope: 'all' }); + assert.equal(all.refs.length, 4); + + // A frozen choice: only previewed refs, and never anything else removed. + const frozen = selectDiscordAwarenessRefs(messages, { scope: 'all', refs: [ref('a1'), ref('gone')] }); + assert.deepEqual(frozen.refs.map((r) => r.messageId), ['a1']); + assert.equal(frozen.notRemoved, 1, 'an authorized ref the surgery did not remove'); + assert.equal(frozen.unmarked, 3); + + // Scope still applies to authorized refs. + const frozenAddressed = selectDiscordAwarenessRefs(messages, { scope: 'addressed', refs: [ref('b1'), ref('a2', 'dm')] }); + assert.deepEqual(frozenAddressed.refs.map((r) => r.messageId), ['a2']); }); -test('explicit batches are not activated merely because their target branch is current', () => { - const dir = mkdtempSync(join(tmpdir(), 'discord-awareness-explicit-')); - const path = join(dir, 'outbox.json'); +test('a prepared batch delivers nothing; activation queues one add per ref, written ahead of dispatch', withJournal((path) => { + const outbox = new DiscordAwarenessOutbox(path); + const batch = outbox.prepare({ + agentName: 'cairn', sourceBranch: 'main', targetBranch: 'rollback/cairn/1', refs: [ref('m1'), ref('m2')], scope: 'all', + })!; + assert.equal(outbox.pendingDispatches('discord').length, 0, 'prepared work is not deliverable'); + assert.equal(outbox.activate(batch.id), 2); + const [first] = outbox.pendingDispatches('discord'); + assert.equal(first.action, 'add'); + assert.equal(first.key.emoji, '💤'); + + const attempts = outbox.recordDispatching(first); + // The write-ahead record is durable before the request could leave. + const journal = readFileSync(path, 'utf8'); + assert.match(journal, /"t":"dispatching"/); + // A fresh reader (another process) sees it as unknown, never as unsent. + const other = new DiscordAwarenessOutbox(path); + assert.equal(other.operations().find((op) => op.opId === first.opIds[0])!.attempts.length, 1); + assert.equal(other.operationStatus(other.operations().find((op) => op.opId === first.opIds[0])!), 'unknown'); + + outbox.recordOutcome(attempts, 'confirmed'); + assert.deepEqual(outbox.pendingDispatches('discord').map((d) => d.key.messageId), ['m2']); +})); + +test('startup derives nothing from ancestry: exact-target crash completion, everything else held', withJournal((path) => { + const outbox = new DiscordAwarenessOutbox(path); + const crashed = outbox.prepare({ + agentName: 'cairn', sourceBranch: 'main', targetBranch: 'rollback/cairn/1', refs: [ref('m1')], scope: 'all', + })!; + const elsewhere = outbox.prepare({ + agentName: 'cairn', sourceBranch: 'main', targetBranch: 'rollback/cairn/2', refs: [ref('m2')], scope: 'addressed', + })!; + const suppression = outbox.prepare({ + agentName: 'cairn', sourceBranch: 'main', targetBranch: 'rollback/cairn/1', refs: [ref('m3')], scope: 'all', + activationPolicy: 'explicit', suppressionIntervals: [{ fromId: 'i1', toId: 'i1' }], + })!; + const active = activeBatch(outbox, [ref('m4')]); + const [dispatch] = outbox.pendingDispatches('discord').filter((d) => d.key.messageId === 'm4'); + outbox.recordDispatching(dispatch); // the process dies with this request on the wire + + const restarted = new DiscordAwarenessOutbox(path); + const recovered = restarted.recoverAtStartup('rollback/cairn/1'); + assert.deepEqual(recovered.activated, [crashed.id]); + assert.deepEqual(recovered.held, [elsewhere.id]); + assert.equal(recovered.unknownAttempts, 1); + const status = new Map(restarted.batches().map((b) => [b.id, b.status])); + assert.equal(status.get(crashed.id), 'active'); + assert.equal(status.get(elsewhere.id), 'held'); + assert.equal(status.get(suppression.id), 'prepared', 'left for resumePreparedDiscordSuppressions'); + assert.equal(status.get(active.id), 'active'); + // The unanswered add is unknown, and as the newest op for its key it is + // retried (idempotent); the held batch's ref is not dispatched at all. + assert.deepEqual( + restarted.pendingDispatches('discord').map((d) => d.key.messageId).sort(), + ['m1', 'm4'], + ); + + // A later restart on a different branch changes nothing. + const again = new DiscordAwarenessOutbox(path).recoverAtStartup('main'); + assert.deepEqual(again, { activated: [], held: [suppression.id], unknownAttempts: 0 }); +})); + +test('cancel stops every further send of a batch, never removes, and says what may still land', withJournal((path) => { + const outbox = new DiscordAwarenessOutbox(path); + const batch = activeBatch(outbox, [ref('m1'), ref('m2'), ref('m3'), ref('m4')]); + const dispatches = outbox.pendingDispatches('discord'); + const byId = new Map(dispatches.map((d) => [d.key.messageId, d])); + outbox.recordOutcome(outbox.recordDispatching(byId.get('m1')!), 'confirmed'); + outbox.recordOutcome(outbox.recordDispatching(byId.get('m2')!), 'unknown', { error: 'did not respond' }); + const m3Attempts = outbox.recordDispatching(byId.get('m3')!); // on the wire right now + + const receipt = outbox.cancel(batch.id, 'operator'); + assert.deepEqual( + { cancelled: receipt.cancelled, inFlight: receipt.inFlight, unknown: receipt.unknown, confirmed: receipt.confirmed }, + { cancelled: 1, inFlight: 1, unknown: 1, confirmed: 1 }, + ); + // Nothing new is sent, no removal is queued, and the unknown add is not + // retried; the key with a request on the wire stays busy meanwhile. + assert.equal(outbox.pendingDispatches('discord').length, 0); + + // The in-flight request lands after the cancel: that is recorded as what + // happened, and it is still not followed by anything. + outbox.recordOutcome(m3Attempts, 'confirmed'); + const m3 = outbox.operations().find((op) => op.key.messageId === 'm3')!; + assert.equal(outbox.operationStatus(m3), 'confirmed'); + assert.equal(outbox.pendingDispatches('discord').length, 0); + + // Cancellation is durable: a restart or a branch move re-arms nothing. + const restarted = new DiscordAwarenessOutbox(path); + restarted.recoverAtStartup('some/other/branch'); + assert.ok(restarted.batches().find((b) => b.id === batch.id)?.cancelled); + assert.equal(restarted.pendingDispatches('discord').length, 0); + // Retract still reaches what may have landed: m1, m2 (unknown) and m3. + const retracted = restarted.retract(batch.id); + assert.equal(retracted.removalsQueued, 3); + assert.equal(retracted.keysWithUnresolvedAdds, 1); +})); + +test('cancelling a batch whose surgery has not activated it stops every mark, and its body can still resume', withJournal((path) => { + const outbox = new DiscordAwarenessOutbox(path); + const rollback = outbox.prepare({ + agentName: 'cairn', sourceBranch: 'main', targetBranch: 'rollback/cairn/1', refs: [ref('m1'), ref('m2')], scope: 'all', + })!; + const receipt = outbox.cancel(rollback.id, 'operator'); + assert.equal(receipt.cancelled, 2); + assert.throws(() => outbox.activate(rollback.id), /cancelled before activation/); + // Not activated by startup's crash completion, and not held either. + assert.deepEqual(outbox.recoverAtStartup('rollback/cairn/1'), { activated: [], held: [], unknownAttempts: 0 }); + assert.equal(outbox.pendingDispatches('discord').length, 0); + + const suppression = outbox.prepare({ + agentName: 'cairn', sourceBranch: 'main', targetBranch: 'suppress/cairn/1', refs: [ref('m3')], scope: 'all', + activationPolicy: 'explicit', suppressionIntervals: [{ fromId: 'i1', toId: 'i1' }], + })!; + outbox.cancel(suppression.id); + // The suppression's body still resumes on its branch; its marks never go. + assert.deepEqual(outbox.preparedSuppressionsForBranch('suppress/cairn/1').map((b) => b.id), [suppression.id]); + outbox.recordSuppressionComplete(suppression.id); + assert.deepEqual(outbox.preparedSuppressionsForBranch('suppress/cairn/1'), []); + assert.equal(outbox.pendingDispatches('discord').length, 0); +})); + +test('retract removes this bot\'s mark across batches, supersedes unsent adds, and discloses unresolved attempts', withJournal((path) => { + const outbox = new DiscordAwarenessOutbox(path); + const a = activeBatch(outbox, [ref('m1'), ref('m2'), ref('m3')]); + const b = activeBatch(outbox, [ref('m1')]); // another batch asked for the same reaction + // m1's adds share one dispatch (same key, same action, both requested). + const m1 = outbox.pendingDispatches('discord').find((d) => d.key.messageId === 'm1')!; + assert.equal(m1.opIds.length, 2); + const m1Attempts = outbox.recordDispatching(m1); + outbox.recordOutcome(m1Attempts, 'unknown', { error: 'did not respond within 75ms' }); + // m2's add never left the host; m3's was confirmed. + const m2 = outbox.pendingDispatches('discord').find((d) => d.key.messageId === 'm2')!; + outbox.recordOutcome(outbox.recordDispatching(m2), 'not-sent'); + const m3 = outbox.pendingDispatches('discord').find((d) => d.key.messageId === 'm3')!; + outbox.recordOutcome(outbox.recordDispatching(m3), 'confirmed'); + + const receipt = outbox.retract(a.id, 'operator'); + // m1 and m3 had adds leave the host; m2's never did, so nothing to remove. + assert.equal(receipt.removalsQueued, 2); + assert.equal(receipt.addsSuperseded, 1, "m2's unsent add is superseded"); + assert.equal(receipt.keysWithUnresolvedAdds, 1, 'm1: a timed-out add may land after the removal'); + assert.equal(receipt.unresolvedAddAttempts, 1, "one request carried both batches' adds"); + + const dispatches = outbox.pendingDispatches('discord'); + assert.deepEqual(dispatches.map((d) => `${d.action}:${d.key.messageId}`).sort(), ['remove:m1', 'remove:m3']); + + // The removal is confirmed, and the earlier unknown add stays unresolved: + // a confirmation of a different request never resolves it. + drain(outbox, () => 'confirmed'); + const views = new Map(outbox.view().map((v) => [v.id, v])); + assert.equal(views.get(a.id)!.adds.unknown, 1); + assert.equal(views.get(a.id)!.removals.confirmed, 2); + assert.equal(views.get(b.id)!.unresolvedAttempts, 1); +})); + +test('a retry confirming an unknown add never erases the unresolved first attempt', withJournal((path) => { + const outbox = new DiscordAwarenessOutbox(path); + const batch = activeBatch(outbox, [ref('m1')]); + const first = outbox.pendingDispatches('discord')[0]; + outbox.recordOutcome(outbox.recordDispatching(first), 'unknown'); + // Still the newest op for its key: retried (idempotent), as attempt 2. + const retry = outbox.pendingDispatches('discord')[0]; + assert.deepEqual(retry.opIds, first.opIds); + const retryAttempts = outbox.recordDispatching(retry); + assert.equal(retryAttempts[0].attempt, 2); + outbox.recordOutcome(retryAttempts, 'confirmed'); + assert.equal(outbox.pendingDispatches('discord').length, 0); + assert.equal(outbox.view()[0].unresolvedAttempts, 1, 'attempt 1 is still unknown'); + + const receipt = outbox.retract(batch.id); + assert.equal(receipt.keysWithUnresolvedAdds, 1); + assert.equal(receipt.unresolvedAddAttempts, 1); +})); + +test('a new request reaches Discord even when an earlier one got the same answer', withJournal((path) => { + const outbox = new DiscordAwarenessOutbox(path); + const first = activeBatch(outbox, [ref('m1')]); + drain(outbox, () => 'confirmed'); + outbox.retract(first.id); + drain(outbox, () => 'confirmed'); + // Someone else's tool may have reacted again since; a fresh, explicit + // placement is sent rather than skipped on the strength of old receipts. + activeBatch(outbox, [ref('m1')]); + assert.deepEqual(outbox.pendingDispatches('discord').map((d) => d.action), ['add']); +})); + +test('retract supersedes an add that never left the host instead of churning the reaction', withJournal((path) => { + const outbox = new DiscordAwarenessOutbox(path); + const batch = activeBatch(outbox, [ref('m2'), ref('m3')]); + // m3's add was sent and confirmed; m2's has not been sent yet. + const m3 = outbox.pendingDispatches('discord').find((d) => d.key.messageId === 'm3')!; + outbox.recordOutcome(outbox.recordDispatching(m3), 'confirmed'); + const receipt = outbox.retract(batch.id); + assert.equal(receipt.addsSuperseded, 1); + assert.equal(receipt.removalsQueued, 1, 'nothing was ever sent for m2, so there is nothing to remove'); + assert.deepEqual(outbox.pendingDispatches('discord').map((d) => `${d.action}:${d.key.messageId}`), ['remove:m3']); + const m2 = outbox.operations().find((op) => op.key.messageId === 'm2')!; + assert.equal(m2.cancelled?.reason, 'superseded'); +})); + +test('release queues a held batch\'s recorded operations, explicitly', withJournal((path) => { + const outbox = new DiscordAwarenessOutbox(path); + const batch = outbox.prepare({ + agentName: 'cairn', sourceBranch: 'main', targetBranch: 'rollback/cairn/9', refs: [ref('m1'), ref('m2')], scope: 'addressed', + })!; + outbox.recoverAtStartup('main'); + assert.equal(outbox.batches()[0].status, 'held'); + assert.equal(outbox.pendingDispatches('discord').length, 0); + const receipt = outbox.release(batch.id, 'operator'); + assert.deepEqual({ adds: receipt.addsQueued, removals: receipt.removalsQueued }, { adds: 2, removals: 0 }); + assert.equal(outbox.batches()[0].status, 'active'); + assert.equal(outbox.pendingDispatches('discord').length, 2); + assert.throws(() => outbox.release(batch.id), /not held/); +})); + +test('a torn or uncommitted append applies none of its records', withJournal((path) => { + const outbox = new DiscordAwarenessOutbox(path); + const batch = outbox.prepare({ + agentName: 'cairn', sourceBranch: 'main', targetBranch: 'rollback/cairn/1', refs: [ref('m1'), ref('m2')], scope: 'all', + })!; + outbox.activate(batch.id); + const committed = readFileSync(path, 'utf8'); + // Simulate an activation whose append was cut off before its commit. + const lines = committed.trimEnd().split('\n'); + const activation = lines.filter((line) => line.includes('"txn"')); + assert.ok(activation.length >= 3, 'activation is a transaction'); + writeFileSync(path, lines.filter((line) => !line.includes('"txn"') && !line.includes('"t":"commit"')).join('\n') + '\n'); + appendFileSync(path, activation.slice(0, -1).join('\n') + '\n' + '{"t":"requested","at":1,"op'); + const reread = new DiscordAwarenessOutbox(path); + assert.equal(reread.batches()[0].status, 'prepared', 'the activation never committed'); + assert.equal(reread.pendingDispatches('discord').length, 0); + assert.equal(reread.operations().length, 0); +})); + +test('an append after a torn tail repairs it instead of corrupting the journal', withJournal((path) => { + const outbox = new DiscordAwarenessOutbox(path); + const batch = activeBatch(outbox, [ref('m1'), ref('m2')]); + // A single-record append cut off mid-line (the process died writing it). + appendFileSync(path, '{"t":"dispatching","at":1,"opId":"x","att'); + const reread = new DiscordAwarenessOutbox(path); + assert.equal(reread.pendingDispatches('discord').length, 2, 'the torn record never applied'); + // The next append must not share the torn line. + const dispatch = reread.pendingDispatches('discord')[0]; + reread.recordOutcome(reread.recordDispatching(dispatch), 'confirmed'); + const again = new DiscordAwarenessOutbox(path); + assert.equal(again.pendingDispatches('discord').length, 1); + assert.ok(!readFileSync(path, 'utf8').includes('"att{'), 'the fragment was truncated away'); + + // A complete record that only lost its newline is kept, then terminated. + const lines = readFileSync(path, 'utf8'); + writeFileSync(path, lines.trimEnd()); + const kept = new DiscordAwarenessOutbox(path); + assert.equal(kept.pendingDispatches('discord').length, 1); + kept.cancel(batch.id); + assert.equal(new DiscordAwarenessOutbox(path).pendingDispatches('discord').length, 0); + assert.equal(new DiscordAwarenessOutbox(path).batches()[0].cancelled !== undefined, true); +})); + +test('v2 import keeps what the old ledger recorded, holds pending work, and happens once', withJournal((path, dir) => { + const legacyPath = join(dir, 'recovery', 'discord-awareness-outbox.json'); + const entry = (messageId: string, fields: Record) => ({ ...ref(messageId), attempts: 1, lastAction: 'add', ...fields }); + const legacy = { + version: 2, + batches: [ + { + // The household's shape: an active target-branch batch, some marks + // applied, the rest marked permanent-failure by hand after shutdown. + id: 'dd34b043', status: 'active', agentName: 'resident', sourceBranch: 'main', + targetBranch: 'rollback/resident/1791358473307', emoji: '💤', createdAt: 1791358473400, + activationPolicy: 'target-branch', + refs: [ + entry('a1', { desired: true, markerPresent: true, deliveryStatus: 'applied' }), + entry('a2', { desired: true, markerPresent: true, deliveryStatus: 'applied' }), + entry('p1', { desired: true, markerPresent: false, deliveryStatus: 'permanent-failure', lastError: 'Cannot send request: connection to "discord" is closed' }), + ], + }, + { + // An interrupted drain that nobody edited: its pending work is held. + id: 'pending-batch', status: 'active', agentName: 'resident', sourceBranch: 'main', + targetBranch: 'rollback/resident/2', emoji: '💤', createdAt: 1791358473500, + refs: [ + entry('q1', { desired: true, markerPresent: false, deliveryStatus: 'pending', lastError: 'Cannot send request' }), + entry('q2', { desired: true, markerPresent: true, deliveryStatus: 'applied' }), + ], + }, + ], + }; + mkdirSync(join(dir, 'recovery'), { recursive: true }); + writeFileSync(legacyPath, JSON.stringify(legacy)); + + const outbox = new DiscordAwarenessOutbox(path); + // Reading imports in memory only; nothing is dispatched from an import. + assert.equal(outbox.pendingDispatches('discord').length, 0); + const views = new Map(outbox.view().map((v) => [v.id, v])); + assert.equal(views.get('dd34b043')!.adds.confirmed, 2); + assert.equal(views.get('dd34b043')!.adds.failed, 1); + assert.equal(views.get('pending-batch')!.status, 'held'); + assert.equal(views.get('pending-batch')!.held!.releaseActions, 1); + + // The first write persists the import (with its hash) and retires the file. + const receipt = outbox.retract('dd34b043', 'operator'); + assert.equal(receipt.removalsQueued, 3, 'every add that left the host is removed; the import cannot tell which were sent'); + assert.equal(existsSync(legacyPath), false); + assert.equal(existsSync(`${legacyPath}.migrated-v2`), true); + + // A crash between the import append and the rename re-reads the same + // file: its hash is recorded, so nothing is imported or queued twice. + writeFileSync(legacyPath, JSON.stringify(legacy)); + const again = new DiscordAwarenessOutbox(path); + assert.equal(again.batches().filter((b) => b.id === 'dd34b043').length, 1); + assert.equal(again.pendingDispatches('discord').length, 3); + again.cancel('pending-batch'); + assert.equal(existsSync(legacyPath), false, 'the next write finishes the rename'); +})); + +test('framework drain classifies outcomes from what reached the server', async () => { + const dir = mkdtempSync(join(tmpdir(), 'discord-awareness-drain-')); try { - const outbox = new DiscordAwarenessOutbox(path); - const batch = outbox.prepare({ - agentName: 'cairn', - sourceBranch: 'main', - targetBranch: 'recovery/cairn/suppressed', - activationPolicy: 'explicit', - refs: [ - { serverId: 'discord', channelId: 'discord:g1:c1', messageId: 'm1' }, - ], - })!; + const outbox = new DiscordAwarenessOutbox(join(dir, 'journal.jsonl')); + activeBatch(outbox, [ref('m1'), ref('m2'), ref('m3'), ref('m4'), ref('m5')]); + const calls: string[] = []; + let connected = true; + const connection = { + get isConnected() { return connected; }, + sendToolsCallWithDeadline: async (name: string, args: Record) => { + calls.push(`${name}:${args.messageId}`); + if (args.messageId === 'm2') return { isError: true, content: [{ type: 'text', text: 'Unknown Message' }] }; + if (args.messageId === 'm3') throw new McplRequestError('Cannot send request: connection to "discord" is closed', 'not-sent'); + if (args.messageId === 'm4') throw new McplRequestError('did not respond', 'no-response'); + return { content: [{ type: 'text', text: 'Reaction applied' }] }; + }, + }; + const framework = Object.create(AgentFramework.prototype) as any; + framework.discordAwarenessOutbox = outbox; + framework.discordAwarenessDrains = new Map(); + framework.discordAwarenessDeadlineMs = 1000; + framework.mcplServerRegistry = { getServer: () => connection }; + const originalError = console.error; + console.error = () => {}; + try { + await framework.drainDiscordAwarenessOutbox('discord'); + } finally { + console.error = originalError; + } + assert.deepEqual(calls, ['add_reaction:m1', 'add_reaction:m2', 'add_reaction:m3', 'add_reaction:m4', 'add_reaction:m5']); + const status = new Map(outbox.operations().map((op) => [op.key.messageId, outbox.operationStatus(op)])); + assert.equal(status.get('m1'), 'confirmed'); + assert.equal(status.get('m2'), 'failed', 'Discord said the message is gone: final'); + assert.equal(status.get('m3'), 'requested', 'never written: still queued'); + assert.equal(status.get('m4'), 'unknown', 'written, no answer: unknown'); + assert.equal(status.get('m5'), 'confirmed'); + + // A disconnected route leaves its work queued without writing anything. + connected = false; + const before = readFileSync(join(dir, 'journal.jsonl'), 'utf8'); + console.error = () => {}; + try { + await framework.drainDiscordAwarenessOutbox('discord'); + } finally { + console.error = originalError; + } + assert.equal(readFileSync(join(dir, 'journal.jsonl'), 'utf8'), before); - assert.equal(outbox.activatePreparedForBranch('recovery/cairn/suppressed'), 0); - assert.equal(outbox.pending('discord').length, 0); - outbox.activate(batch.id); - assert.equal(outbox.pending('discord').length, 1); + // Back online: the unsent and the unknown (newest for their keys) go again. + connected = true; + calls.length = 0; + await framework.drainDiscordAwarenessOutbox('discord'); + assert.deepEqual(calls.sort(), ['add_reaction:m3', 'add_reaction:m4']); } finally { rmSync(dir, { recursive: true, force: true }); } }); -test('framework resumes an interrupted suppression before activating its markers', async () => { +test('framework resumes an interrupted suppression before activating its marks', async () => { const dir = mkdtempSync(join(tmpdir(), 'discord-awareness-resume-')); - const path = join(dir, 'outbox.json'); try { - const outbox = new DiscordAwarenessOutbox(path); + const outbox = new DiscordAwarenessOutbox(join(dir, 'journal.jsonl')); const batch = outbox.prepare({ agentName: 'cairn', sourceBranch: 'main', targetBranch: 'recovery/cairn/suppressed', activationPolicy: 'explicit', - suppressionIntervals: [ - { fromId: 'i1', toId: 'i2' }, - { fromId: 'i3', toId: 'i3' }, - ], - refs: [ - { serverId: 'discord', channelId: 'discord:g1:c1', messageId: 'm1' }, - ], + suppressionIntervals: [{ fromId: 'i1', toId: 'i2' }, { fromId: 'i3', toId: 'i3' }], + refs: [ref('m1')], + scope: 'all', })!; const present = new Set(['i1', 'i2']); // i3 interval committed before crash. const removals: string[] = []; @@ -127,157 +506,153 @@ test('framework resumes an interrupted suppression before activating its markers }; const framework = Object.create(AgentFramework.prototype) as any; framework.discordAwarenessOutbox = outbox; - framework.store = { - currentBranch: () => ({ name: 'recovery/cairn/suppressed' }), - listBranches: () => [], - }; + framework.store = { currentBranch: () => ({ name: 'recovery/cairn/suppressed' }) }; framework.agents = new Map([['cairn', { getContextManager: () => cm }]]); await framework.resumePreparedDiscordSuppressions(); assert.deepEqual(removals, ['i1..i2']); assert.equal(outbox.batches().find((candidate) => candidate.id === batch.id)?.status, 'active'); - assert.deepEqual(outbox.pending('discord').map((operation) => operation.ref.messageId), ['m1']); + assert.deepEqual(outbox.pendingDispatches('discord').map((d) => d.key.messageId), ['m1']); } finally { rmSync(dir, { recursive: true, force: true }); } }); -test('framework drain records failures per ref and continues past permanent failures', async () => { - const dir = mkdtempSync(join(tmpdir(), 'discord-awareness-drain-')); - const path = join(dir, 'outbox.json'); - try { - const outbox = new DiscordAwarenessOutbox(path); - const batch = outbox.prepare({ - agentName: 'cairn', - sourceBranch: 'main', - targetBranch: 'recovery/cairn/2', - refs: [ - { serverId: 'discord', channelId: 'discord:g1:c1', messageId: 'm1' }, - { serverId: 'discord', channelId: 'discord:g1:c1', messageId: 'm2' }, - { serverId: 'discord', channelId: 'discord:g1:c1', messageId: 'm3' }, - { serverId: 'discord', channelId: 'discord:g1:c1', messageId: 'm4' }, - ], - })!; - outbox.activate(batch.id); - - const calls: Array<{ name: string; args: Record }> = []; - let retryableFailure = true; - const connection = { - isConnected: true, - sendToolsCallWithDeadline: async (name: string, args: Record) => { - calls.push({ name, args }); - if (args.messageId === 'm2') { - return { isError: true, content: [{ type: 'text', text: 'Unknown Message' }] }; - } - if (args.messageId === 'm3' && retryableFailure) throw new Error('Discord unavailable'); - return { content: [{ type: 'text', text: 'Reaction added' }] }; - }, - }; - const framework = Object.create(AgentFramework.prototype) as any; - framework.discordAwarenessOutbox = outbox; - framework.discordAwarenessDrains = new Map(); - framework.mcplServerRegistry = { getServer: () => connection }; - - const originalError = console.error; - console.error = () => {}; - try { - await framework.drainDiscordAwarenessOutbox('discord'); - } finally { - console.error = originalError; - } - assert.deepEqual(calls[0], { - name: 'add_reaction', - args: { channelId: 'c1', messageId: 'm1', emoji: '💤' }, - }); - assert.deepEqual(outbox.pending('discord').map((operation) => operation.ref.messageId), ['m3']); - assert.deepEqual(calls.map((call) => call.args.messageId), ['m1', 'm2', 'm3', 'm4']); - const entries = outbox.batches()[0].refs; - assert.equal(entries.find((entry) => entry.messageId === 'm2')?.deliveryStatus, 'permanent-failure'); - assert.equal(entries.find((entry) => entry.messageId === 'm3')?.deliveryStatus, 'pending'); - assert.equal(entries.find((entry) => entry.messageId === 'm4')?.deliveryStatus, 'applied'); - - retryableFailure = false; - await framework.drainDiscordAwarenessOutbox('discord'); - assert.equal(outbox.pending('discord').length, 0); - - outbox.reconcileForBranch('main'); - calls.length = 0; - await framework.drainDiscordAwarenessOutbox('discord'); - assert.deepEqual(calls.map((call) => [call.name, call.args.messageId]), [ - ['remove_reaction', 'm1'], - ['remove_reaction', 'm3'], - ['remove_reaction', 'm4'], - ]); - assert.ok(outbox.batches()[0].refs.every((entry) => !entry.markerPresent)); - } finally { - rmSync(dir, { recursive: true, force: true }); - } -}); - -test('message-granular undo prepares markers before switching branches', async () => { +test('message-granular host/command undo places no marks unless the command asks', async () => { const dir = mkdtempSync(join(tmpdir(), 'discord-awareness-undo-')); - const path = join(dir, 'outbox.json'); try { const messages = [ - { id: 'i0', metadata: { serverId: 'discord', channelId: 'discord:g1:c1', messageId: 'm0' } }, - { id: 'i1', metadata: { serverId: 'discord', channelId: 'discord:g1:c1', messageId: 'm1' } }, - { id: 'i2', metadata: { serverId: 'discord', channelId: 'discord:g1:c1', messageId: 'm2' } }, + { id: 'i0', metadata: { ...ref('m0') } }, + { id: 'i1', metadata: { ...ref('m1'), tags: ['chat:addressed'] } }, + { id: 'i2', metadata: { ...ref('m2'), tags: ['chat:ambient'] } }, + { id: 'i3', metadata: { ...ref('m3') } }, ]; - const calls: string[] = []; let currentBranch = 'main'; const contextManager = { getAllMessages: () => messages, - // Windowed reads — what the live rollback path uses so it never - // re-inflates every attachment on the branch. getMessageCount: () => messages.length, getMessageWindow: (offset: number, limit: number) => ({ messages: messages.slice(offset, offset + limit), startIndex: offset, totalCount: messages.length, }), - branchAt: (_id: string, name: string) => { calls.push(`branch:${name}`); return name; }, - switchBranch: async (name: string) => { calls.push(`switch:${name}`); currentBranch = name; }, + branchAt: (_id: string, name: string) => name, + switchBranch: async (name: string) => { currentBranch = name; }, }; const framework = Object.create(AgentFramework.prototype) as any; - framework.agents = new Map([['cairn', { - state: { status: 'idle' }, - getContextManager: () => contextManager, - }]]); - framework.store = { - currentBranch: () => ({ name: currentBranch }), - listBranches: () => [ - { id: 'b-main', name: 'main' }, - { id: 'b-undo', name: currentBranch, parentId: 'b-main' }, - ], - }; - framework.discordAwarenessOutbox = new DiscordAwarenessOutbox(path); + framework.agents = new Map([['cairn', { state: { status: 'idle' }, getContextManager: () => contextManager }]]); + framework.store = { currentBranch: () => ({ name: currentBranch }) }; + framework.discordAwarenessOutbox = new DiscordAwarenessOutbox(join(dir, 'journal.jsonl')); framework.discordAwarenessEmoji = '🫥'; framework.discordAwarenessDrains = new Map(); framework.mcplServerRegistry = null; framework.moduleRegistry = { getModule: () => null }; framework.lastVisiblePreview = async () => null; - // Live rollback reserves the store: it mints turn tokens for every agent - // and flushes deferred writers on release — give the stub those fields. framework.activeTurnTokens = new Map(); framework.nextTurnToken = 1; framework.deferredMessages = []; framework.pendingAssistantBlocks = new Map(); framework.operatorLog = new OperatorLog(undefined); + const originalError = console.error; + console.error = () => {}; + try { + const silent = await framework.handleHostCommand('discord', { command: 'undo', agentName: 'cairn', messages: 1 }); + assert.equal(silent.ok, true); + assert.deepEqual(silent.markers, { scope: 'none', unmarked: 1, notRemoved: 0, status: 'none', queued: 0 }); + assert.equal(framework.discordAwarenessOutbox.pendingDispatches('discord').length, 0); + + currentBranch = 'main'; + const marked = await framework.handleHostCommand('discord', { + command: 'undo', agentName: 'cairn', messages: 3, marks: 'addressed', + }); + assert.equal(marked.markers.status, 'queued'); + assert.equal(marked.markers.queued, 1); + assert.equal(marked.markers.unmarked, 2); + const dispatches = framework.discordAwarenessOutbox.pendingDispatches('discord'); + assert.deepEqual(dispatches.map((d: { key: { messageId: string; emoji: string } }) => `${d.key.messageId}${d.key.emoji}`), ['m1🫥']); + + // A mistyped scope is refused, never guessed. + currentBranch = 'main'; + const typo = await framework.handleHostCommand('discord', { + command: 'undo', agentName: 'cairn', messages: 1, marks: 'adressed', + }); + assert.equal(typo.ok, false); + assert.match(typo.error, /marks must be none, addressed or all/); - const result = await framework.handleHostCommand('discord', { - command: 'undo', agentName: 'cairn', messages: 2, - }); - assert.equal(result.ok, true); - assert.deepEqual(result.removedRefs.map((ref: { messageId: string }) => ref.messageId), ['m1', 'm2']); - assert.deepEqual(calls.map((call) => call.split('/')[0]), ['branch:undo-msgs', 'switch:undo-msgs']); - assert.deepEqual( - framework.discordAwarenessOutbox.pending('discord').map((operation: { ref: { messageId: string } }) => operation.ref.messageId), - ['m1', 'm2'], - ); - assert.ok(framework.discordAwarenessOutbox.pending('discord').every( - (operation: { emoji: string }) => operation.emoji === '🫥', - )); + // The journal's controls are reachable through host/command too. + const listed = await framework.handleHostCommand('discord', { command: 'marks', action: 'list' }); + assert.equal(listed.ok, true); + assert.equal(listed.awareness.length, 1); + const cancelled = await framework.handleHostCommand('discord', { + command: 'marks', action: 'cancel', batchId: listed.awareness[0].id, requesterName: 'op', + }); + assert.equal(cancelled.ok, true); + assert.equal(cancelled.awareness.cancelled, 1); + assert.equal(framework.discordAwarenessOutbox.pendingDispatches('discord').length, 0); + const missing = await framework.handleHostCommand('discord', { command: 'marks', action: 'retract' }); + assert.equal(missing.ok, false); + } finally { + console.error = originalError; + } + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); + +test('host/command hide marks only on request, through the journal, routed by the issuing server for old records', async () => { + const dir = mkdtempSync(join(tmpdir(), 'discord-awareness-hide-')); + try { + const messages = [ + { id: 'i0', metadata: { ...ref('m0') } }, + { id: 'i1', metadata: { channelId: 'discord:g1:c1', messageId: 'm1', tags: ['chat:addressed'] } }, // pre-serverId record + { id: 'i2', metadata: { ...ref('m2'), tags: ['chat:ambient'] } }, + { id: 'i3', metadata: { ...ref('m3') } }, + ]; + const removed: string[] = []; + const contextManager = { + getAllMessages: () => messages.filter((m) => !removed.includes(m.id)), + removeMessage: (id: string) => { removed.push(id); }, + removeMessages: (from: string, to: string) => { + const ids = messages.map((m) => m.id); + for (const id of ids.slice(ids.indexOf(from), ids.indexOf(to) + 1)) removed.push(id); + }, + }; + const framework = Object.create(AgentFramework.prototype) as any; + framework.agents = new Map([['cairn', { state: { status: 'idle' }, getContextManager: () => contextManager }]]); + framework.store = { currentBranch: () => ({ name: 'main' }) }; + framework.discordAwarenessOutbox = new DiscordAwarenessOutbox(join(dir, 'journal.jsonl')); + framework.discordAwarenessEmoji = '💤'; + framework.discordAwarenessDrains = new Map(); + framework.mcplServerRegistry = null; + framework.lastVisiblePreview = async () => null; + framework.operatorLog = new OperatorLog(undefined); + const originalError = console.error; + console.error = () => {}; + try { + const silent = await framework.handleHostCommand('discord', { command: 'hide', agentName: 'cairn', fromMessageId: 'm3' }); + assert.equal(silent.ok, true); + assert.equal(silent.markers.status, 'none'); + assert.equal(silent.markers.unmarked, 1); + assert.equal(framework.discordAwarenessOutbox.batches().length, 0); + + const marked = await framework.handleHostCommand('discord', { + command: 'hide', agentName: 'cairn', fromMessageId: 'm1', toMessageId: 'm2', marks: 'addressed', + }); + assert.equal(marked.ok, true); + assert.equal(marked.hidden, 2); + assert.equal(marked.markers.status, 'queued'); + assert.equal(marked.markers.queued, 1); + assert.equal(marked.markers.unmarked, 1); + const [dispatch] = framework.discordAwarenessOutbox.pendingDispatches('discord'); + assert.deepEqual(dispatch.key, { serverId: 'discord', channelId: 'discord:g1:c1', messageId: 'm1', emoji: '💤' }); + // Explicit and already applied: a restart on this branch changes nothing. + const restarted = new DiscordAwarenessOutbox(join(dir, 'journal.jsonl')); + assert.deepEqual(restarted.recoverAtStartup('main'), { activated: [], held: [], unknownAttempts: 0 }); + assert.equal(restarted.batches()[0].status, 'active'); + } finally { + console.error = originalError; + } } finally { rmSync(dir, { recursive: true, force: true }); } diff --git a/test/fixtures/awareness-barrier-mcpl-server.mjs b/test/fixtures/awareness-barrier-mcpl-server.mjs deleted file mode 100644 index a16b342b3..000000000 --- a/test/fixtures/awareness-barrier-mcpl-server.mjs +++ /dev/null @@ -1,276 +0,0 @@ -import { appendFileSync, existsSync, readFileSync, writeFileSync } from 'node:fs'; - -const statusPath = process.env.STATUS_PATH; -const ledgerPath = process.env.LEDGER_PATH; -const releasePath = process.env.RELEASE_PATH; -const crashPath = process.env.CRASH_PATH; -const generationPath = process.env.GENERATION_PATH; -const listChangePath = process.env.LIST_CHANGE_PATH; -const failReaction = process.env.FAIL_REACTION === '1'; -const permanentReaction = process.env.PERMANENT_REACTION === '1'; -const failInitialGeneration = process.env.FAIL_INITIAL_GENERATION === '1'; -const nestedListChangeGeneration = Number(process.env.NESTED_LIST_CHANGE_GENERATION ?? 0); -const dataMethod = process.env.DATA_METHOD ?? 'push/event'; -const serverId = process.env.SERVER_ID ?? 'discord'; -const arrivalPath = process.env.ARRIVAL_PATH; -const waitForArrivalPath = process.env.WAIT_FOR_ARRIVAL_PATH; - -let generation = 1; -if (generationPath) { - try { - generation = Number(readFileSync(generationPath, 'utf8')) + 1; - } catch { - // First process for this test. - } - writeFileSync(generationPath, String(generation)); -} - -const log = (event, extra = {}) => { - if (!statusPath) return; - appendFileSync(statusPath, JSON.stringify({ event, generation, serverId, ...extra }) + '\n'); -}; -const send = (message) => process.stdout.write(JSON.stringify(message) + '\n'); -const request = (id, method, params) => send({ jsonrpc: '2.0', id, method, params }); -const reply = (id, result) => send({ jsonrpc: '2.0', id, result }); -const channel = (suffix) => ({ - id: `${serverId}:guild:${suffix}`, - type: serverId, - label: suffix, - direction: 'bidirectional', -}); - -if (failInitialGeneration && generation === 1) { - log('initial-connect-failure'); - process.exit(9); -} - -let initialized = false; -let registered = false; -let registrationReceived = false; -let pendingToolRequest = null; -let reactionQueued = false; -let pendingControl = false; -let listChangeTriggered = false; -let reactionOrdinal = 0; -let buf = ''; - -function ledgerDeliveryStatus() { - if (!ledgerPath) return 'missing'; - try { - const document = JSON.parse(readFileSync(ledgerPath, 'utf8')); - return document.batches?.[0]?.refs?.[0]?.deliveryStatus ?? 'missing'; - } catch { - return 'unreadable'; - } -} - -function maybeReplyToReaction() { - if (!pendingToolRequest || pendingControl || failReaction) return; - if (releasePath) { - if (!existsSync(releasePath)) return; - const releaseValue = readFileSync(releasePath, 'utf8').trim(); - const releaseOrdinal = Number(releaseValue); - if (Number.isFinite(releaseOrdinal) && releaseOrdinal < pendingToolRequest.ordinal) return; - } - const requestId = pendingToolRequest.id; - pendingToolRequest = null; - log('reaction-response'); - reply(requestId, permanentReaction - ? { isError: true, content: [{ type: 'text', text: 'Unknown Message' }] } - : { content: [{ type: 'text', text: 'Reaction applied' }] }); -} - -function serviceReaction(message) { - if (waitForArrivalPath && !existsSync(waitForArrivalPath)) { - pendingToolRequest = message; - return; - } - if (!registered) { - pendingToolRequest = message; - if (!reactionQueued) { - reactionQueued = true; - log('reaction-queued-before-registration'); - } - return; - } - reactionOrdinal++; - pendingToolRequest = { ...message, ordinal: reactionOrdinal }; - pendingControl = true; - log('reaction-call', { name: message.params?.name, ordinal: reactionOrdinal }); - request(201, 'channels/register', { channels: [channel('control-during-barrier')] }); - if ( - nestedListChangeGeneration === generation - && reactionOrdinal === 1 - && !listChangeTriggered - ) { - listChangeTriggered = true; - log('tools-list-changed-notification'); - send({ jsonrpc: '2.0', method: 'notifications/tools/list_changed', params: {} }); - sendDataRequest(302, 'list-change'); - } -} - -function maybeFinishRegistration() { - if (!registrationReceived || registered) return; - if (pendingToolRequest && waitForArrivalPath && !existsSync(waitForArrivalPath)) return; - if (pendingToolRequest && !reactionQueued) serviceReaction(pendingToolRequest); - registered = true; - log('registration-response'); - const queued = pendingToolRequest; - pendingToolRequest = null; - if (queued) serviceReaction(queued); -} - -function sendDataRequest(id, phase) { - let params; - if (dataMethod === 'inference/request') { - params = { - featureSet: 'chat', - messages: [{ role: 'user', content: `must wait for ${phase} awareness` }], - }; - } else if (dataMethod === 'channels/incoming') { - params = { - messages: [{ - channelId: `${serverId}:guild:startup`, - messageId: `${serverId}-${phase}-incoming-${generation}`, - author: { id: 'human', name: 'Human' }, - timestamp: new Date().toISOString(), - content: [{ type: 'text', text: `must wait for ${phase} awareness` }], - }], - }; - } else if (dataMethod === 'host/command') { - params = { command: 'barrier-probe', agentName: 'assistant' }; - } else { - params = { - featureSet: 'chat', - eventId: `${serverId}-${phase}-push-${generation}`, - timestamp: new Date().toISOString(), - payload: { content: [{ type: 'text', text: `must wait for ${phase} awareness` }] }, - }; - } - log('data-request-sent', { method: dataMethod, phase }); - if (arrivalPath) writeFileSync(arrivalPath, `${serverId}:${dataMethod}:${phase}`); - request(id, dataMethod, params); -} - -function beginServerTraffic() { - if (initialized) return; - initialized = true; - log('initialized'); - request(200, 'channels/register', { channels: [channel('startup')] }); - // The reconnect generation carries an inference-bearing event in the same - // registration window. Initial-start tests use generation 1 as well. - sendDataRequest(202, 'startup'); -} - -function handle(message) { - if (message.method === 'initialize') { - reply(message.id, { - capabilities: { - experimental: { - mcpl: { - version: '0.5', - // §5.4/§6.2: the grant is computed from this advertisement, and - // admission denies what it cannot contain. This fixture SENDS - // channels/register, channels/incoming and inference/request, so - // it must advertise them — with only pushEvents declared, the - // 0.5 host correctly rejected the released data requests - // (accepted:false), which is exactly what PR #79's ubuntu CI - // caught on the inference/request + channels/incoming matrix. - pushEvents: true, - inferenceRequest: true, - channels: { register: true, incoming: true }, - featureSets: { - chat: { description: 'chat', uses: ['pushEvents'] }, - }, - }, - }, - }, - }); - return; - } - if (message.method === 'featureSets/update') { - // MCPL 0.5 (§5.3/§6.7): the host sends initial policy as a Request and - // activates the grant only on this degradation receipt. Without it the - // host times out (15s), the grant stays empty, and every push/channel - // in these tests is rejected fail-closed. - if (message.id !== undefined && message.id !== null) { - reply(message.id, { accepted: true }); - } - return; - } - if (message.method === 'notifications/initialized') { - beginServerTraffic(); - return; - } - if (message.method === 'tools/list') { - log('tools-list'); - reply(message.id, { - tools: [ - { name: 'add_reaction', description: 'add', inputSchema: { type: 'object' } }, - { name: 'remove_reaction', description: 'remove', inputSchema: { type: 'object' } }, - ], - }); - return; - } - if (message.method === 'tools/call') { - serviceReaction(message); - return; - } - if (message.id === 200) { - registrationReceived = true; - maybeFinishRegistration(); - return; - } - if (message.id === 201) { - pendingControl = false; - log('control-response-during-barrier'); - maybeReplyToReaction(); - return; - } - if (message.id === 202 || message.id === 302) { - const event = message.id === 202 ? 'push-response' : 'list-change-push-response'; - const accepted = message.result?.accepted - ?? message.result?.results?.[0]?.accepted - ?? (message.error == null); - log(event, { - method: dataMethod, - accepted, - ledgerStatus: ledgerDeliveryStatus(), - }); - } -} - -process.stdin.on('data', (chunk) => { - buf += chunk.toString('utf8'); - let newline; - while ((newline = buf.indexOf('\n')) >= 0) { - const line = buf.slice(0, newline); - buf = buf.slice(newline + 1); - if (!line.trim()) continue; - try { - handle(JSON.parse(line)); - } catch (error) { - log('server-error', { message: error instanceof Error ? error.message : String(error) }); - } - } -}); - -const timer = setInterval(() => { - if (pendingToolRequest && !registered && waitForArrivalPath && existsSync(waitForArrivalPath)) { - serviceReaction(pendingToolRequest); - maybeFinishRegistration(); - } - maybeReplyToReaction(); - if (listChangePath && existsSync(listChangePath) && !listChangeTriggered) { - listChangeTriggered = true; - log('tools-list-changed-notification'); - send({ jsonrpc: '2.0', method: 'notifications/tools/list_changed', params: {} }); - sendDataRequest(302, 'list-change'); - } - if (crashPath && existsSync(crashPath) && generation === 1) { - log('crashing'); - process.exit(7); - } -}, 5); -timer.unref(); diff --git a/test/mcpl-awareness-barrier.test.ts b/test/mcpl-awareness-barrier.test.ts deleted file mode 100644 index 9cb27c5d3..000000000 --- a/test/mcpl-awareness-barrier.test.ts +++ /dev/null @@ -1,798 +0,0 @@ -import { test } from 'node:test'; -import assert from 'node:assert/strict'; -import { fileURLToPath } from 'node:url'; -import { - mkdirSync, - mkdtempSync, - readFileSync, - rmSync, - writeFileSync, -} from 'node:fs'; -import { join } from 'node:path'; - -import { AgentFramework } from '../src/framework.js'; -import { McplServerConnection } from '../src/mcpl/server-connection.js'; -import { CapabilityGrant } from '../src/mcpl/capability-grant.js'; -import type { McplServerConfig } from '../src/mcpl/types.js'; -import { DiscordAwarenessOutbox } from '../src/recovery/discord-awareness-outbox.js'; -import { MockMembrane } from './helpers/mock-membrane.js'; - -const FIXTURE = fileURLToPath(new URL('./fixtures/awareness-barrier-mcpl-server.mjs', import.meta.url)); -const TEST_ROOT = join(process.cwd(), '.test-tmp'); - -interface StatusRecord { - event: string; - generation: number; - serverId?: string; - accepted?: boolean; - ledgerStatus?: string; - method?: string; - ordinal?: number; -} - -function records(path: string): StatusRecord[] { - try { - return readFileSync(path, 'utf8') - .split('\n') - .filter(Boolean) - .map((line) => JSON.parse(line) as StatusRecord); - } catch { - return []; - } -} - -async function waitFor( - description: string, - predicate: () => boolean, - timeoutMs = 3_000, -): Promise { - const deadline = Date.now() + timeoutMs; - while (Date.now() < deadline) { - if (predicate()) return; - await new Promise((resolve) => setTimeout(resolve, 10)); - } - assert.fail(`timed out waiting for ${description}`); -} - -function preparePending(outboxPath: string): DiscordAwarenessOutbox { - const outbox = new DiscordAwarenessOutbox(outboxPath); - const batch = outbox.prepare({ - agentName: 'assistant', - sourceBranch: 'source', - targetBranch: 'main', - activationPolicy: 'explicit', - refs: [{ serverId: 'discord', channelId: 'discord:guild:startup', messageId: 'message-1' }], - }); - assert.ok(batch); - outbox.activate(batch.id); - return outbox; -} - -function frameworkConfig( - dir: string, - extraEnv: Record = {}, - requestTimeoutMs = 1_000, - serverOverrides: Partial = {}, - awarenessDeadlineMs = 1_000, -) { - const outboxPath = join(dir, 'awareness.json'); - return { - outboxPath, - config: { - storePath: join(dir, 'store'), - discordAwarenessOutboxPath: outboxPath, - discordAwarenessDeadlineMs: awarenessDeadlineMs, - membrane: new MockMembrane().asMembrane(), - agents: [{ name: 'assistant', model: 'test', systemPrompt: 'test' }], - modules: [], - mcplServers: [{ - id: 'discord', - command: process.execPath, - args: [FIXTURE], - requestTimeoutMs, - env: extraEnv, - enabledFeatureSets: ['chat'], - ...serverOverrides, - }], - }, - }; -} - -function twoServerFrameworkConfig( - dir: string, - order: 'heartbeat-first' | 'discord-first', - dataMethod: typeof DATA_METHODS[number], - runtime = false, -) { - const outboxPath = join(dir, 'awareness.json'); - const statusPath = join(dir, 'status.jsonl'); - const releasePath = join(dir, 'release'); - const listChangePath = join(dir, 'list-change'); - const arrivalPath = join(dir, 'heartbeat-arrival'); - const server = (id: 'heartbeat' | 'discord'): McplServerConfig => ({ - id, - command: process.execPath, - args: [FIXTURE], - // The host/command matrix leg models the legitimate slash-command relay: - // host-owned authority is the operator's config decision (PR #79 - // blocker 9), so the test config makes it, exactly as the fleet's - // discord config will at rollout. - allowHostCommands: true, - requestTimeoutMs: 0, - env: { - STATUS_PATH: statusPath, - LEDGER_PATH: outboxPath, - SERVER_ID: id, - DATA_METHOD: dataMethod, - ...(id === 'discord' - ? { RELEASE_PATH: releasePath, WAIT_FOR_ARRIVAL_PATH: arrivalPath } - : { ARRIVAL_PATH: arrivalPath }), - ...(runtime && id === 'heartbeat' ? { LIST_CHANGE_PATH: listChangePath } : {}), - }, - enabledFeatureSets: ['chat'], - }); - const servers = order === 'heartbeat-first' - ? [server('heartbeat'), server('discord')] - : [server('discord'), server('heartbeat')]; - return { - outboxPath, - statusPath, - releasePath, - listChangePath, - config: { - storePath: join(dir, 'store'), - discordAwarenessOutboxPath: outboxPath, - discordAwarenessDeadlineMs: 1_000, - membrane: new MockMembrane().asMembrane(), - agents: [{ name: 'assistant', model: 'test', systemPrompt: 'test' }], - modules: [], - mcplServers: servers, - }, - }; -} - -const DATA_METHODS = [ - 'push/event', - 'inference/request', - 'channels/incoming', - 'host/command', -] as const; - -for (const order of ['heartbeat-first', 'discord-first'] as const) { - for (const dataMethod of DATA_METHODS) test( - `startup globally gates ${dataMethod} with ${order} config`, - async () => { - mkdirSync(TEST_ROOT, { recursive: true }); - const dir = mkdtempSync(join(TEST_ROOT, 'awareness-startup-')); - const { outboxPath, statusPath, releasePath, config } = twoServerFrameworkConfig( - dir, - order, - dataMethod, - ); - const outbox = preparePending(outboxPath); - let framework: AgentFramework | undefined; - let createSettled = false; - const creating = AgentFramework.create(config).then((created) => { - framework = created; - createSettled = true; - return created; - }); - - try { - await waitFor('registration and a second control response', () => { - const events = records(statusPath) - .filter((record) => record.serverId === 'discord') - .map((record) => record.event); - return events.includes('registration-response') - && events.includes('control-response-during-barrier'); - }); - - const discordOrdering = records(statusPath) - .filter((record) => record.serverId === 'discord') - .map((record) => record.event) - .filter((event) => [ - 'reaction-queued-before-registration', - 'registration-response', - 'reaction-call', - ].includes(event)); - assert.deepEqual(discordOrdering, [ - 'reaction-queued-before-registration', - 'registration-response', - 'reaction-call', - ]); - const allStatus = records(statusPath); - const heartbeatArrival = allStatus.findIndex((record) => - record.serverId === 'heartbeat' && record.event === 'data-request-sent'); - const discordDrain = allStatus.findIndex((record) => - record.serverId === 'discord' && record.event === 'reaction-queued-before-registration'); - assert.ok( - heartbeatArrival >= 0 && heartbeatArrival < discordDrain, - 'non-Discord data arrives before Discord registration/drain starts', - ); - - assert.equal(createSettled, false, 'framework readiness must wait for the marker outcome'); - assert.equal( - records(statusPath).some((record) => - record.serverId === 'heartbeat' && record.event === 'push-response'), - false, - `non-Discord ${dataMethod} must remain held while the Discord marker is pending`, - ); - assert.equal(outbox.pending('discord').length, 1); - - writeFileSync(releasePath, 'release'); - await creating; - await waitFor('gated push response', () => - records(statusPath).some((record) => - record.serverId === 'heartbeat' && record.event === 'push-response')); - - const status = records(statusPath); - const push = status.find((record) => - record.serverId === 'heartbeat' && record.event === 'push-response'); - assert.equal(push?.method, dataMethod); - assert.equal(push?.accepted, true); - assert.equal(push?.ledgerStatus, 'applied', 'push released only after durable success'); - assert.equal(outbox.pending('discord').length, 0); - assert.equal(status.filter((record) => - record.serverId === 'discord' && record.event === 'reaction-call').length, 1); - } finally { - await framework?.stop(); - rmSync(dir, { recursive: true, force: true }); - } - }, - ); -} - -test('mandatory awareness deadline survives requestTimeoutMs: 0 and durably releases startup', async () => { - mkdirSync(TEST_ROOT, { recursive: true }); - const dir = mkdtempSync(join(TEST_ROOT, 'awareness-timeout-')); - const statusPath = join(dir, 'status.jsonl'); - const { outboxPath, config } = frameworkConfig(dir, { - STATUS_PATH: statusPath, - LEDGER_PATH: join(dir, 'awareness.json'), - FAIL_REACTION: '1', - }, 0, {}, 60); - const outbox = preparePending(outboxPath); - let framework: AgentFramework | undefined; - - try { - framework = await AgentFramework.create(config); - await waitFor('push response after timeout accounting', () => - records(statusPath).some((record) => record.event === 'push-response')); - - const entry = outbox.batches()[0]!.refs[0]!; - assert.equal(entry.deliveryStatus, 'pending'); - assert.equal(entry.attempts, 1); - assert.match(entry.lastError ?? '', /did not respond to tools\/call/); - assert.match(entry.lastError ?? '', /outcome is unknown/i); - assert.equal(records(statusPath).filter((record) => record.event === 'reaction-call').length, 1); - const push = records(statusPath).find((record) => record.event === 'push-response'); - assert.equal(push?.ledgerStatus, 'pending', 'failure was durable before inference release'); - } finally { - await framework?.stop(); - rmSync(dir, { recursive: true, force: true }); - } -}); - -test('permanent marker failure is durably accounted before startup data is released', async () => { - mkdirSync(TEST_ROOT, { recursive: true }); - const dir = mkdtempSync(join(TEST_ROOT, 'awareness-permanent-')); - const statusPath = join(dir, 'status.jsonl'); - const { outboxPath, config } = frameworkConfig(dir, { - STATUS_PATH: statusPath, - LEDGER_PATH: join(dir, 'awareness.json'), - PERMANENT_REACTION: '1', - }); - const outbox = preparePending(outboxPath); - let framework: AgentFramework | undefined; - - try { - framework = await AgentFramework.create(config); - await waitFor('data response after permanent failure accounting', () => - records(statusPath).some((record) => record.event === 'push-response')); - - const entry = outbox.batches()[0]!.refs[0]!; - assert.equal(entry.deliveryStatus, 'permanent-failure'); - assert.equal(entry.attempts, 1); - assert.match(entry.lastError ?? '', /Unknown Message/); - const response = records(statusPath).find((record) => record.event === 'push-response'); - assert.equal(response?.ledgerStatus, 'permanent-failure'); - } finally { - await framework?.stop(); - rmSync(dir, { recursive: true, force: true }); - } -}); - -test('initial reconnect stub durably records unavailable work, then retries behind a fresh gate', async () => { - mkdirSync(TEST_ROOT, { recursive: true }); - const dir = mkdtempSync(join(TEST_ROOT, 'awareness-initial-unavailable-')); - const statusPath = join(dir, 'status.jsonl'); - const releasePath = join(dir, 'release'); - const generationPath = join(dir, 'generation'); - const { outboxPath, config } = frameworkConfig(dir, { - STATUS_PATH: statusPath, - LEDGER_PATH: join(dir, 'awareness.json'), - RELEASE_PATH: releasePath, - GENERATION_PATH: generationPath, - FAIL_INITIAL_GENERATION: '1', - }, 1_000, { - reconnect: true, - reconnectIntervalMs: 100, - reconnectMaxIntervalMs: 100, - }); - const outbox = preparePending(outboxPath); - let framework: AgentFramework | undefined; - - try { - framework = await AgentFramework.create(config); - const deferred = outbox.batches()[0]!.refs[0]!; - assert.equal(deferred.deliveryStatus, 'pending'); - assert.equal(deferred.attempts, 1, 'ready state must include a durable unavailable attempt'); - assert.match(deferred.lastError ?? '', /not attempted.*connection unavailable/i); - - await waitFor('successful reconnect registration and awareness call', () => { - const second = records(statusPath).filter((record) => record.generation === 2); - return second.some((record) => record.event === 'registration-response') - && second.some((record) => record.event === 'reaction-call'); - }); - assert.equal( - records(statusPath).some((record) => - record.generation === 2 && record.event === 'push-response'), - false, - 'reconnect data remains behind the newly installed awareness gate', - ); - - writeFileSync(releasePath, 'release'); - await waitFor('reconnected data after durable marker success', () => - records(statusPath).some((record) => - record.generation === 2 && record.event === 'push-response')); - assert.equal(outbox.pending('discord').length, 0); - assert.equal(outbox.batches()[0]!.refs[0]!.attempts, 2); - } finally { - await framework?.stop(); - rmSync(dir, { recursive: true, force: true }); - } -}); - -test('tools/list_changed installs the data gate before a following push', async () => { - mkdirSync(TEST_ROOT, { recursive: true }); - const dir = mkdtempSync(join(TEST_ROOT, 'awareness-list-change-')); - const statusPath = join(dir, 'status.jsonl'); - const releasePath = join(dir, 'release'); - const listChangePath = join(dir, 'list-change'); - const { outboxPath, config } = frameworkConfig(dir, { - STATUS_PATH: statusPath, - LEDGER_PATH: join(dir, 'awareness.json'), - RELEASE_PATH: releasePath, - LIST_CHANGE_PATH: listChangePath, - }); - const outbox = new DiscordAwarenessOutbox(outboxPath); - let framework: AgentFramework | undefined; - - try { - framework = await AgentFramework.create(config); - await waitFor('initial zero-pending push response', () => - records(statusPath).some((record) => record.event === 'push-response')); - - const batch = outbox.prepare({ - agentName: 'assistant', - sourceBranch: 'source', - targetBranch: 'main', - activationPolicy: 'explicit', - refs: [{ serverId: 'discord', channelId: 'discord:guild:startup', messageId: 'message-list' }], - }); - assert.ok(batch); - outbox.activate(batch.id); - writeFileSync(listChangePath, 'change'); - - await waitFor('control response while list-change barrier is pending', () => { - const status = records(statusPath); - return status.some((record) => record.event === 'tools-list-changed-notification') - && status.some((record) => record.event === 'control-response-during-barrier'); - }); - assert.equal( - records(statusPath).some((record) => record.event === 'list-change-push-response'), - false, - ); - - writeFileSync(releasePath, 'release'); - await waitFor('list-change gated push response', () => - records(statusPath).some((record) => record.event === 'list-change-push-response')); - const push = records(statusPath).find((record) => record.event === 'list-change-push-response'); - assert.equal(push?.ledgerStatus, 'applied'); - assert.equal(outbox.pending('discord').length, 0); - } finally { - await framework?.stop(); - rmSync(dir, { recursive: true, force: true }); - } -}); - -for (const dataMethod of DATA_METHODS) test( - `runtime awareness generation globally gates non-Discord ${dataMethod}`, - async () => { - mkdirSync(TEST_ROOT, { recursive: true }); - const dir = mkdtempSync(join(TEST_ROOT, 'awareness-runtime-global-')); - const { - outboxPath, - statusPath, - releasePath, - listChangePath, - config, - } = twoServerFrameworkConfig(dir, 'discord-first', dataMethod, true); - const outbox = new DiscordAwarenessOutbox(outboxPath); - let framework: AgentFramework | undefined; - - try { - framework = await AgentFramework.create(config); - await waitFor('both initial data responses', () => { - const initial = records(statusPath).filter((record) => record.event === 'push-response'); - return initial.some((record) => record.serverId === 'discord') - && initial.some((record) => record.serverId === 'heartbeat'); - }); - - const batch = outbox.prepare({ - agentName: 'assistant', - sourceBranch: 'source', - targetBranch: 'main', - activationPolicy: 'explicit', - refs: [{ - serverId: 'discord', - channelId: 'discord:guild:startup', - messageId: `runtime-global-${dataMethod}`, - }], - }); - assert.ok(batch); - outbox.activate(batch.id); - writeFileSync(listChangePath, 'change'); - - await waitFor('Discord marker call for heartbeat generation', () => - records(statusPath).some((record) => - record.serverId === 'discord' && record.event === 'reaction-call')); - assert.equal( - records(statusPath).some((record) => - record.serverId === 'heartbeat' && record.event === 'list-change-push-response'), - false, - `fresh heartbeat ${dataMethod} must wait for Discord accounting`, - ); - - writeFileSync(releasePath, 'release'); - await waitFor('globally gated runtime response', () => - records(statusPath).some((record) => - record.serverId === 'heartbeat' && record.event === 'list-change-push-response')); - const response = records(statusPath).find((record) => - record.serverId === 'heartbeat' && record.event === 'list-change-push-response'); - assert.equal(response?.method, dataMethod); - assert.equal(response?.ledgerStatus, 'applied'); - assert.equal(outbox.pending('discord').length, 0); - } finally { - await framework?.stop(); - rmSync(dir, { recursive: true, force: true }); - } - }, -); - -test('reconnect pauses fresh inbound data but permits registration and awareness tool service', async () => { - mkdirSync(TEST_ROOT, { recursive: true }); - const dir = mkdtempSync(join(TEST_ROOT, 'awareness-reconnect-')); - const statusPath = join(dir, 'status.jsonl'); - const releasePath = join(dir, 'release'); - const crashPath = join(dir, 'crash'); - const generationPath = join(dir, 'generation'); - const { outboxPath, config } = frameworkConfig(dir, { - STATUS_PATH: statusPath, - LEDGER_PATH: join(dir, 'awareness.json'), - RELEASE_PATH: releasePath, - CRASH_PATH: crashPath, - GENERATION_PATH: generationPath, - }, 1_000, { - reconnect: true, - reconnectIntervalMs: 20, - reconnectMaxIntervalMs: 20, - }); - const outbox = new DiscordAwarenessOutbox(outboxPath); - let framework: AgentFramework | undefined; - - try { - // No startup marker: this path must release push responders synchronously. - framework = await AgentFramework.create(config); - await waitFor('initial synchronous push response', () => - records(statusPath).some((record) => record.generation === 1 && record.event === 'push-response')); - - const batch = outbox.prepare({ - agentName: 'assistant', - sourceBranch: 'source', - targetBranch: 'main', - activationPolicy: 'explicit', - refs: [{ serverId: 'discord', channelId: 'discord:guild:startup', messageId: 'message-2' }], - }); - assert.ok(batch); - outbox.activate(batch.id); - writeFileSync(crashPath, 'crash'); - - await waitFor('reconnect control plane during pending barrier', () => { - const second = records(statusPath).filter((record) => record.generation === 2); - return second.some((record) => record.event === 'registration-response') - && second.some((record) => record.event === 'control-response-during-barrier'); - }); - assert.equal( - records(statusPath).some((record) => record.generation === 2 && record.event === 'push-response'), - false, - 'fresh reconnect data must not pass the new gate', - ); - - writeFileSync(releasePath, 'release'); - await waitFor('reconnect gated push response', () => - records(statusPath).some((record) => record.generation === 2 && record.event === 'push-response')); - const push = records(statusPath).find( - (record) => record.generation === 2 && record.event === 'push-response', - ); - assert.equal(push?.ledgerStatus, 'applied'); - assert.equal(outbox.pending('discord').length, 0); - } finally { - await framework?.stop(); - rmSync(dir, { recursive: true, force: true }); - } -}); - -test('startup fails explicitly when awareness reconciliation throws', async () => { - mkdirSync(TEST_ROOT, { recursive: true }); - const dir = mkdtempSync(join(TEST_ROOT, 'awareness-reconcile-failure-')); - const statusPath = join(dir, 'status.jsonl'); - const { outboxPath, config } = frameworkConfig(dir, { - STATUS_PATH: statusPath, - LEDGER_PATH: join(dir, 'awareness.json'), - }); - preparePending(outboxPath); - - const prototype = DiscordAwarenessOutbox.prototype as any; - const original = prototype.reconcileForBranch; - let calls = 0; - prototype.reconcileForBranch = function (...args: unknown[]) { - calls++; - if (calls === 2) throw new Error('injected reconcile failure'); - return original.apply(this, args); - }; - try { - await assert.rejects( - AgentFramework.create(config), - /awareness accounting failed.*injected reconcile failure/i, - ); - assert.equal( - records(statusPath).some((record) => record.event === 'push-response'), - false, - 'failed startup never reports a released data plane', - ); - } finally { - prototype.reconcileForBranch = original; - rmSync(dir, { recursive: true, force: true }); - } -}); - -test('startup fails explicitly when the awareness ledger read throws', async () => { - mkdirSync(TEST_ROOT, { recursive: true }); - const dir = mkdtempSync(join(TEST_ROOT, 'awareness-read-failure-')); - const { outboxPath, config } = frameworkConfig(dir); - preparePending(outboxPath); - - const prototype = DiscordAwarenessOutbox.prototype as any; - const original = prototype.pending; - prototype.pending = () => { throw new Error('injected read failure'); }; - try { - await assert.rejects( - AgentFramework.create(config), - /awareness accounting failed.*injected read failure/i, - ); - } finally { - prototype.pending = original; - rmSync(dir, { recursive: true, force: true }); - } -}); - -for (const writeMethod of ['recordSuccess', 'recordFailure'] as const) test( - `startup fails explicitly when ${writeMethod} cannot persist`, - async () => { - mkdirSync(TEST_ROOT, { recursive: true }); - const dir = mkdtempSync(join(TEST_ROOT, `awareness-${writeMethod}-failure-`)); - const statusPath = join(dir, 'status.jsonl'); - const { outboxPath, config } = frameworkConfig(dir, { - STATUS_PATH: statusPath, - LEDGER_PATH: join(dir, 'awareness.json'), - ...(writeMethod === 'recordFailure' ? { PERMANENT_REACTION: '1' } : {}), - }); - preparePending(outboxPath); - - const prototype = DiscordAwarenessOutbox.prototype as any; - const original = prototype[writeMethod]; - prototype[writeMethod] = () => { throw new Error(`injected ${writeMethod} write failure`); }; - try { - await assert.rejects( - AgentFramework.create(config), - new RegExp(`awareness accounting failed.*injected ${writeMethod} write failure`, 'i'), - ); - assert.equal( - records(statusPath).some((record) => record.event === 'push-response'), - false, - ); - } finally { - prototype[writeMethod] = original; - rmSync(dir, { recursive: true, force: true }); - } - }, -); - -test('runtime reconciliation failure recycles the connection and keeps fresh data gated', async () => { - mkdirSync(TEST_ROOT, { recursive: true }); - const dir = mkdtempSync(join(TEST_ROOT, 'awareness-runtime-accounting-')); - const statusPath = join(dir, 'status.jsonl'); - const releasePath = join(dir, 'release'); - const listChangePath = join(dir, 'list-change'); - const generationPath = join(dir, 'generation'); - const { outboxPath, config } = frameworkConfig(dir, { - STATUS_PATH: statusPath, - LEDGER_PATH: join(dir, 'awareness.json'), - RELEASE_PATH: releasePath, - LIST_CHANGE_PATH: listChangePath, - GENERATION_PATH: generationPath, - }, 1_000, { - reconnect: true, - reconnectIntervalMs: 20, - reconnectMaxIntervalMs: 20, - }); - const outbox = new DiscordAwarenessOutbox(outboxPath); - let framework: AgentFramework | undefined; - - try { - framework = await AgentFramework.create(config); - await waitFor('initial response', () => - records(statusPath).some((record) => record.event === 'push-response')); - const batch = outbox.prepare({ - agentName: 'assistant', - sourceBranch: 'source', - targetBranch: 'main', - activationPolicy: 'explicit', - refs: [{ serverId: 'discord', channelId: 'discord:guild:startup', messageId: 'runtime-fail' }], - })!; - outbox.activate(batch.id); - - const prototype = DiscordAwarenessOutbox.prototype as any; - const original = prototype.reconcileForBranch; - let injected = false; - prototype.reconcileForBranch = function (...args: unknown[]) { - if (!injected) { - injected = true; - throw new Error('injected runtime reconcile failure'); - } - return original.apply(this, args); - }; - try { - writeFileSync(listChangePath, 'change'); - await waitFor('replacement connection awareness call', () => - records(statusPath).some((record) => - record.generation === 2 && record.event === 'reaction-call')); - } finally { - prototype.reconcileForBranch = original; - } - - assert.equal( - records(statusPath).some((record) => - record.generation === 2 && record.event === 'push-response'), - false, - ); - writeFileSync(releasePath, 'release'); - await waitFor('replacement connection data release', () => - records(statusPath).some((record) => - record.generation === 2 && record.event === 'push-response')); - assert.equal(outbox.pending('discord').length, 0); - } finally { - await framework?.stop(); - rmSync(dir, { recursive: true, force: true }); - } -}); - -function deferred() { - let resolve!: (value: T) => void; - let reject!: (error: unknown) => void; - const promise = new Promise((res, rej) => { resolve = res; reject = rej; }); - return { promise, resolve, reject }; -} - -test('a control flush that installs a newer generation cannot be released by the old barrier', async () => { - const oldDrain = deferred(); - const newDrain = deferred(); - const framework = Object.create(AgentFramework.prototype) as any; - framework.discordAwarenessBarrier = null; - framework.discordAwarenessBarrierGeneration = 0; - const drains = [oldDrain.promise, newDrain.promise]; - framework.beginDiscordAwarenessBarrier = () => ({ - requiresBarrier: true, - promise: drains.shift(), - }); - - let readyCalls = 0; - let newer: any; - const connection = { - id: 'discord', - pauseDataPlane: () => {}, - ready: () => { readyCalls++; }, - readyControlPlane: () => { - if (newer) return; - newer = framework.installMcplDataPlaneGate(); - framework.releaseMcplDataPlaneGate(newer); - }, - }; - framework.mcplServerRegistry = { getAllServers: () => [connection] }; - - const older = framework.installMcplDataPlaneGate(); - framework.releaseMcplDataPlaneGate(older); - assert.equal(newer.generation, older.generation + 1); - - oldDrain.resolve({ status: 'delivered', delivered: 1, failed: 0 }); - await older.promise; - assert.equal(framework.completeMcplDataPlaneGate(older), false); - assert.equal(readyCalls, 0, 'old completion leaves fresh data held'); - - newDrain.resolve({ status: 'delivered', delivered: 1, failed: 0 }); - await newer.promise; - assert.equal(framework.completeMcplDataPlaneGate(newer), true); - assert.equal(readyCalls, 1); -}); - -test('overlapping reconnect/list-change generations release only the newest barrier', async () => { - const reconnectDrain = deferred(); - const listChangeDrain = deferred(); - const framework = Object.create(AgentFramework.prototype) as any; - framework.discordAwarenessBarrier = null; - framework.discordAwarenessBarrierGeneration = 0; - const drains = [reconnectDrain.promise, listChangeDrain.promise]; - framework.beginDiscordAwarenessBarrier = () => ({ - requiresBarrier: true, - promise: drains.shift(), - }); - let readyCalls = 0; - const connection = { - id: 'discord', - pauseDataPlane: () => {}, - readyControlPlane: () => {}, - ready: () => { readyCalls++; }, - }; - framework.mcplServerRegistry = { getAllServers: () => [connection] }; - - const reconnectBarrier = framework.installMcplDataPlaneGate(); - framework.releaseMcplDataPlaneGate(reconnectBarrier); - const listChangeBarrier = framework.installMcplDataPlaneGate(); - framework.releaseMcplDataPlaneGate(listChangeBarrier); - - reconnectDrain.resolve({ status: 'delivered', delivered: 1, failed: 0 }); - await reconnectBarrier.promise; - framework.completeMcplDataPlaneGate(reconnectBarrier); - assert.equal(readyCalls, 0, 'reconnect completion cannot bypass overlapping list-change work'); - - listChangeDrain.resolve({ status: 'delivered', delivered: 1, failed: 0 }); - await listChangeBarrier.promise; - framework.completeMcplDataPlaneGate(listChangeBarrier); - assert.equal(readyCalls, 1); -}); - -test('synchronous zero-pending flush rechecks the gate after a nested list change', () => { - const connection = new (McplServerConnection as any)( - 'discord', null, null, - ) as McplServerConnection; - // This test exercises plane-flush mechanics, not authorization: admission - // now enforces the grant (§5.4), so model the post-policy state or the - // push is correctly discarded before it can count. - connection.establishGrant(new CapabilityGrant(new Set(['pushEvents']), [])); - let dataDeliveries = 0; - connection.on('tools-list-changed', () => connection.pauseDataPlane()); - connection.on('push-event', () => { dataDeliveries++; }); - - connection.emit('tools-list-changed'); - connection.emit('push-event', {}); - connection.ready(); - assert.equal( - dataDeliveries, - 0, - 'the nested list-change pause must retain later items from the same ready() snapshot', - ); - - connection.ready(); - assert.equal(dataDeliveries, 1); -}); diff --git a/test/mcpl-reconnect-reregistration.test.ts b/test/mcpl-reconnect-reregistration.test.ts index a7627a77f..35a1fadd6 100644 --- a/test/mcpl-reconnect-reregistration.test.ts +++ b/test/mcpl-reconnect-reregistration.test.ts @@ -131,14 +131,11 @@ async function makeHarness() { getAllServers: () => [connection], getServer: (id: string) => id === connection.id ? connection : null, }; - fw.discordAwarenessBarrier = null; - fw.discordAwarenessBarrierGeneration = 0; fw.wireMcplEvents(connection); await fw.registerMcplServerFeatures(config, connection); - // The push handler responds synchronously when no awareness barrier is - // up, but during reconnect the §5.3 policy round-trip holds the barrier - // across a microtask — so the harness settles the loop once before + // During reconnect the §5.3 policy round-trip spans a microtask before + // the data plane reopens — so the harness settles the loop once before // asserting. (Was 'must respond synchronously' pre-0.5.) const sendPush = async (eventId: string): Promise => { let result: PushEventResult | undefined; diff --git a/test/offline-recovery-branch.test.ts b/test/offline-recovery-branch.test.ts index c514bc3d5..8b34e9aff 100644 --- a/test/offline-recovery-branch.test.ts +++ b/test/offline-recovery-branch.test.ts @@ -30,6 +30,7 @@ test('offline recovery branches without compiling and queues discarded Discord r store.close(); const result = await createOfflineRecoveryBranch({ + marks: { scope: 'all' }, storePath, agentName: 'cairn', messageId: 'm-safe', @@ -54,9 +55,9 @@ test('offline recovery branches without compiling and queues discarded Discord r const batches = new DiscordAwarenessOutbox( defaultDiscordAwarenessOutboxPath(storePath), - ).pending('discord'); + ).pendingDispatches('discord'); assert.equal(batches.length, 2); - assert.deepEqual(batches.map((operation) => operation.ref.messageId), ['m-toxic-1', 'm-toxic-2']); + assert.deepEqual(batches.map((dispatch) => dispatch.key.messageId), ['m-toxic-1', 'm-toxic-2']); // The outbox is metadata-only: quarantined text must never leak to it. const rawOutbox = await import('node:fs').then(({ readFileSync }) => @@ -91,6 +92,7 @@ test('offline recovery can branch at the current message and suppress an exact l store.close(); const result = await createOfflineRecoveryBranch({ + marks: { scope: 'all' }, storePath, agentName: 'cairn', messageId: 'm-current', @@ -150,6 +152,7 @@ test('offline recovery suppresses inclusive ranges in context order', async () = store.close(); const result = await createOfflineRecoveryBranch({ + marks: { scope: 'all' }, storePath, agentName: 'cairn', messageId: 'm-current', @@ -179,10 +182,10 @@ test('offline recovery suppresses inclusive ranges in context order', async () = const batches = new DiscordAwarenessOutbox( defaultDiscordAwarenessOutboxPath(storePath), - ).pending('discord'); + ).pendingDispatches('discord'); assert.equal(batches.length, 3); assert.deepEqual( - batches.map((operation) => operation.ref.messageId), + batches.map((dispatch) => dispatch.key.messageId), ['m-from', 'm-inside', 'm-to'], ); } finally { @@ -207,6 +210,7 @@ test('offline recovery accepts an exact internal context ID anchor', async () => store.close(); const result = await createOfflineRecoveryBranch({ + marks: { scope: 'all' }, storePath, agentName: 'cairn', contextId: String(coherentAssistantId), @@ -268,3 +272,56 @@ test('suppression rejects a range that splits a tool exchange', async () => { rmSync(dir, { recursive: true, force: true }); } }); + +test('offline recovery is local by default; addressed marks only messages that addressed the agent', async () => { + const dir = mkdtempSync(join(tmpdir(), 'offline-recovery-marks-')); + const storePath = join(dir, 'agent.chronicle'); + try { + const seed = async () => { + const store = JsStore.openOrCreate({ path: storePath }); + const cm = await ContextManager.open({ store, namespace: 'agents/cairn' }); + cm.addMessage('user', [{ type: 'text', text: 'safe' }], { + serverId: 'discord', channelId: 'discord:g1:c1', messageId: 'm-safe', + }); + cm.addMessage('user', [{ type: 'text', text: 'hey @cairn' }], { + serverId: 'discord', channelId: 'discord:g1:c1', messageId: 'm-addressed', tags: ['chat:addressed', 'chat:mention'], + }); + cm.addMessage('user', [{ type: 'text', text: 'ambient chatter' }], { + serverId: 'discord', channelId: 'discord:g1:c1', messageId: 'm-ambient', tags: ['chat:ambient'], + }); + cm.close(); + store.close(); + }; + await seed(); + const local = await createOfflineRecoveryBranch({ + storePath, agentName: 'cairn', messageId: 'm-safe', branchName: 'recovery/cairn/local', + }); + assert.equal(local.marksScope, 'none'); + assert.equal(local.discordAddressable, 2); + assert.equal(local.discordMarkersQueued, 0); + assert.equal(local.unmarked, 2); + const outbox = new DiscordAwarenessOutbox(defaultDiscordAwarenessOutboxPath(storePath)); + assert.equal(outbox.pendingDispatches('discord').length, 0, 'nothing is queued for Discord'); + assert.equal(outbox.batches().length, 0); + + // A dry run with addressed shows exactly what would be marked. + rmSync(storePath, { recursive: true, force: true }); + await seed(); + const preview = await createOfflineRecoveryBranch({ + storePath, agentName: 'cairn', messageId: 'm-safe', marks: { scope: 'addressed' }, dryRun: true, + }); + assert.deepEqual(preview.refs.map((ref) => ref.messageId), ['m-addressed']); + const marked = await createOfflineRecoveryBranch({ + storePath, agentName: 'cairn', messageId: 'm-safe', marks: { scope: 'addressed' }, branchName: 'recovery/cairn/marked', + }); + assert.equal(marked.discordMarkersQueued, 1); + assert.equal(marked.unmarked, 1); + assert.deepEqual( + new DiscordAwarenessOutbox(defaultDiscordAwarenessOutboxPath(storePath)).pendingDispatches('discord') + .map((dispatch) => dispatch.key.messageId), + ['m-addressed'], + ); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); diff --git a/test/operator-surgery.test.ts b/test/operator-surgery.test.ts index aeb162b59..f6180a632 100644 --- a/test/operator-surgery.test.ts +++ b/test/operator-surgery.test.ts @@ -89,7 +89,8 @@ describe('live operator surgery', () => { assert.equal(entry.agent, 'scout'); assert.deepEqual(entry.requester, { via: 'webui', name: 'antra' }); assert.equal(entry.note, 'bad turn'); - assert.deepEqual(entry.params, { messageId: ids[2] }); + // The marks choice is recorded with the operator's other parameters. + assert.deepEqual(entry.params, { messageId: ids[2], marks: 'none' }); assert.equal(entry.result?.messagesRemoved, 2); assert.equal(entry.result?.targetBranch, r.targetBranch); assert.ok(typeof entry.at === 'string' && !Number.isNaN(Date.parse(entry.at))); @@ -297,7 +298,7 @@ describe('live operator surgery', () => { const entry = readLog(framework.getOperatorLogPath()!).find((e) => e.kind === 'suppress'); assert.ok(entry); - assert.deepEqual(entry.params, { messageIds: [ids[1], ids[3]] }); + assert.deepEqual(entry.params, { messageIds: [ids[1], ids[3]], marks: 'none' }); assert.deepEqual(new Set(entry.result?.removedIds as string[]), new Set([ids[1], ids[3]])); }); diff --git a/test/surgery-marks-receipt.test.ts b/test/surgery-marks-receipt.test.ts index f2a1b4c1b..674d08f90 100644 --- a/test/surgery-marks-receipt.test.ts +++ b/test/surgery-marks-receipt.test.ts @@ -1,17 +1,22 @@ /** - * Live surgery reports its body change apart from the Discord awareness marks - * it schedules, and does not wait on Discord to accept them. + * Live surgery and Discord awareness marks, end to end against a fake + * Discord MCPL server. * * Field incident (2026-10-07): a web-UI rollback removed 918 Discord messages - * from a resident's context. rollbackToMessage awaited the serial reaction - * drain inside its store reservation, so the operator's surgery-result never - * arrived within the client's 60 s, the store stayed reserved, and every MCPL - * data plane stayed paused for the whole drain. These tests hold the fake - * server's reaction replies to model that drain. + * from a resident's context, and the framework queued a 💤 reaction on every + * one of them, mostly other people's ambient messages in a busy channel. The + * rollback awaited the serial reaction drain inside its store reservation and + * held every MCPL data plane until it finished. * - * The receipt is a scheduling statement (`markers`), never a delivery claim, - * and a ledger failure after the body change landed is reported as - * `not-scheduled` instead of a failed surgery an operator might retry. + * The contract these tests pin: + * - a surgery is local unless the operator chooses marks (`marks`, default + * `none`); `addressed` covers messages tagged chat:addressed; a choice + * with `refs` marks only those refs, never later arrivals; + * - the surgery returns with a marker-scheduling receipt, never a delivery + * claim, and never fails an applied body change over marker bookkeeping; + * - delivery runs in the background and never holds MCPL traffic; + * - cancel and retract are explicit, and their receipts disclose requests + * whose outcome is unknown. */ import { describe, it, beforeEach, afterEach } from 'node:test'; import assert from 'node:assert/strict'; @@ -19,6 +24,7 @@ import { fileURLToPath } from 'node:url'; import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; +import { JsStore } from '@animalabs/chronicle'; import { AgentFramework, DiscordAwarenessOutbox } from '../src/index.js'; import type { OperatorLogEntry } from '../src/index.js'; import { MockMembrane } from './helpers/mock-membrane.js'; @@ -54,7 +60,7 @@ async function settlesWithin(promise: Promise, ms: number, what: string): } } -describe('surgery marker receipt', () => { +describe('surgery and awareness marks', () => { let dir: string; let storePath: string; let callsPath: string; @@ -64,35 +70,35 @@ describe('surgery marker receipt', () => { let framework: AgentFramework; const cm = () => framework.getAgent('resident')!.getContextManager(); - const outboxPath = () => join(storePath, 'recovery', 'discord-awareness-outbox.json'); - const ledger = () => new DiscordAwarenessOutbox(outboxPath()).batches(); + const outbox = () => new DiscordAwarenessOutbox(join(storePath, 'recovery', 'discord-awareness-journal.jsonl')); + const answered = () => jsonl(eventsPath).filter((e) => e.event === 'reaction-answered').length; + const calls = () => jsonl<{ name: string; args: { messageId: string; emoji: string } }>(callsPath); - async function start(withDiscord: boolean): Promise { + async function start(withDiscord: boolean, awarenessDeadlineMs?: number): Promise { framework = await AgentFramework.create({ storePath, membrane: new MockMembrane().asMembrane(), agents: [{ name: 'resident', model: 'test-model', systemPrompt: 'You are a resident.' }], modules: [], maintenanceIntervalMs: 0, + ...(awarenessDeadlineMs ? { discordAwarenessDeadlineMs: awarenessDeadlineMs } : {}), ...(withDiscord ? { mcplServers: [{ id: 'discord', command: process.execPath, args: [FIXTURE], - env: { - CALLS_PATH: callsPath, - EVENTS_PATH: eventsPath, - HOLD_PATH: holdPath, - PROBE_PATH: probePath, - }, + env: { CALLS_PATH: callsPath, EVENTS_PATH: eventsPath, HOLD_PATH: holdPath, PROBE_PATH: probePath }, enabledFeatureSets: ['chat'], }], } : {}), }); } - /** A kept message, the rollback target, then `n` addressable messages. */ - function seed(n: number): { tail: string; removed: string[] } { + /** + * A kept message, the rollback target, then `n` messages from a busy + * channel; those whose index is in `addressed` addressed the resident. + */ + function seed(n: number, addressed: number[] = []): { tail: string; removed: string[] } { cm().addMessage('Operator', [{ type: 'text', text: 'kept' }], { serverId: 'discord', channelId: 'discord:g1:dm', messageId: 'kept', }); @@ -101,8 +107,11 @@ describe('surgery marker receipt', () => { })); const removed: string[] = []; for (let i = 0; i < n; i++) { - removed.push(String(cm().addMessage(`Member${i}`, [{ type: 'text', text: `ambient ${i}` }], { - serverId: 'discord', channelId: 'discord:g1:c1', messageId: `amb-${i}`, + removed.push(String(cm().addMessage(`Member${i}`, [{ type: 'text', text: `message ${i}` }], { + serverId: 'discord', + channelId: 'discord:g1:c1', + messageId: `amb-${i}`, + tags: addressed.includes(i) ? ['chat:addressed', 'chat:mention'] : ['chat:ambient'], }))); } return { tail, removed }; @@ -123,144 +132,114 @@ describe('surgery marker receipt', () => { rmSync(dir, { recursive: true, force: true }); }); - it('rollback returns once the batch is active, before Discord has accepted any reaction', async () => { + it('a rollback is local by default: no reaction, and the receipt counts what stayed unmarked', async () => { await start(true); - const { tail } = seed(3); - - const result = await settlesWithin( - framework.rollbackToMessage('resident', { messageId: tail }), - 3_000, - 'rollbackToMessage with reactions held', - ); - - assert.equal(result.messagesRemoved, 3); - assert.equal(result.markers.status, 'queued'); - assert.equal(result.markers.queued, 3); - const [batch] = ledger(); - assert.equal(batch.status, 'active', 'the batch is durably active before the call returns'); - assert.equal(result.markers.status === 'queued' && result.markers.batchId, batch.id); - assert.equal( - jsonl(eventsPath).filter((e) => e.event === 'reaction-answered').length, - 0, - 'no reaction had been accepted when the rollback returned', - ); + const { tail } = seed(5, [3]); + writeFileSync(holdPath, '1'); + const result = await framework.rollbackToMessage('resident', { messageId: tail }); + assert.equal(result.messagesRemoved, 5); + assert.equal(result.removedRefs.length, 5); + assert.deepEqual(result.markers, { scope: 'none', unmarked: 5, notRemoved: 0, status: 'none', queued: 0 }); + await framework.syncDiscordAwarenessMarkers(); + assert.equal(calls().length, 0); + assert.equal(outbox().batches().length, 0); + const entry = framework.getOperatorLog({ limit: 5 }).find((e) => e.kind === 'rollback'); + assert.equal(entry?.params?.marks, 'none'); + }); - // Delivery continues in the background and completes once Discord answers. + it('addressed marks only the messages that addressed the resident', async () => { + await start(true); + const { tail } = seed(6, [1, 4]); writeFileSync(holdPath, '1'); - await waitFor('all three reactions answered', () => - jsonl(eventsPath).filter((e) => e.event === 'reaction-answered').length === 3); + const preview = framework.previewSurgeryMarks('resident', { rollbackTo: tail }); + assert.equal(preview.messagesRemoved, 6); + assert.equal(preview.addressable, 6); + assert.equal(preview.scopes.addressed.count, 2); + assert.deepEqual(preview.scopes.addressed.channels, [{ channelId: 'discord:g1:c1', count: 2 }]); + assert.equal(preview.scopes.all.count, 6); + + const result = await framework.rollbackToMessage('resident', { messageId: tail, marks: { scope: 'addressed' } }); + assert.equal(result.markers.status, 'queued'); + assert.equal(result.markers.queued, 2); + assert.equal(result.markers.unmarked, 4); + await waitFor('two reactions answered', () => answered() === 2); await framework.syncDiscordAwarenessMarkers(); - const entries = ledger()[0].refs; - assert.deepEqual(entries.map((ref) => ref.deliveryStatus), ['applied', 'applied', 'applied']); - assert.deepEqual( - jsonl<{ name: string; args: { emoji: string } }>(callsPath).map((call) => `${call.name}:${call.args.emoji}`), - ['add_reaction:💤', 'add_reaction:💤', 'add_reaction:💤'], - ); + assert.deepEqual(calls().map((c) => `${c.name}:${c.args.messageId}:${c.args.emoji}`).sort(), [ + 'add_reaction:amb-1:💤', + 'add_reaction:amb-4:💤', + ]); + }); - const logged = new Map( - framework.getOperatorLog({ limit: 10 }).map((entry) => [entry.kind, entry]), - ); - assert.deepEqual(logged.get('rollback')?.result?.markers, result.markers); + it('a choice bound to previewed refs never widens to messages that arrive before it applies', async () => { + await start(true); + const { tail } = seed(3, [0, 2]); + writeFileSync(holdPath, '1'); + const preview = framework.previewSurgeryMarks('resident', { rollbackTo: tail }); + const authorized = preview.scopes.addressed.refs; + assert.equal(authorized.length, 2); + // An addressed message arrives while the operator is deciding. + cm().addMessage('Late', [{ type: 'text', text: '@resident are you there?' }], { + serverId: 'discord', channelId: 'discord:g1:c1', messageId: 'late-1', tags: ['chat:addressed'], + }); + const result = await framework.rollbackToMessage('resident', { + messageId: tail, + marks: { scope: 'addressed', refs: authorized }, + }); + assert.equal(result.messagesRemoved, 4, 'the rollback still removes everything after its anchor'); + assert.equal(result.markers.queued, 2); + assert.equal(result.markers.unmarked, 2, 'the late arrival is removed locally, unmarked'); + await waitFor('two reactions answered', () => answered() === 2); + assert.ok(!calls().some((c) => c.args.messageId === 'late-1')); }); - it('releases the store while delivery runs, but keeps MCPL traffic behind the delivery gate', async () => { + it('returns once marks are scheduled, before Discord has accepted any, and never holds channel traffic', async () => { await start(true); - const { removed } = seed(4); + const { tail, removed } = seed(4); - await settlesWithin( - framework.rollbackToMessage('resident', { messageId: removed[1] }), + const result = await settlesWithin( + framework.rollbackToMessage('resident', { messageId: tail, marks: { scope: 'all' } }), 3_000, 'rollbackToMessage with reactions held', ); - await waitFor('the first reaction call to reach the server', () => jsonl(callsPath).length >= 1); + assert.equal(result.markers.status, 'queued'); + assert.equal(result.markers.queued, 4); + assert.equal(answered(), 0, 'no reaction had been accepted when the rollback returned'); + await waitFor('the first reaction call to reach the server', () => calls().length >= 1); - // The store reservation is released: a second surgery is admitted while - // the first one's reactions are still unanswered. + // The store is free: a second surgery is admitted during delivery. + cm().addMessage('Someone', [{ type: 'text', text: 'new' }], { + serverId: 'discord', channelId: 'discord:g1:c1', messageId: 'after-1', + }); + const fresh = cm().getAllMessages().at(-1)!; const second = await settlesWithin( - framework.suppressMessages('resident', { messageIds: [removed[0]] }), + framework.suppressMessages('resident', { messageIds: [String(fresh.id)] }), 3_000, 'a second surgery during delivery', ); - assert.equal(second.messagesRemoved, 1); - assert.equal(second.markers.status, 'queued'); + assert.equal(second.markers.status, 'none'); - // The gate was installed before release: ordinary channel traffic sent - // during delivery is not answered until the reactions are. + // Channel traffic sent while reactions are held is answered anyway. writeFileSync(probePath, '1'); - await waitFor('the probe to be sent', () => jsonl(eventsPath).some((e) => e.event === 'probe-sent')); - await new Promise((resolve) => setTimeout(resolve, 200)); - assert.equal( - jsonl(eventsPath).some((e) => e.event === 'probe-answered'), - false, - 'channel traffic waits behind the delivery gate', - ); + await waitFor('the probe to be answered while reactions are still held', () => + jsonl(eventsPath).some((e) => e.event === 'probe-answered')); + assert.equal(answered(), 0, 'the probe was not waiting behind the marks'); writeFileSync(holdPath, '1'); - await waitFor('the probe to be answered after delivery', () => - jsonl(eventsPath).some((e) => e.event === 'probe-answered')); - // The server's own event order, not timestamps (two events can share a - // millisecond), and every expected reaction by id: one compared only with - // the reactions that happened to be answered would pass if the second - // surgery's batch were never delivered. - const events = jsonl<{ event: string; messageId?: string }>(eventsPath); - const probeIndex = events.findIndex((e) => e.event === 'probe-answered'); - const answeredBeforeProbe = events.slice(0, probeIndex) - .filter((e) => e.event === 'reaction-answered') - .map((e) => e.messageId) - .sort(); + await waitFor('all four adds confirmed in the journal', () => { + const journal = outbox(); + return journal.operations().filter((op) => journal.operationStatus(op) === 'confirmed').length === 4; + }); + // Exactly the rolled-back messages, nothing from the later suppression. assert.deepEqual( - answeredBeforeProbe, - ['amb-0', 'amb-2', 'amb-3'], - 'both surgeries\' reactions are answered before channel traffic is released', + outbox().operations().map((op) => `${op.action}:${op.key.messageId}`).sort(), + ['add:amb-0', 'add:amb-1', 'add:amb-2', 'add:amb-3'], ); - }); + assert.equal(removed.length, 4); - it('restoring the source while delivery runs keeps traffic gated until the running drain is done', async () => { - await start(true); - const { tail } = seed(3); - - const rollback = await settlesWithin( - framework.rollbackToMessage('resident', { messageId: tail }), - 3_000, - 'rollbackToMessage with reactions held', - ); - await waitFor('the first reaction call to reach the server', () => jsonl(callsPath).length >= 1); - - // The operator restores the source the way the branch-switch API does - // (store switch, then synchronization) while the first add_reaction still - // waits for its reply. Reconciliation makes every mark undesired, and the - // held one is not present yet, so the ledger shows nothing pending: only - // the running drain knows it will still send its saved adds. - framework.getStore().switchBranch(rollback.sourceBranch); - const restored = framework.syncDiscordAwarenessMarkers(); - - writeFileSync(probePath, '1'); - await waitFor('the probe to be sent', () => jsonl(eventsPath).some((e) => e.event === 'probe-sent')); - await new Promise((resolve) => setTimeout(resolve, 200)); - assert.equal( - jsonl(eventsPath).some((e) => e.event === 'probe-answered'), - false, - 'channel traffic waits for the running drain even though nothing appears pending', - ); - - writeFileSync(holdPath, '1'); - await settlesWithin(restored, 5_000, "the restore's synchronization"); - await waitFor('the probe to be answered after delivery', () => - jsonl(eventsPath).some((e) => e.event === 'probe-answered')); - - // Every reaction request the drain made, including the removals the - // restore needs, was answered before channel traffic was released, and - // the restore leaves no mark behind. - const events = jsonl<{ event: string }>(eventsPath); - const probeIndex = events.findIndex((e) => e.event === 'probe-answered'); - const answered = (from: number, to?: number) => - events.slice(from, to).filter((e) => e.event === 'reaction-answered').length; - assert.equal(answered(0, probeIndex), jsonl(callsPath).length, 'every reaction call answered before the probe'); - assert.equal(answered(probeIndex), 0, 'no reaction answered after the probe'); - assert.deepEqual( - ledger()[0].refs.map((ref) => [ref.messageId, ref.desired, ref.markerPresent, ref.deliveryStatus]), - [['amb-0', false, false, 'applied'], ['amb-1', false, false, 'applied'], ['amb-2', false, false, 'applied']], + const logged = new Map( + framework.getOperatorLog({ limit: 10 }).map((entry) => [entry.kind, entry]), ); + assert.deepEqual(logged.get('rollback')?.result?.markers, result.markers); }); it('a rollback whose marker bookkeeping fails after the switch reports the body as applied', async () => { @@ -272,29 +251,25 @@ describe('surgery marker receipt', () => { }; let result: Awaited>; try { - result = await framework.rollbackToMessage('resident', { messageId: tail }); + result = await framework.rollbackToMessage('resident', { messageId: tail, marks: { scope: 'all' } }); } finally { DiscordAwarenessOutbox.prototype.activate = original; } assert.equal(cm().currentBranch().name, result.targetBranch, 'the rollback stands'); - assert.equal(result.messagesRemoved, 2); assert.equal(result.markers.status, 'not-scheduled'); assert.match(result.markers.status === 'not-scheduled' ? result.markers.error : '', /injected ledger failure/); const entry = framework.getOperatorLog({ limit: 10 }).find((e) => e.kind === 'rollback'); assert.equal(entry?.error, undefined, 'logged as an applied rollback, not a failure'); - assert.equal((entry?.result?.markers as { status?: string } | undefined)?.status, 'not-scheduled'); - // "Not scheduled" holds across the reconciliation delivery runs and - // across a restart: the batch was retired, so nothing can promote it. + // "Not scheduled" holds across delivery and a full restart. writeFileSync(holdPath, '1'); await framework.syncDiscordAwarenessMarkers(); - assert.deepEqual(ledger(), []); await framework.stop(); await start(true); await framework.syncDiscordAwarenessMarkers(); - assert.deepEqual(ledger(), []); - assert.equal(jsonl(callsPath).length, 0, 'no reaction was ever sent'); + assert.equal(outbox().operations().length, 0); + assert.equal(calls().length, 0, 'no reaction was ever sent'); }); it('reports unresolved bookkeeping when the batch can be neither activated nor retired', async () => { @@ -310,19 +285,16 @@ describe('surgery marker receipt', () => { }; let result: Awaited>; try { - result = await framework.rollbackToMessage('resident', { messageId: tail }); + result = await framework.rollbackToMessage('resident', { messageId: tail, marks: { scope: 'all' } }); } finally { DiscordAwarenessOutbox.prototype.activate = activate; DiscordAwarenessOutbox.prototype.discard = discard; } assert.equal(cm().currentBranch().name, result.targetBranch, 'the rollback stands'); - // The receipt promises nothing about this batch: it is still in the - // ledger, and a reconciliation may promote and deliver it. assert.equal(result.markers.status, 'unresolved'); - const batches = ledger(); + const batches = outbox().batches(); assert.equal(batches.length, 1); assert.equal(result.markers.status === 'unresolved' && result.markers.batchId, batches[0].id); - assert.match(result.markers.status === 'unresolved' ? result.markers.error : '', /injected ledger failure/); }); it('a suppression whose marker bookkeeping fails after the last redaction stands', async () => { @@ -334,24 +306,73 @@ describe('surgery marker receipt', () => { }; let result: Awaited>; try { - result = await framework.suppressMessages('resident', { messageIds: [removed[1]] }); + result = await framework.suppressMessages('resident', { messageIds: [removed[1]], marks: { scope: 'all' } }); } finally { DiscordAwarenessOutbox.prototype.activate = original; } - assert.equal(cm().currentBranch().name, result.targetBranch, 'the suppression is not undone'); - assert.deepEqual(result.removedIds, [removed[1]]); assert.ok(!cm().getAllMessages().some((m) => String(m.id) === removed[1])); assert.equal(result.markers.status, 'not-scheduled'); - assert.deepEqual(ledger(), [], 'every interval committed, so the retired journal loses nothing'); + assert.equal(outbox().batches().every((batch) => batch.status === 'discarded' || batch.status === 'active'), true); + }); + + it('cancel stops unsent marks without removing any; retract removes this bot\'s marks and discloses an unknown add', async () => { + // A short awareness deadline turns a withheld reply into "no response". + await start(true, 150); + const { tail } = seed(3); + const result = await framework.rollbackToMessage('resident', { messageId: tail, marks: { scope: 'all' } }); + assert.equal(result.markers.status, 'queued'); + const batchId = result.markers.status === 'queued' ? result.markers.batchId : ''; + // The first add goes out and is never answered within the deadline. + await waitFor('the first add to time out as unknown', () => + outbox().operations().some((op) => outbox().operationStatus(op) === 'unknown')); + + const cancelled = framework.cancelDiscordAwareness(batchId, { requester: { via: 'test', name: 'operator' } }); + assert.equal(cancelled.confirmed, 0); + assert.equal(cancelled.unknown + cancelled.inFlight + cancelled.cancelled, 3); + assert.ok(cancelled.unknown + cancelled.inFlight >= 1, 'the timed-out add may still land'); + + // Discord finally answers the withheld request: the late add lands. + writeFileSync(holdPath, '1'); + await waitFor('the withheld add to be answered', () => answered() >= 1); + const sentBeforeRetract = calls().length; + assert.ok(calls().every((c) => c.name === 'add_reaction')); + + const retracted = framework.retractDiscordAwareness(batchId, { requester: { via: 'test', name: 'operator' } }); + assert.ok(retracted.removalsQueued >= 1); + assert.ok(retracted.keysWithUnresolvedAdds >= 1, 'the unknown add is disclosed, not assumed absent'); + await waitFor('the removals to be sent', () => + calls().slice(sentBeforeRetract).filter((c) => c.name === 'remove_reaction').length === retracted.removalsQueued); + const kinds = framework.getOperatorLog({ limit: 10 }).map((e) => e.kind); + assert.ok(kinds.includes('awareness-cancel') && kinds.includes('awareness-retract')); + }); + + it('marks chosen where no awareness journal exists are reported as not scheduled, not as unwanted', async () => { + // A caller-supplied store and no storePath: no default journal location. + framework = await AgentFramework.create({ + store: JsStore.openOrCreate({ path: storePath }), + membrane: new MockMembrane().asMembrane(), + agents: [{ name: 'resident', model: 'test-model', systemPrompt: 'You are a resident.' }], + modules: [], + maintenanceIntervalMs: 0, + operatorLogPath: false, + }); + const { tail } = seed(2, [0]); + const result = await framework.rollbackToMessage('resident', { messageId: tail, marks: { scope: 'addressed' } }); + assert.equal(result.messagesRemoved, 2); + assert.equal(result.markers.status, 'not-scheduled'); + assert.match(result.markers.status === 'not-scheduled' ? result.markers.error : '', /no awareness journal/); + assert.equal(result.markers.unmarked, 1); + const local = await framework.rollbackToMessage('resident', { messageId: String(cm().getAllMessages()[0].id) }); + assert.equal(local.markers.status, 'none', 'nothing chosen is still none'); }); - it('a surgery that removes nothing addressable schedules no marks', async () => { + it('a surgery that removes nothing addressable schedules no marks even when marks are chosen', async () => { await start(false); const keep = String(cm().addMessage('Operator', [{ type: 'text', text: 'kept' }], {})); cm().addMessage('resident', [{ type: 'text', text: 'local only' }], {}); - const result = await framework.rollbackToMessage('resident', { messageId: keep }); - assert.deepEqual(result.markers, { status: 'none', queued: 0 }); - assert.equal(ledger().length, 0); + const result = await framework.rollbackToMessage('resident', { messageId: keep, marks: { scope: 'all' } }); + assert.deepEqual(result.markers, { scope: 'all', unmarked: 0, notRemoved: 0, status: 'none', queued: 0 }); + assert.equal(outbox().batches().length, 0); }); }); From 6bb0c61e201f6eca7f7131861383cb977534beae Mon Sep 17 00:00:00 2001 From: Ada-1017 Date: Wed, 7 Oct 2026 07:43:33 -0400 Subject: [PATCH 13/73] docs(awareness): document opt-in, one-shot marks, operator controls and the journal README: the offline-recovery section now says recovery is local by default and explains --marks addressed|all, --dry-run counts and the journal path; the old ledger/projection/global-generation paragraph is replaced by --awareness controls; a new "Live surgery and Discord awareness marks" section covers marks/preview/frozen refs, the receipt statuses (including not-scheduled when no journal exists), background non-gating delivery and the deadline/emoji config, one-shot semantics with unknown outcomes and held batches, the operator controls, host/command undo/hide marks and the marks verb, and the one-time v2 import. Changelog: breaking (default none + who must act/migration/unchanged; the DiscordAwarenessOutbox API), changed (one-shot; delivery no longer gates MCPL; corrupt journal still stops startup), added (preview + controls on every surface). #246's fixed fragment drops "behind the same MCPL data-plane gate as before", which this branch makes untrue. --- README.md | 103 ++++++++++++++---- .../fix-surgery-marks-receipt.fixed.md | 19 ++-- changelog.d/surgery-marks-controls.added.md | 18 +++ changelog.d/surgery-marks-one-shot.changed.md | 17 +++ changelog.d/surgery-marks-opt-in.breaking.md | 35 ++++++ 5 files changed, 159 insertions(+), 33 deletions(-) create mode 100644 changelog.d/surgery-marks-controls.added.md create mode 100644 changelog.d/surgery-marks-one-shot.changed.md create mode 100644 changelog.d/surgery-marks-opt-in.breaking.md diff --git a/README.md b/README.md index 9edfeb41b..d3d9d359c 100644 --- a/README.md +++ b/README.md @@ -161,15 +161,18 @@ agent-framework-recover \ The specified Discord message is the last message the agent will still see; everything after it is left on the old branch. The command scans in bounded -windows, never compiles message content, activates -the new Chronicle branch, and writes only Discord `{serverId, channelId, -messageId}` metadata to -`/recovery/discord-awareness-outbox.json`. When the host and that -agent's `discord-mcpl` bot reconnect, each addressable discarded message is -marked with 💤. Delivery state is recorded per message: retryable failures -remain queued, permanent deleted/inaccessible-message failures remain in the -audit ledger without blocking later markers. Portal and other non-Discord -records are ignored. +windows, never compiles message content, and activates the new Chronicle +branch. By default the recovery is local to the resident: nothing is posted to +Discord. To let the people involved see that the agent no longer has their +messages, choose awareness marks explicitly with `--marks addressed` (removed +messages that mentioned or replied to the bot, and DMs) or `--marks all` +(every removed Discord message). The output counts what will be marked and +what stays unmarked, and `--dry-run` shows exactly which addresses each choice +covers. Only `{serverId, channelId, messageId}` metadata is written, to the +awareness journal `/recovery/discord-awareness-journal.jsonl`, and the +marks are delivered when the host and that agent's `discord-mcpl` bot next +connect (see [Live surgery and Discord awareness marks](#live-surgery-and-discord-awareness-marks)). +Portal and other non-Discord records are ignored. When the safe point is an assistant/tool-side record rather than a Discord message, use its exact ContextManager message ID: @@ -202,7 +205,7 @@ agent-framework-recover \ `--suppress-range ..` suppresses the inclusive context interval between two Discord messages, including intervening agent/tool entries. These removals exist only on the recovery branch; the source branch remains intact. -Suppressed Discord messages are queued for the same 💤 awareness marker. +Suppressed Discord messages follow the same `--marks` choice. Selections that split a sharded body group or a tool-use/tool-result exchange are rejected. The suppression plan is journaled before the branch switch; if the recovery process is interrupted between interval removals, framework @@ -212,17 +215,10 @@ For old stores whose messages lack `metadata.serverId`, supply normal agent host must be stopped while this command has the Chronicle store open. -The marker sidecar is a retained operation ledger, not a delete-on-success -queue. Switching back to the source branch queues removal of the bot's marker; -returning to the recovery branch queues it again. Initial MCPL events remain -buffered until the ledger has been reconciled. Startup, reconnect, runtime -list-change, and online undo use one framework-global generation: every MCPL -data plane waits while all control planes remain live for registration and -marker service. Awareness marker calls also have a mandatory deadline that is -independent of `requestTimeoutMs` (including when that value is `0`); configure -it with `discordAwarenessDeadlineMs` (default 10000ms, clamped to 50..60000ms). -Configure the online marker with `discordAwarenessEmoji`; the offline CLI -accepts `--emoji`. +With the host stopped, the same command inspects and controls the awareness +journal: `--awareness list`, `--awareness cancel `, +`--awareness retract ` and `--awareness release ` behave as +the live controls described below. ### MCPL (MCP Live) @@ -324,6 +320,71 @@ servers granted `allowHostCommands`. The HTTP host verbs accept an optional shared secret (`ApiServerConfig.adminToken`, sent as `x-admin-token`) for deployments that front the port with a proxy. +### Live surgery and Discord awareness marks + +`rollbackToMessage(agent, { messageId })` forks the agent's context at a +message and switches to the fork; `suppressMessages(agent, { messageIds })` +forks at the head and removes the chosen messages on the fork. The source +branch stays intact either way. Removing Discord messages from an agent's +context is local to that agent unless the operator also chooses to mark them: + +```typescript +const preview = framework.previewSurgeryMarks('cairn', { rollbackTo: messageId }); +// preview.scopes.addressed / .all: count, per-channel counts and refs +const result = await framework.rollbackToMessage('cairn', { + messageId, + marks: { scope: 'addressed', refs: preview.scopes.addressed.refs }, +}); +// result.markers: { status: 'none' | 'queued' | 'not-scheduled' | 'unresolved', ... } +``` + +- **`marks`** is `'none'` (the default), or `{ scope: 'addressed' | 'all', refs? }`. + `addressed` covers messages tagged `chat:addressed` (mentions, replies to + the bot, DMs). Passing a preview's `refs` binds the choice to exactly that + set: messages that arrive before the surgery applies are removed locally but + never marked. The choice is recorded in the operator log. +- **The surgery returns once marks are scheduled**, never after Discord has + accepted them. `result.markers` says whether they were scheduled (`queued` + with a count and batch id), not chosen or not in scope (`none`), chosen but + never to be delivered (`not-scheduled`: the framework has no awareness + journal, having neither `storePath` nor `discordAwarenessOutboxPath`, or + recording the batch failed after the body change and the batch was retired; + `error` says which), or + `unresolved` (neither activation nor retirement could be recorded; it names + a batch that may still be delivered). Every receipt also counts the removed + messages left unmarked. Marker bookkeeping never fails or undoes an applied + rollback or suppression. +- **Delivery runs in the background** and never holds MCPL traffic or turns. + A route that is not connected keeps its work queued until it connects. + Each reaction call has a mandatory deadline independent of + `requestTimeoutMs` (including `0`): `discordAwarenessDeadlineMs` (default + 10000ms, clamped to 50..60000ms). The emoji is `discordAwarenessEmoji` + (offline: `--emoji`). +- **Marks are one-shot.** Switching branches, undo/redo and restarts never add + or remove a mark. The journal is an append-only history of requests and + their attempts, written ahead of each dispatch. A request written and never + answered is recorded as `unknown` and is never resolved by a later + confirmation of a different attempt. A batch whose surgery was interrupted + before its branch switch was recorded is held at startup until an operator + releases it. +- **Operator controls:** `listDiscordAwareness()`; + `cancelDiscordAwareness(batch)` stops a batch's further sends and retries + and never removes a reaction (its receipt counts requests in flight or + unknown, which may still land); `retractDiscordAwareness(batch | 'all')` + removes this bot's reaction, through each address's configured MCPL route, + from a batch's messages or from every message an add was ever sent for, and + its receipt discloses earlier adds whose outcome is unknown, any of which + may land after the removal; `releaseDiscordAwareness(batch)` queues a held + batch. Each is recorded in the operator log. +- **Over `host/command`** (servers granted `allowHostCommands`): message-granular + `undo` takes `marks: 'none' | 'addressed' | 'all'` (default `none`; any + other value is refused), and the `marks` verb takes `action: 'list' | + 'cancel' | 'retract' | 'release'` with a `batchId` (`all` for retract). + +A pre-journal awareness ledger (`discord-awareness-outbox.json`) is imported +once on first use: its recorded outcomes are kept, its undelivered work is +held for an explicit release, and the old file is renamed `.migrated-v2`. + ## Observability Subscribe to trace events for logging, UI updates, or debugging: diff --git a/changelog.d/fix-surgery-marks-receipt.fixed.md b/changelog.d/fix-surgery-marks-receipt.fixed.md index 20abfe420..0fc3c82a3 100644 --- a/changelog.d/fix-surgery-marks-receipt.fixed.md +++ b/changelog.d/fix-surgery-marks-receipt.fixed.md @@ -5,15 +5,10 @@ operator got no answer, for the whole serial drain. At the reported pace of about one reaction per second, 918 queued marks would take roughly 15 minutes; the household's client timed out after 60 seconds. Delivery now - continues in the background, behind the same MCPL data-plane gate as - before. A gate installed while a drain is still running (restoring the - source branch mid-delivery, say) now waits for that drain even when the - switch leaves nothing pending, so traffic stays held through the drain's - remaining reactions and the removals they need. The result's new - `markers` field (`none` | `queued` | `not-scheduled` | `unresolved`) says - whether marks were scheduled, never that Discord accepted them. If the - ledger write after the switch fails, the rollback that already applied is - no longer reported as failed: the batch is retired (`not-scheduled`), or, - if even that write fails, the result says the batch may still be delivered - (`unresolved`). A suppression whose redactions all committed is no longer - undone by that failure. + continues in the background. The result's new `markers` field (`none` | + `queued` | `not-scheduled` | `unresolved`) says whether marks were scheduled, never + that Discord accepted them. If the ledger write after the switch fails, the + rollback that already applied is no longer reported as failed: the batch is + retired (`not-scheduled`), or, if even that write fails, the result says + the batch may still be delivered (`unresolved`). A suppression whose + redactions all committed is no longer undone by that failure. diff --git a/changelog.d/surgery-marks-controls.added.md b/changelog.d/surgery-marks-controls.added.md new file mode 100644 index 000000000..2746480cc --- /dev/null +++ b/changelog.d/surgery-marks-controls.added.md @@ -0,0 +1,18 @@ +- Operator controls for awareness marks: + - `previewSurgeryMarks(agent, { rollbackTo } | { suppress })` shows what a + surgery would remove and which messages each scope would mark, per + channel. + - `listDiscordAwareness()` lists the journal's batches. + - `cancelDiscordAwareness(batch)` stops a batch's unsent requests and + retries, and never removes a reaction. Its receipt counts requests in + flight or unknown, which may still land. + - `retractDiscordAwareness(batch | 'all')` queues removal of this bot's + reaction through each message's configured MCPL route. Its receipt + discloses earlier adds whose outcome is unknown, which may land after the + removal. + - `releaseDiscordAwareness(batch)` queues a held batch. + + Each control is recorded in the operator log. The same controls are + available as the `host/command` `marks` verb (`action: list | cancel | + retract | release`, `batchId`) and, with the host stopped, as + `agent-framework-recover --awareness list|cancel|retract|release`. diff --git a/changelog.d/surgery-marks-one-shot.changed.md b/changelog.d/surgery-marks-one-shot.changed.md new file mode 100644 index 000000000..1a62549b3 --- /dev/null +++ b/changelog.d/surgery-marks-one-shot.changed.md @@ -0,0 +1,17 @@ +- Awareness marks are one-shot. Switching branches, undo/redo and restarts no + longer add or remove marks; previously a mark followed the branch, removed + when the source branch became active and re-added when the recovery branch + did. Each request is written ahead of its dispatch. A request that went out + and got no answer is recorded as `unknown`, and a later confirmation of a + different attempt never resolves it. A batch whose surgery was interrupted + before its branch switch was recorded is held at startup until an operator + releases it. +- Awareness delivery no longer holds MCPL traffic. Previously every MCPL data + plane waited at startup, at reconnect, at a tools list change and after each + surgery until every queued mark had been attempted, which at Discord's pace + could leave the resident deaf for minutes. Delivery now runs in the + background. A route that is not connected keeps its work queued until it + connects, and a delivery failure raises an ops alert instead of recycling + connections. A journal that cannot be read still stops startup, because it + also carries the suppression-resume record that protects the agent's + context. diff --git a/changelog.d/surgery-marks-opt-in.breaking.md b/changelog.d/surgery-marks-opt-in.breaking.md new file mode 100644 index 000000000..0bcc98841 --- /dev/null +++ b/changelog.d/surgery-marks-opt-in.breaking.md @@ -0,0 +1,35 @@ +- **Hosts and operator surfaces that remove Discord messages from an agent's + context:** awareness marks are now an explicit publication choice, and the + default is none. Previously every live rollback or suppression, + message-granular `host/command` undo, and offline `agent-framework-recover` + run queued a 💤 reaction on every removed Discord message, including other + people's ambient messages in a busy channel. In the incident behind this + change, one web-UI rollback queued 918 of them. + - **Who needs to act:** a surface that should keep offering marks passes the + operator's choice: `marks: { scope: 'addressed' | 'all', refs? }` on + `rollbackToMessage`/`suppressMessages`, `marks: 'addressed' | 'all'` on + `host/command` `undo` and `hide`, and `--marks addressed|all` on the CLI. + `addressed` covers messages tagged `chat:addressed` (mentions, replies to + the bot, DMs). Show `previewSurgeryMarks()` before the choice, and pass a + scope's `refs` back to bind the choice to exactly the previewed messages. + Any other `host/command` `marks` value is refused rather than guessed. + - **Unchanged:** the body change itself (fork, switch, redaction), the + receipt's `status` values, the emoji (`discordAwarenessEmoji`) and the + reaction deadline (`discordAwarenessDeadlineMs`). Marks already on Discord + are left alone; nothing is removed automatically. + - Receipts (`markers`) now also carry `scope`, `unmarked` and `notRemoved`. + Marks that were chosen but can't be recorded, because there is no journal + (no `storePath` or `discordAwarenessOutboxPath`), are reported as + `not-scheduled` rather than `none`. + - No sibling version is required: a host that ignores the new option simply + places no marks. The connectome-host and discord-mcpl companions add the + choice to their surfaces. +- **Code using `DiscordAwarenessOutbox` directly:** the awareness ledger is now + an append-only journal of requests and attempts, + `/recovery/discord-awareness-journal.jsonl`. `pending()` and + `reconcileForBranch()` are gone. Use `pendingDispatches()`, + `recordDispatching()`/`recordOutcome()`, `view()`, and + `cancel()`/`retract()`/`release()`. A previous `discord-awareness-outbox.json` + is imported once on first use: its recorded outcomes are kept, its undelivered + work is held for an explicit release, and the file is renamed + `.migrated-v2`. From 6e9e51b5c01c61819484c882284fd2f54a79d3cf Mon Sep 17 00:00:00 2001 From: Ada-1017 Date: Wed, 7 Oct 2026 08:00:43 -0400 Subject: [PATCH 14/73] fix(awareness): bound outside error text before it is journaled, logged or returned June (#43747) pointed out that message-only logging is not bounded logging: an MCPL tool result's error text or a filesystem error has no length limit. On this lane it is also persisted: every failed or unknown outcome's error lands in the awareness journal, which is re-read on each operation. boundDiscordAwarenessText (500 chars: head + tail with the omitted length, never splitting a surrogate pair) is applied in recordOutcome before journaling; in the drain after permanence is classified on the full text and before the log line; and on the receipt/opsAlert errors of activation/retire failures, hide-mark preparation failures and background-delivery failures. Test: a 1 MB mixed-width string with an emoji near the cut stays <= 500 chars with no lone surrogate, and the journal holds only the bounded copy. --- src/framework.ts | 13 +++++++++---- src/recovery/discord-awareness-outbox.ts | 20 +++++++++++++++++++- test/discord-awareness-outbox.test.ts | 15 +++++++++++++++ 3 files changed, 43 insertions(+), 5 deletions(-) diff --git a/src/framework.ts b/src/framework.ts index 8ea76cbc8..a40b7f56d 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -407,6 +407,7 @@ import { defaultDiscordAwarenessOutboxPath, extractDiscordAwarenessRefs, selectDiscordAwarenessRefs, + boundDiscordAwarenessText, type DiscordAwarenessBatch, type DiscordAwarenessBatchView, type DiscordAwarenessCancelReceipt, @@ -6542,13 +6543,15 @@ export class AgentFramework { try { this.discordAwarenessOutbox!.activate(batch.id); } catch (error) { - const detail = error instanceof Error ? error.message : String(error); + const detail = boundDiscordAwarenessText(error instanceof Error ? error.message : String(error)); let retired = false; let retireDetail = ''; try { retired = this.discordAwarenessOutbox!.discard(batch.id); } catch (discardError) { - retireDetail = discardError instanceof Error ? discardError.message : String(discardError); + retireDetail = boundDiscordAwarenessText( + discardError instanceof Error ? discardError.message : String(discardError), + ); } console.error( `[discord-awareness] ${verb} agent=${agentName}: body applied, but marker batch ${batch.id} ` + @@ -6612,7 +6615,7 @@ export class AgentFramework { }) ?? null; } catch (error) { // The redaction stands; nothing was recorded, so nothing will be sent. - const detail = error instanceof Error ? error.message : String(error); + const detail = boundDiscordAwarenessText(error instanceof Error ? error.message : String(error)); this.opsAlert( 'discord-awareness-not-scheduled', agentName, @@ -6638,7 +6641,7 @@ export class AgentFramework { ): void { if (!this.discordAwarenessOutbox) return; this.syncDiscordAwarenessMarkers().catch((error) => { - const detail = error instanceof Error ? error.message : String(error); + const detail = boundDiscordAwarenessText(error instanceof Error ? error.message : String(error)); console.error(`[discord-awareness] delivery after ${occasion} failed: ${detail}`); this.opsAlert( 'discord-awareness-delivery', @@ -13700,7 +13703,9 @@ export class AgentFramework { outcome = 'unknown'; } } + // Classify on the full text, then keep only a bounded copy. const permanent = outcome === 'failed' && isPermanentDiscordReactionFailure(detail ?? ''); + if (detail !== undefined) detail = boundDiscordAwarenessText(detail); try { outbox.recordOutcome(attempts, outcome, { ...(permanent ? { permanent: true } : {}), diff --git a/src/recovery/discord-awareness-outbox.ts b/src/recovery/discord-awareness-outbox.ts index cde3dfc3a..c75d93c4c 100644 --- a/src/recovery/discord-awareness-outbox.ts +++ b/src/recovery/discord-awareness-outbox.ts @@ -39,6 +39,23 @@ import { createHash, randomUUID } from 'node:crypto'; export const DEFAULT_DISCORD_AWARENESS_EMOJI = '💤'; /** The journal file under a store, next to the other recovery state. */ +/** + * Bound text that came from elsewhere (a Discord error body, a filesystem + * error) before it is journaled, logged or returned in a receipt: the journal + * is re-read on every operation, and a server's error text has no length + * limit of its own. Head and tail are kept, with the omitted length stated; + * a cut never splits a surrogate pair. + */ +export function boundDiscordAwarenessText(text: string, max = 500): string { + if (text.length <= max) return text; + const tailLength = Math.min(100, Math.floor(max / 4)); + let head = text.slice(0, max - tailLength - 40); + let tail = text.slice(text.length - tailLength); + if (/[\uD800-\uDBFF]$/.test(head)) head = head.slice(0, -1); + if (/^[\uDC00-\uDFFF]/.test(tail)) tail = tail.slice(1); + return `${head} … [${text.length - head.length - tail.length} chars omitted] … ${tail}`; +} + export function defaultDiscordAwarenessOutboxPath(storePath: string): string { return join(storePath, 'recovery', 'discord-awareness-journal.jsonl'); } @@ -733,6 +750,7 @@ export class DiscordAwarenessOutbox { detail: { permanent?: boolean; error?: string } = {}, ): void { const at = Date.now(); + const error = detail.error !== undefined ? boundDiscordAwarenessText(detail.error) : undefined; try { this.append(attempts.map(({ opId, attempt }) => ({ t: 'outcome' as const, @@ -741,7 +759,7 @@ export class DiscordAwarenessOutbox { attempt, outcome, ...(outcome === 'failed' && detail.permanent ? { permanent: true } : {}), - ...(detail.error !== undefined ? { error: detail.error } : {}), + ...(error !== undefined ? { error } : {}), }))); } finally { for (const { opId } of attempts) this.inFlight.delete(opId); diff --git a/test/discord-awareness-outbox.test.ts b/test/discord-awareness-outbox.test.ts index ba8e1b9b6..4ba916680 100644 --- a/test/discord-awareness-outbox.test.ts +++ b/test/discord-awareness-outbox.test.ts @@ -16,6 +16,7 @@ import { join } from 'node:path'; import { tmpdir } from 'node:os'; import { DiscordAwarenessOutbox, + boundDiscordAwarenessText, extractDiscordAwarenessRefs, selectDiscordAwarenessRefs, type DiscordAwarenessRef, @@ -324,6 +325,20 @@ test('release queues a held batch\'s recorded operations, explicitly', withJourn assert.throws(() => outbox.release(batch.id), /not held/); })); +test('outside error text is bounded before it is journaled', withJournal((path) => { + assert.equal(boundDiscordAwarenessText('short'), 'short'); + const huge = `${'x'.repeat(300)}${'界'.repeat(500_000)}😀tail`; + const bounded = boundDiscordAwarenessText(huge); + assert.ok(bounded.length <= 500, `bounded to ${bounded.length}`); + assert.match(bounded, /chars omitted/); + assert.ok(!/[\uD800-\uDBFF](?![\uDC00-\uDFFF])|(? { const outbox = new DiscordAwarenessOutbox(path); const batch = outbox.prepare({ From 799a16b88535bdf176455b7a387f3353f69f0d5b Mon Sep 17 00:00:00 2001 From: Ada-1017 Date: Wed, 7 Oct 2026 09:37:31 -0400 Subject: [PATCH 15/73] refactor(awareness): keep the journal in the store (RecordJournal); fix June's review findings Storage (June #44070): the awareness journal moves from an fsynced JSONL file to typed Chronicle records on the shared RecordJournal (Basil, 2153e3d, merged as a shared base; Hugo's helper approval room-220 #41860). One entry = one group of records that apply together (replaces txn/commit records, the parser and torn-tail repair). The outbox keeps reduced state in memory, checkpoints every 256 entries (constructor seam for tests), reloads after an ambiguous append (RecordJournal unreconciled guard). Constructor is now (store, {legacyPath?, checkpointEvery?}); discordAwarenessOutboxPath names only a pre-journal JSON ledger to import; every framework has a journal, so the "marks chosen but no journal" receipt branch is gone. Crash ordering this fixes: Chronicle buffers branch-state commits until store.sync(), so a JSONL activation could outlive the switch it asserted. Activation, discard and suppression-complete now append with afterCommittedState (sync first) + durable; batch prepare, dispatch write-ahead, startup reconciliation and operator acts are durable; outcomes are not (a lost outcome reads as unknown and is retried). Review findings, each with a regression: - #44236 cancel vs a running drain: the drain no longer iterates a snapshot. claimDispatch(serverId, skip) recomputes due dispatches and writes the chosen one ahead in one synchronous step; recordDispatching narrows a stale dispatch to still-due ops and returns [] when none remain. - #44283/#44349/#44458 legacy uncertainty: v2 import no longer synthesizes attempts or presence. Each entry with attempts becomes legacy-evidence facts (attempts, lastAction, lastStatus, bounded lastError, lastErrorKind pre-write-refusal|discord-answer|other, outcomesUnrecorded = earlier attempts + last unless applied/pre-write/Discord answer). Retract now queues a removal for every selected key whatever history says (idempotent removal covers absence), supersedes due adds, and discloses native unresolved requests (per dispatch) and keysWithLegacyUncertainty; cancel reports legacyOutcomesUnrecorded; the view reports legacy facts. - #44493 release vs unfinished body: only activation (surgery or its resume) certifies a suppression body; release does not. The resume selector takes every non-discarded explicit suppression with an incomplete body on its exact target, whatever its marks state; prepared ones are crash-completed, others get suppression-complete only. - #44517 cancel history: cancel receipt gains unresolvedAttempts across all attempts (a timeout then a confirmed retry, or then a permanent failure). - #44551 offline CLI separates body from marks (settleActivation: activate, else retire, else unresolved; result.markers); barrier on every record that certifies or retires a body; turn-based host/command undo honours marks via an addressing-metadata diff of the agent's context. - #44585 a newer opposite request supersedes due retries (retryable failures and pre-write refusals included) of the old one, symmetric; history kept; the in-flight request still settles first. - #44609 failure paths restore the source first and retire the prepared batch only when the store is confirmed back on it; otherwise it stays as the unfinished body's record and the error says so; a failed retire is reported and alerted, never masking the surgery's error. - #44638 cancel(target) also takes a retract request id (request-cancelled); the view lists retract groups (kind 'retract') so the id survives restart. host/command marks verb param is now `target`. - Kit/Agnes-1009 (room-203 #44126): host/command hide takes reserveStoreForSurgery around its removal and marks (agent-busy otherwise). Tests: discord-awareness-outbox 24, surgery-marks-receipt 14 (real store + fake Discord server: cancel during delivery sends exactly one request; restore-unconfirmed keeps the batch; failed retire reported; prepared -> held -> release -> target-startup resumes the body), delivery 5, offline 7 (body stands when activation fails), operator-surgery 18: 68/68. --- README.md | 87 +- changelog.d/surgery-marks-controls.added.md | 19 +- changelog.d/surgery-marks-one-shot.changed.md | 10 +- changelog.d/surgery-marks-opt-in.breaking.md | 37 +- src/framework.ts | 332 +++++--- src/index.ts | 4 + src/operator-log.ts | 8 +- src/recovery/discord-awareness-outbox.ts | 784 +++++++++++------ src/recovery/offline-branch.ts | 41 +- src/recovery/recover-cli.ts | 43 +- test/discord-awareness-delivery.test.ts | 67 +- test/discord-awareness-outbox.test.ts | 792 +++++++++--------- test/offline-recovery-branch.test.ts | 79 +- test/surgery-marks-receipt.test.ts | 113 ++- 14 files changed, 1496 insertions(+), 920 deletions(-) diff --git a/README.md b/README.md index d3d9d359c..1cba189ae 100644 --- a/README.md +++ b/README.md @@ -169,9 +169,11 @@ messages that mentioned or replied to the bot, and DMs) or `--marks all` (every removed Discord message). The output counts what will be marked and what stays unmarked, and `--dry-run` shows exactly which addresses each choice covers. Only `{serverId, channelId, messageId}` metadata is written, to the -awareness journal `/recovery/discord-awareness-journal.jsonl`, and the -marks are delivered when the host and that agent's `discord-mcpl` bot next -connect (see [Live surgery and Discord awareness marks](#live-surgery-and-discord-awareness-marks)). +awareness journal kept in the store itself, and the marks are delivered when +the host and that agent's `discord-mcpl` bot next connect (see +[Live surgery and Discord awareness marks](#live-surgery-and-discord-awareness-marks)). +If the branch is made but the marks can't be recorded, the output's `markers` +says so and the branch stands. Portal and other non-Discord records are ignored. When the safe point is an assistant/tool-side record rather than a Discord @@ -216,7 +218,7 @@ normal agent host must be stopped while this command has the Chronicle store open. With the host stopped, the same command inspects and controls the awareness -journal: `--awareness list`, `--awareness cancel `, +journal: `--awareness list`, `--awareness cancel `, `--awareness retract ` and `--awareness release ` behave as the live controls described below. @@ -345,15 +347,13 @@ const result = await framework.rollbackToMessage('cairn', { never marked. The choice is recorded in the operator log. - **The surgery returns once marks are scheduled**, never after Discord has accepted them. `result.markers` says whether they were scheduled (`queued` - with a count and batch id), not chosen or not in scope (`none`), chosen but - never to be delivered (`not-scheduled`: the framework has no awareness - journal, having neither `storePath` nor `discordAwarenessOutboxPath`, or - recording the batch failed after the body change and the batch was retired; - `error` says which), or - `unresolved` (neither activation nor retirement could be recorded; it names - a batch that may still be delivered). Every receipt also counts the removed - messages left unmarked. Marker bookkeeping never fails or undoes an applied - rollback or suppression. + with a count and batch id), not chosen or not in scope (`none`), not to be + delivered because recording the batch failed after the body change and it + was retired (`not-scheduled`), or `unresolved` (neither activation nor + retirement could be recorded; it names a batch that may still be + delivered). Every receipt also counts the removed messages left unmarked. + Marker bookkeeping never fails or undoes an applied rollback or + suppression. - **Delivery runs in the background** and never holds MCPL traffic or turns. A route that is not connected keeps its work queued until it connects. Each reaction call has a mandatory deadline independent of @@ -362,28 +362,45 @@ const result = await framework.rollbackToMessage('cairn', { (offline: `--emoji`). - **Marks are one-shot.** Switching branches, undo/redo and restarts never add or remove a mark. The journal is an append-only history of requests and - their attempts, written ahead of each dispatch. A request written and never - answered is recorded as `unknown` and is never resolved by a later - confirmation of a different attempt. A batch whose surgery was interrupted - before its branch switch was recorded is held at startup until an operator - releases it. -- **Operator controls:** `listDiscordAwareness()`; - `cancelDiscordAwareness(batch)` stops a batch's further sends and retries - and never removes a reaction (its receipt counts requests in flight or - unknown, which may still land); `retractDiscordAwareness(batch | 'all')` - removes this bot's reaction, through each address's configured MCPL route, - from a batch's messages or from every message an add was ever sent for, and - its receipt discloses earlier adds whose outcome is unknown, any of which - may land after the removal; `releaseDiscordAwareness(batch)` queues a held - batch. Each is recorded in the operator log. -- **Over `host/command`** (servers granted `allowHostCommands`): message-granular - `undo` takes `marks: 'none' | 'addressed' | 'all'` (default `none`; any - other value is refused), and the `marks` verb takes `action: 'list' | - 'cancel' | 'retract' | 'release'` with a `batchId` (`all` for retract). - -A pre-journal awareness ledger (`discord-awareness-outbox.json`) is imported -once on first use: its recorded outcomes are kept, its undelivered work is -held for an explicit release, and the old file is renamed `.migrated-v2`. + their attempts, kept as typed records in the Chronicle store: it survives + rollbacks, branch deletion and a killed process, and lives with its store. + Each request is written ahead of its dispatch, and a dispatch is admitted + from the journal as it is at that moment, so a cancel takes effect even + while an earlier request is on the wire. A request written and never + answered is recorded as `unknown`, and a later confirmation of a different + attempt never resolves it. A record that certifies a body change (an + activation, a completed suppression, a retirement) is written only after + that change is synced. A batch whose surgery was interrupted before its + branch switch was recorded is held at startup until an operator releases + it; an interrupted suppression's redactions are resumed whenever its branch + is active at startup, whatever its marks' state. +- **Operator controls,** each recorded in the operator log: + - `listDiscordAwareness()` lists batches and retract requests; + - `cancelDiscordAwareness(batch | retractRequest)` stops all further sends + and retries of a batch's marks or a retract's removals, and never removes + or undoes anything. Its receipt counts requests in flight or unknown, + including earlier attempts later answered, any of which may still land; + - `retractDiscordAwareness(batch | 'all')` queues removal of this bot's + reaction, through each address's configured MCPL route, for every message + of the batch (or every message any batch or imported ledger recorded an + add for), whatever history says: removing an absent reaction does + nothing. It supersedes adds not yet sent, and its receipt discloses + earlier requests whose outcome is unknown and imported history that + leaves outcomes unrecorded; + - `releaseDiscordAwareness(batch)` queues a held batch. +- **Over `host/command`** (servers granted `allowHostCommands`): `undo` (by + messages or by turns) and `hide` take `marks: 'none' | 'addressed' | 'all'` + (default `none`; any other value is refused) and return `markers`. `hide` + holds the store like every surgery. The `marks` verb takes `action: 'list' + | 'cancel' | 'retract' | 'release'` with a `target` (a batch id; for cancel + also a retract request id; for retract also `all`). + +A pre-journal awareness ledger (`discord-awareness-outbox.json` under the +store, or `discordAwarenessOutboxPath`) is imported once on first use and the +file renamed `.migrated-v2`. What it recorded about each message is kept as +evidence (attempt count, last action, status and error, and how many attempt +outcomes it leaves unrecorded), never as a claim about what is on Discord. +Its undelivered work is held for an explicit release. ## Observability diff --git a/changelog.d/surgery-marks-controls.added.md b/changelog.d/surgery-marks-controls.added.md index 2746480cc..28be5e31f 100644 --- a/changelog.d/surgery-marks-controls.added.md +++ b/changelog.d/surgery-marks-controls.added.md @@ -2,17 +2,20 @@ - `previewSurgeryMarks(agent, { rollbackTo } | { suppress })` shows what a surgery would remove and which messages each scope would mark, per channel. - - `listDiscordAwareness()` lists the journal's batches. - - `cancelDiscordAwareness(batch)` stops a batch's unsent requests and - retries, and never removes a reaction. Its receipt counts requests in - flight or unknown, which may still land. + - `listDiscordAwareness()` lists the journal's batches and retract + requests. + - `cancelDiscordAwareness(batch | retractRequest)` stops all further sends + and retries of a batch's marks, or of a retract's removals, and never + removes or undoes anything. Its receipt counts requests in flight or + unknown, including earlier attempts later answered, which may still land. - `retractDiscordAwareness(batch | 'all')` queues removal of this bot's - reaction through each message's configured MCPL route. Its receipt - discloses earlier adds whose outcome is unknown, which may land after the - removal. + reaction, through each message's configured MCPL route, for every + selected message, whatever history says. It supersedes adds not yet + sent, and its receipt discloses requests whose outcome is unknown and + imported history that leaves outcomes unrecorded. - `releaseDiscordAwareness(batch)` queues a held batch. Each control is recorded in the operator log. The same controls are available as the `host/command` `marks` verb (`action: list | cancel | - retract | release`, `batchId`) and, with the host stopped, as + retract | release`, `target`) and, with the host stopped, as `agent-framework-recover --awareness list|cancel|retract|release`. diff --git a/changelog.d/surgery-marks-one-shot.changed.md b/changelog.d/surgery-marks-one-shot.changed.md index 1a62549b3..043a702d6 100644 --- a/changelog.d/surgery-marks-one-shot.changed.md +++ b/changelog.d/surgery-marks-one-shot.changed.md @@ -1,11 +1,17 @@ - Awareness marks are one-shot. Switching branches, undo/redo and restarts no longer add or remove marks; previously a mark followed the branch, removed when the source branch became active and re-added when the recovery branch - did. Each request is written ahead of its dispatch. A request that went out + did. Each request is written ahead of its dispatch, and a dispatch is + admitted from the journal as it is at that moment. A request that went out and got no answer is recorded as `unknown`, and a later confirmation of a different attempt never resolves it. A batch whose surgery was interrupted before its branch switch was recorded is held at startup until an operator - releases it. + releases it. An interrupted suppression's redactions are resumed whenever + its branch is active at startup, independent of what happened to its marks. +- A journal record that certifies a body change (a marks activation, a + completed suppression, a retired batch) is now written only after that + change is synced to the store. Previously a crash could keep marks active + for a rollback the store had lost. - Awareness delivery no longer holds MCPL traffic. Previously every MCPL data plane waited at startup, at reconnect, at a tools list change and after each surgery until every queued mark had been attempted, which at Discord's pace diff --git a/changelog.d/surgery-marks-opt-in.breaking.md b/changelog.d/surgery-marks-opt-in.breaking.md index 0bcc98841..0b403f5ff 100644 --- a/changelog.d/surgery-marks-opt-in.breaking.md +++ b/changelog.d/surgery-marks-opt-in.breaking.md @@ -8,28 +8,33 @@ - **Who needs to act:** a surface that should keep offering marks passes the operator's choice: `marks: { scope: 'addressed' | 'all', refs? }` on `rollbackToMessage`/`suppressMessages`, `marks: 'addressed' | 'all'` on - `host/command` `undo` and `hide`, and `--marks addressed|all` on the CLI. - `addressed` covers messages tagged `chat:addressed` (mentions, replies to - the bot, DMs). Show `previewSurgeryMarks()` before the choice, and pass a - scope's `refs` back to bind the choice to exactly the previewed messages. - Any other `host/command` `marks` value is refused rather than guessed. + `host/command` `undo` (by messages or by turns) and `hide`, and + `--marks addressed|all` on the CLI. `addressed` covers messages tagged + `chat:addressed` (mentions, replies to the bot, DMs). Show + `previewSurgeryMarks()` before the choice, and pass a scope's `refs` back + to bind the choice to exactly the previewed messages. Any other + `host/command` `marks` value is refused rather than guessed. A surface + that reacted to `hide`'s `hiddenRefs` itself should stop when the result + carries `markers`: the framework marked through its journal. - **Unchanged:** the body change itself (fork, switch, redaction), the receipt's `status` values, the emoji (`discordAwarenessEmoji`) and the reaction deadline (`discordAwarenessDeadlineMs`). Marks already on Discord are left alone; nothing is removed automatically. - Receipts (`markers`) now also carry `scope`, `unmarked` and `notRemoved`. - Marks that were chosen but can't be recorded, because there is no journal - (no `storePath` or `discordAwarenessOutboxPath`), are reported as - `not-scheduled` rather than `none`. + - `host/command` `hide` now holds the store like every surgery, so it is + refused (`agent-busy`) while any agent sharing the store is mid-turn. - No sibling version is required: a host that ignores the new option simply places no marks. The connectome-host and discord-mcpl companions add the choice to their surfaces. - **Code using `DiscordAwarenessOutbox` directly:** the awareness ledger is now - an append-only journal of requests and attempts, - `/recovery/discord-awareness-journal.jsonl`. `pending()` and - `reconcileForBranch()` are gone. Use `pendingDispatches()`, - `recordDispatching()`/`recordOutcome()`, `view()`, and - `cancel()`/`retract()`/`release()`. A previous `discord-awareness-outbox.json` - is imported once on first use: its recorded outcomes are kept, its undelivered - work is held for an explicit release, and the file is renamed - `.migrated-v2`. + an append-only journal of typed records in the Chronicle store + (`af:discord-awareness`, on the shared `RecordJournal`), so the constructor + takes the store: `new DiscordAwarenessOutbox(store, { legacyPath? })`. + `pending()` and `reconcileForBranch()` are gone. Use `claimDispatch()` (or + `pendingDispatches()` and `recordDispatching()`), `recordOutcome()`, + `view()`, `cancel()`, `retract()`, `release()` and `settleActivation()`. + `discordAwarenessOutboxPath` now only names a previous JSON ledger to + import. Such a file (by default `/recovery/discord-awareness-outbox.json`) + is imported once on first use and renamed `.migrated-v2`. What it recorded + is kept as evidence, never as synthesized attempts, and its undelivered + work is held for an explicit release. diff --git a/src/framework.ts b/src/framework.ts index a40b7f56d..fb58eea51 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -408,8 +408,9 @@ import { extractDiscordAwarenessRefs, selectDiscordAwarenessRefs, boundDiscordAwarenessText, + isPermanentDiscordReactionFailure, type DiscordAwarenessBatch, - type DiscordAwarenessBatchView, + type DiscordAwarenessView, type DiscordAwarenessCancelReceipt, type DiscordAwarenessMarks, type DiscordAwarenessReleaseReceipt, @@ -754,10 +755,6 @@ function hostCommandRequester(serverId: string, params: HostCommandParams): Oper }; } -function isPermanentDiscordReactionFailure(message: string): boolean { - return /unknown message|unknown channel|missing access|missing permissions|missing permission|cannot access|channel .* not found|message .* not found/i - .test(message); -} /** * Default error policy - retry with exponential backoff. @@ -904,8 +901,9 @@ interface HostCommandParams { marks?: string; /** For the `marks` command: `list`, `cancel`, `retract` or `release`. */ action?: string; - /** For the `marks` command: the batch to act on (`all` for retract). */ - batchId?: string; + /** For the `marks` command: a batch id; for cancel also a retract request + * id; for retract also `all`. */ + target?: string; /** For the `hide` command: Discord message id of the (first) message to * remove. With `toMessageId`, removes the inclusive range between them. */ fromMessageId?: string; @@ -1550,11 +1548,12 @@ export class AgentFramework { } } - const discordAwarenessOutboxPath = config.discordAwarenessOutboxPath - ?? (config.storePath ? defaultDiscordAwarenessOutboxPath(config.storePath) : undefined); - const discordAwarenessOutbox = discordAwarenessOutboxPath - ? new DiscordAwarenessOutbox(discordAwarenessOutboxPath) - : null; + // The awareness journal lives in the store (typed records); a + // pre-journal JSON ledger, if one is found, is imported once. + const discordAwarenessOutbox = new DiscordAwarenessOutbox(store, { + legacyPath: config.discordAwarenessOutboxPath + ?? (config.storePath ? defaultDiscordAwarenessOutboxPath(config.storePath) : undefined), + }); const operatorLogPath = config.operatorLogPath === false ? undefined @@ -5088,7 +5087,7 @@ export class AgentFramework { /** For the `marks` command: the journal's batches (`list`) or the * action's receipt. */ awareness?: - | DiscordAwarenessBatchView[] + | DiscordAwarenessView[] | DiscordAwarenessCancelReceipt | DiscordAwarenessRetractReceipt | DiscordAwarenessReleaseReceipt; @@ -5135,20 +5134,20 @@ export class AgentFramework { // The awareness journal's operator controls, for surfaces that act // through host/command (the web UI calls the framework methods). const by = { requester: hostCommandRequester(serverId, params) }; - const batchId = params.batchId; + const target = params.target; try { switch (params.action) { case 'list': return { ok: true, awareness: this.listDiscordAwareness() }; case 'cancel': - if (!batchId) return { ok: false, error: 'marks cancel needs batchId' }; - return { ok: true, awareness: this.cancelDiscordAwareness(batchId, by) }; + if (!target) return { ok: false, error: 'marks cancel needs target (a batch or retract request id)' }; + return { ok: true, awareness: this.cancelDiscordAwareness(target, by) }; case 'retract': - if (!batchId) return { ok: false, error: 'marks retract needs batchId (or all)' }; - return { ok: true, awareness: this.retractDiscordAwareness(batchId, by) }; + if (!target) return { ok: false, error: 'marks retract needs target (a batch id, or all)' }; + return { ok: true, awareness: this.retractDiscordAwareness(target, by) }; case 'release': - if (!batchId) return { ok: false, error: 'marks release needs batchId' }; - return { ok: true, awareness: this.releaseDiscordAwareness(batchId, by) }; + if (!target) return { ok: false, error: 'marks release needs target (a held batch id)' }; + return { ok: true, awareness: this.releaseDiscordAwareness(target, by) }; default: return { ok: false, error: 'marks needs action list, cancel, retract or release' }; } @@ -5287,6 +5286,18 @@ export class AgentFramework { }; } + // Like every surgery, a redaction holds the whole store: no turn (of + // any agent sharing it) may start while messages are being removed. + let release: () => void; + try { + release = this.reserveStoreForSurgery('hide', agentName); + } catch (error) { + return { + ok: false, + error: error instanceof Error ? error.message : String(error), + ...(error instanceof OperatorActionError ? { code: error.code } : {}), + }; + } try { if (params.toMessageId) { const toIdx = byDiscordId(params.toMessageId); @@ -5297,7 +5308,7 @@ export class AgentFramework { const refs = refsIn(lo, hi); const rangeIds = all.slice(lo, hi + 1).map((m) => String(m.id)); cm.removeMessages(all[lo].id, all[hi].id); - const markers = this.scheduleHideMarks(agentName, serverId, all.slice(lo, hi + 1), hideMarks); + const markers = this.scheduleAppliedMarks('hide', agentName, serverId, all.slice(lo, hi + 1), hideMarks); console.error( `[host-command] hide agent=${agentName} range removed=${hi - lo + 1} ` + `(${params.fromMessageId}..${params.toMessageId}) by=${params.requesterName ?? params.requesterId ?? 'unknown'} (server=${serverId})`, @@ -5331,7 +5342,7 @@ export class AgentFramework { const refs = refsIn(fromIdx, fromIdx); const hiddenId = String(all[fromIdx].id); cm.removeMessage(all[fromIdx].id); - const markers = this.scheduleHideMarks(agentName, serverId, [all[fromIdx]], hideMarks); + const markers = this.scheduleAppliedMarks('hide', agentName, serverId, [all[fromIdx]], hideMarks); console.error( `[host-command] hide agent=${agentName} removed=1 (${params.fromMessageId}) ` + `by=${params.requesterName ?? params.requesterId ?? 'unknown'} (server=${serverId})`, @@ -5352,6 +5363,8 @@ export class AgentFramework { }; } catch (error) { return { ok: false, error: error instanceof Error ? error.message : String(error) }; + } finally { + release(); } } @@ -5402,6 +5415,12 @@ export class AgentFramework { } const requested = Math.max(1, Math.min(20, Math.floor(params.turns ?? 1))); + const turnMarks: DiscordAwarenessMarks = params.marks === 'addressed' || params.marks === 'all' + ? { scope: params.marks } + : 'none'; + // What the undone turns remove from this agent's context is whatever is + // in it now and not after: addressing metadata only, never content. + const before = this.addressingSnapshot(agentName); let undone = 0; try { for (let i = 0; i < requested; i++) { @@ -5423,12 +5442,48 @@ export class AgentFramework { ); if (undone === 0) { - return { ok: true, undone: 0, requested, lastVisible: null }; + return { + ok: true, + undone: 0, + requested, + markers: { scope: marksScope(turnMarks) as SurgeryMarkerReceipt['scope'], unmarked: 0, notRemoved: 0, status: 'none', queued: 0 }, + lastVisible: null, + }; } - // Turn undo is a branch move: awareness marks are one-shot and never - // follow branch state, so there is nothing to reconcile here. - return { ok: true, undone, requested, lastVisible: await this.lastVisiblePreview(agentName) }; + // Marks are a one-shot choice about the messages these turns removed; + // the branch move itself never adds or removes any. + const after = new Set(this.addressingSnapshot(agentName).keys()); + const removed = [...before].filter(([id]) => !after.has(id)).map(([, carrier]) => carrier); + const markers = this.scheduleAppliedMarks('undo', agentName, serverId, removed, turnMarks); + return { ok: true, undone, requested, markers, lastVisible: await this.lastVisiblePreview(agentName) }; + } + + /** + * The addressing metadata of every message in an agent's context, by + * message id, read in bounded windows without resolving blobs: the Discord + * address and MCPL tags only, never content. + */ + private addressingSnapshot(agentName: string): Map }> { + const snapshot = new Map }>(); + const cm = this.agents.get(agentName)?.getContextManager(); + if (!cm) return snapshot; + const total = cm.getMessageCount(); + for (let offset = 0; offset < total; offset += 500) { + const { messages } = cm.getMessageWindow(offset, Math.min(500, total - offset), { resolveBlobs: false }); + for (const message of messages) { + const metadata = (message.metadata ?? {}) as Record; + snapshot.set(String(message.id), { + metadata: { + serverId: metadata.serverId, + channelId: metadata.channelId, + messageId: metadata.messageId, + tags: metadata.tags, + }, + }); + } + } + return snapshot; } /** @@ -5990,6 +6045,37 @@ export class AgentFramework { }; } + /** + * After a failed surgery has put the source back: retire its prepared + * awareness batch, but only when the store is confirmed on the source. If + * it isn't, the batch stays prepared as the evidence of an unfinished body + * (startup resumes or holds it). Never throws: the caller is already + * reporting the surgery's failure; returns a clause to append to it. + */ + private retireAfterRestore( + batch: DiscordAwarenessBatch | null, + sourceBranch: string, + agentName: string, + ): string { + if (!batch || !this.discordAwarenessOutbox) return ''; + if (this.store.currentBranch().name !== sourceBranch) { + return `; awareness batch ${batch.id} kept, since the source branch could not be confirmed`; + } + try { + this.discordAwarenessOutbox.discard(batch.id); + return ''; + } catch (error) { + const detail = boundDiscordAwarenessText(error instanceof Error ? error.message : String(error)); + this.opsAlert( + 'discord-awareness-unresolved', + agentName, + `A failed surgery's awareness batch ${batch.id} could not be retired; startup will hold it: ${detail}`, + { data: { batchId: batch.id } }, + ); + return `; awareness batch ${batch.id} could not be retired (${detail})`; + } + } + /** * After a failed switch or redaction: put the store back on `sourceBranch` * if it moved, and return a sentence saying what actually happened. Never @@ -6107,11 +6193,11 @@ export class AgentFramework { // initialization, so a rejection here can leave the store on the // new branch with an uninitialized strategy. Retire the batch and // go back to the source; keep the branch for diagnosis. - if (markerBatch) this.discordAwarenessOutbox!.discard(markerBatch.id); const restored = await this.restoreSourceBranch(cm, sourceBranch, targetBranch, agentName); + const retired = this.retireAfterRestore(markerBatch, sourceBranch, agentName); throw new OperatorActionError( 'failed', - `Rollback failed; ${restored}: ${error instanceof Error ? error.message : String(error)}`, + `Rollback failed; ${restored}${retired}: ${error instanceof Error ? error.message : String(error)}`, { cause: error }, ); } @@ -6122,7 +6208,7 @@ export class AgentFramework { scope: marksScope(opts.marks) as SurgeryMarkerReceipt['scope'], unmarked: selection.unmarked, notRemoved: selection.notRemoved, - }, selection.refs.length); + }); // Starts delivery without awaiting it (the first reaction request may // already be on the wire when this returns); it never gates traffic. this.deliverDiscordAwarenessInBackground('rollback', agentName); @@ -6242,25 +6328,24 @@ export class AgentFramework { activationPolicy: 'explicit', suppressionIntervals: ordered.map((iv) => ({ fromId: iv.fromId, toId: iv.toId })), }) ?? null; - // Anything short of activation retires the batch: a prepared explicit - // batch is resumed at startup whenever its exact target branch is - // active — e.g. after an operator opens the failed fork to look — and - // an un-completable resume there aborts framework start. - const retireBatch = (): void => { - if (markerBatch) this.discordAwarenessOutbox!.discard(markerBatch.id); - }; + // A failure short of activation puts the source back first, and only + // then retires the batch: a prepared explicit batch is resumed at + // startup whenever its exact target branch is active (e.g. after an + // operator opens the failed fork to look), and an un-completable + // resume there aborts framework start. If the source can't be + // confirmed, the batch stays as the unfinished body's evidence. try { createdBranch = await cm.fork(targetBranch); } catch (error) { // fork() switches the chronicle branch before awaiting strategy // initialization; a rejection can leave the store on the new - // branch. Retire the batch and go back to the source. - retireBatch(); + // branch. Go back to the source, then retire the batch. const restored = await this.restoreSourceBranch(cm, sourceBranch, targetBranch, agentName); + const retired = this.retireAfterRestore(markerBatch, sourceBranch, agentName); throw new OperatorActionError( 'failed', - `Suppression failed creating ${targetBranch}; ${restored}: ${error instanceof Error ? error.message : String(error)}`, + `Suppression failed creating ${targetBranch}; ${restored}${retired}: ${error instanceof Error ? error.message : String(error)}`, { cause: error }, ); } @@ -6273,11 +6358,11 @@ export class AgentFramework { // A partially suppressed branch is not safe to serve from. Keep it // for diagnosis, put the agent back on the untouched source — and // say honestly whether that restore happened. - retireBatch(); const restored = await this.restoreSourceBranch(cm, sourceBranch, createdBranch, agentName); + const retired = this.retireAfterRestore(markerBatch, sourceBranch, agentName); throw new OperatorActionError( 'failed', - `Suppression failed on ${createdBranch}; ${restored}: ${error instanceof Error ? error.message : String(error)}`, + `Suppression failed on ${createdBranch}; ${restored}${retired}: ${error instanceof Error ? error.message : String(error)}`, { cause: error }, ); } @@ -6291,7 +6376,7 @@ export class AgentFramework { scope: marksScope(opts.marks) as SurgeryMarkerReceipt['scope'], unmarked: selection.unmarked, notRemoved: selection.notRemoved, - }, selection.refs.length); + }); // Starts delivery without awaiting it (the first reaction request may // already be on the wire when this returns); it never gates traffic. this.deliverDiscordAwarenessInBackground('suppress', agentName); @@ -6427,22 +6512,23 @@ export class AgentFramework { }; } - /** The awareness journal's batches, for operator surfaces. */ - listDiscordAwareness(): DiscordAwarenessBatchView[] { + /** The awareness journal's batches and retract requests, for operator surfaces. */ + listDiscordAwareness(): DiscordAwarenessView[] { return this.discordAwarenessOutbox?.view() ?? []; } /** - * Stop a batch's awareness marks that have not been sent. Never removes a - * reaction: requests already on the wire, or whose outcome is unknown, are - * reported and may still land; retract explicitly to remove them. + * Stop all further sends and retries of a batch's marks, or of a retract + * request's removals (by its request id). Never removes a reaction and + * never undoes one: requests already on the wire, or whose outcome is + * unknown, are reported and may still land. */ cancelDiscordAwareness( - batchId: string, + target: string, opts: { requester?: OperatorRequester; note?: string } = {}, ): DiscordAwarenessCancelReceipt { - return this.awarenessOperatorAction('awareness-cancel', { batchId }, opts, () => - this.requireDiscordAwarenessOutbox().cancel(batchId, describeRequester(opts.requester))); + return this.awarenessOperatorAction('awareness-cancel', { target }, opts, () => + this.requireDiscordAwarenessOutbox().cancel(target, describeRequester(opts.requester))); } /** @@ -6519,72 +6605,48 @@ export class AgentFramework { */ private activateSurgeryMarkers( batch: DiscordAwarenessBatch | null, - verb: 'rollback' | 'suppress' | 'hide', + verb: 'rollback' | 'suppress' | 'hide' | 'undo', agentName: string, facts: SurgeryMarkerFacts, - selected: number, ): SurgeryMarkerReceipt { - if (!batch) { - if (selected > 0 && !this.discordAwarenessOutbox) { - // Chosen and in scope, but there is nowhere to record them: say so - // rather than report that none were wanted. - console.error( - `[discord-awareness] ${verb} agent=${agentName}: ${selected} mark(s) chosen, but no awareness journal is configured`, - ); - return { - ...facts, - status: 'not-scheduled', - queued: 0, - error: 'no awareness journal is configured (no storePath or discordAwarenessOutboxPath)', - }; - } - return { ...facts, status: 'none', queued: 0 }; - } - try { - this.discordAwarenessOutbox!.activate(batch.id); - } catch (error) { - const detail = boundDiscordAwarenessText(error instanceof Error ? error.message : String(error)); - let retired = false; - let retireDetail = ''; - try { - retired = this.discordAwarenessOutbox!.discard(batch.id); - } catch (discardError) { - retireDetail = boundDiscordAwarenessText( - discardError instanceof Error ? discardError.message : String(discardError), - ); - } - console.error( - `[discord-awareness] ${verb} agent=${agentName}: body applied, but marker batch ${batch.id} ` + - `could not be activated (${detail}); ` + - (retired ? 'batch retired' : `batch left prepared${retireDetail ? ` (retire failed: ${retireDetail})` : ''}`), - ); - if (batch.refs.length === 0) return { ...facts, status: 'none', queued: 0 }; - this.opsAlert( - retired ? 'discord-awareness-not-scheduled' : 'discord-awareness-unresolved', - agentName, - retired - ? `${verb} applied, but its ${batch.refs.length} awareness mark(s) were not scheduled ` + - `(batch ${batch.id} retired): ${detail}` - : `${verb} applied, but awareness batch ${batch.id} (${batch.refs.length} mark(s)) could be ` + - `neither activated nor retired; it may still be delivered: ${detail}`, - { data: { batchId: batch.id, retired } }, - ); - return retired - ? { ...facts, status: 'not-scheduled', queued: 0, error: detail } - : { ...facts, status: 'unresolved', queued: 0, batchId: batch.id, error: detail }; - } + if (!batch || !this.discordAwarenessOutbox) return { ...facts, status: 'none', queued: 0 }; + const settled = this.discordAwarenessOutbox.settleActivation(batch.id); + // A suppression-only batch carries no marks: its activation records the + // body complete, and the receipt has nothing to report. if (batch.refs.length === 0) return { ...facts, status: 'none', queued: 0 }; - return { ...facts, status: 'queued', queued: batch.refs.length, batchId: batch.id }; + if (settled.status === 'queued') { + return { ...facts, status: 'queued', queued: settled.queued, batchId: batch.id }; + } + const retired = settled.status === 'not-scheduled'; + console.error( + `[discord-awareness] ${verb} agent=${agentName}: body applied, but marker batch ${batch.id} ` + + `could not be activated (${settled.error}); ${retired ? 'batch retired' : 'batch left prepared'}`, + ); + this.opsAlert( + retired ? 'discord-awareness-not-scheduled' : 'discord-awareness-unresolved', + agentName, + retired + ? `${verb} applied, but its ${batch.refs.length} awareness mark(s) were not scheduled ` + + `(batch ${batch.id} retired): ${settled.error}` + : `${verb} applied, but awareness batch ${batch.id} (${batch.refs.length} mark(s)) could be ` + + `neither activated nor retired; it may still be delivered: ${settled.error}`, + { data: { batchId: batch.id, retired } }, + ); + return retired + ? { ...facts, status: 'not-scheduled', queued: 0, error: settled.error } + : { ...facts, status: 'unresolved', queued: 0, batchId: batch.id, error: settled.error }; } /** - * Marks for an in-place `hide`, scheduled after its redaction committed. - * The batch targets the current branch and is explicit (activated here); - * a crash between its two journal records leaves it prepared for exactly - * this branch, which startup completes. Old records without a serverId are + * Marks for a change already applied in place (an in-place `hide`, or a + * turn undo), scheduled after it. The batch targets the current branch and + * is explicit (activated here, after the change is synced); a crash + * between its two journal records leaves it prepared for exactly this + * branch, which startup completes. Old records without a serverId are * routed through the server that issued the command. */ - private scheduleHideMarks( + private scheduleAppliedMarks( + verb: 'hide' | 'undo', agentName: string, serverId: string, removed: Array<{ metadata?: Record }>, @@ -6619,12 +6681,12 @@ export class AgentFramework { this.opsAlert( 'discord-awareness-not-scheduled', agentName, - `hide applied, but its ${selection.refs.length} awareness mark(s) were not scheduled: ${detail}`, + `${verb} applied, but its ${selection.refs.length} awareness mark(s) were not scheduled: ${detail}`, ); return { ...facts, status: 'not-scheduled', queued: 0, error: detail }; } - const receipt = this.activateSurgeryMarkers(batch, 'hide', agentName, facts, selection.refs.length); - if (receipt.status === 'queued') this.deliverDiscordAwarenessInBackground('hide', agentName); + const receipt = this.activateSurgeryMarkers(batch, verb, agentName, facts); + if (receipt.status === 'queued') this.deliverDiscordAwarenessInBackground(verb, agentName); return receipt; } @@ -6636,7 +6698,7 @@ export class AgentFramework { * the queued work stays in the journal for the next pass. */ private deliverDiscordAwarenessInBackground( - occasion: 'rollback' | 'suppress' | 'hide' | 'retract' | 'release' | 'startup' | 'connect' | 'list-change' | 'reconnect', + occasion: 'rollback' | 'suppress' | 'hide' | 'undo' | 'retract' | 'release' | 'startup' | 'connect' | 'list-change' | 'reconnect', agentName: string, ): void { if (!this.discordAwarenessOutbox) return; @@ -13600,15 +13662,16 @@ export class AgentFramework { if (interval.fromId === interval.toId) cm.removeMessage(from.id); else cm.removeMessages(from.id, to.id); } - if (batch.status === 'held' || batch.cancelled) { - // Body integrity only: its marks were held at an earlier startup (and - // are released explicitly, never by a branch becoming active), or an - // operator cancelled them. - this.discordAwarenessOutbox.recordSuppressionComplete(batch.id); - console.error(`[discord-awareness] completed held suppression ${batch.id} (marks stay held)`); - } else { + if (batch.status === 'prepared' && !batch.cancelled) { + // Crash completion of the surgery itself: body, then its marks. this.discordAwarenessOutbox.activate(batch.id); console.error(`[discord-awareness] resumed suppression batch ${batch.id}`); + } else { + // Body integrity only. Its marks are the operator's: held at an + // earlier startup, released, or cancelled; a branch becoming active + // never changes them. + this.discordAwarenessOutbox.recordSuppressionComplete(batch.id); + console.error(`[discord-awareness] completed suppression body ${batch.id} (marks ${batch.status})`); } } } @@ -13648,33 +13711,32 @@ export class AgentFramework { let delivered = 0; let failed = 0; const attempted = new Set(); + const signature = (dispatch: { opIds: string[]; action: string }) => + `${dispatch.opIds.join(',')}\0${dispatch.action}`; let announced = false; while (true) { - const dispatches = this.readDiscordAwarenessDispatches(serverId).filter((dispatch) => { - const key = `${dispatch.opIds.join(',')}\0${dispatch.action}`; - if (attempted.has(key)) return false; - attempted.add(key); - return true; - }); - if (dispatches.length === 0) break; + // One dispatch at a time, chosen and written ahead in one step from + // the journal as it is now: a cancel or retract made while the + // previous request was on the wire takes effect here. + let claimed: ReturnType; + try { + claimed = outbox.claimDispatch(serverId, (dispatch) => attempted.has(signature(dispatch))); + } catch (error) { + throw new DiscordAwarenessAccountingError('dispatch write-ahead', error); + } + if (!claimed) break; + const { dispatch, attempts } = claimed; + attempted.add(signature(dispatch)); if (!announced) { announced = true; - console.error( - `[discord-awareness] ${serverId}: delivering ${dispatches.length} queued mark operation(s)`, - ); + console.error(`[discord-awareness] ${serverId}: delivering queued mark operations`); } - for (const dispatch of dispatches) { + { const ref = dispatch.key; const channelId = ref.channelId.startsWith('discord:') ? ref.channelId.split(':').at(-1)! : ref.channelId; const tool = dispatch.action === 'add' ? 'add_reaction' : 'remove_reaction'; - let attempts: Array<{ opId: string; attempt: number }>; - try { - attempts = outbox.recordDispatching(dispatch); - } catch (error) { - throw new DiscordAwarenessAccountingError('dispatch write-ahead', error); - } let outcome: 'confirmed' | 'failed' | 'not-sent' | 'unknown'; let detail: string | undefined; try { diff --git a/src/index.ts b/src/index.ts index fe873b1ce..50ae955b2 100644 --- a/src/index.ts +++ b/src/index.ts @@ -132,6 +132,7 @@ export { DEFAULT_DISCORD_AWARENESS_EMOJI, defaultDiscordAwarenessOutboxPath, extractDiscordAwarenessRefs, + isPermanentDiscordReactionFailure, selectDiscordAwarenessRefs, } from './recovery/discord-awareness-outbox.js'; export type { @@ -140,6 +141,9 @@ export type { DiscordAwarenessBatch, DiscordAwarenessBatchRecord, DiscordAwarenessBatchView, + DiscordAwarenessLegacyEvidence, + DiscordAwarenessRetractView, + DiscordAwarenessView, DiscordAwarenessCancelReceipt, DiscordAwarenessDispatch, DiscordAwarenessMarks, diff --git a/src/operator-log.ts b/src/operator-log.ts index 173192c26..ef7e87bb7 100644 --- a/src/operator-log.ts +++ b/src/operator-log.ts @@ -144,11 +144,9 @@ export interface SurgeryMarkerFacts { * - `queued` — one add request per mark is durably recorded and delivery has * started. It is not a delivery claim: per-request outcomes stay in the * awareness journal. - * - `not-scheduled` — marks were chosen and in scope, but none will be - * delivered: either this framework has no awareness journal (no storePath - * or discordAwarenessOutboxPath), or the body change landed and recording - * the batch as active failed, so it was retired from the journal. `error` - * says which. + * - `not-scheduled` — the body change landed, but recording the batch as + * active failed, so it was retired from the journal: none of its marks will + * be delivered. * - `unresolved` — the journal could record neither the activation nor the * retirement. `batchId` names a retained batch whose scheduling outcome is * unresolved: any later reconciliation or restart that can read it may diff --git a/src/recovery/discord-awareness-outbox.ts b/src/recovery/discord-awareness-outbox.ts index c75d93c4c..f7e6592d7 100644 --- a/src/recovery/discord-awareness-outbox.ts +++ b/src/recovery/discord-awareness-outbox.ts @@ -1,16 +1,8 @@ -import { - closeSync, - existsSync, - fsyncSync, - mkdirSync, - openSync, - readFileSync, - renameSync, - truncateSync, - writeSync, -} from 'node:fs'; -import { basename, dirname, join } from 'node:path'; +import { existsSync, readFileSync, renameSync } from 'node:fs'; +import { basename, join } from 'node:path'; import { createHash, randomUUID } from 'node:crypto'; +import type { JsStore } from '@animalabs/chronicle'; +import { RecordJournal, type AppendOptions } from '../record-journal.js'; /** * Discord awareness marks: a reaction (💤 by default) that an operator may @@ -34,16 +26,22 @@ import { createHash, randomUUID } from 'node:crypto'; * server id is the configured route to a bot, not the bot's identity: if the * route is re-pointed at another bot, retained operations act as that bot, * and nothing here can detect it. + * + * The journal lives in the Chronicle store as typed records (RecordJournal): + * it survives branch switches, rollbacks, `deleteBranch` and a killed + * process, and lives and dies with its store. Each append is one record + * holding a group of journal records that apply together. A record that + * asserts a branch change (an activation after a switch or redaction) is + * appended only after that state is synced, so it can never outlive it. */ export const DEFAULT_DISCORD_AWARENESS_EMOJI = '💤'; -/** The journal file under a store, next to the other recovery state. */ /** * Bound text that came from elsewhere (a Discord error body, a filesystem * error) before it is journaled, logged or returned in a receipt: the journal - * is re-read on every operation, and a server's error text has no length - * limit of its own. Head and tail are kept, with the omitted length stated; + * is kept for the store's life, and a server's error text has no length limit + * of its own. Head and tail are kept, with the omitted length stated; * a cut never splits a surrogate pair. */ export function boundDiscordAwarenessText(text: string, max = 500): string { @@ -56,8 +54,24 @@ export function boundDiscordAwarenessText(text: string, max = 500): string { return `${head} … [${text.length - head.length - tail.length} chars omitted] … ${tail}`; } +/** + * Where a pre-journal awareness ledger (the JSON file earlier releases kept + * under a store) is found, to import once. + */ +/** + * A Discord answer that the reaction can never succeed: the message or + * channel is gone or inaccessible. Retrying cannot help. + */ +export function isPermanentDiscordReactionFailure(message: string): boolean { + return /unknown message|unknown channel|missing access|missing permissions|missing permission|cannot access|channel .* not found|message .* not found/i + .test(message); +} + +/** AF's own refusal to write a request on a closed connection. */ +const PRE_WRITE_REFUSAL = /^Cannot send request:/; + export function defaultDiscordAwarenessOutboxPath(storePath: string): string { - return join(storePath, 'recovery', 'discord-awareness-journal.jsonl'); + return join(storePath, 'recovery', LEGACY_LEDGER_NAME); } /** The pre-journal (version 1 and 2) ledger file name, imported once. */ @@ -129,7 +143,8 @@ export type DiscordAwarenessBatchStatus = 'prepared' | 'active' | 'held' | 'disc export interface DiscordAwarenessBatch extends DiscordAwarenessBatchRecord { status: DiscordAwarenessBatchStatus; - /** A held suppression whose body was completed at a later startup. */ + /** A suppression whose body is recorded complete (by its activation, or by + * a later startup's resume). */ suppressionComplete?: boolean; held?: { reason: string; at: number; releaseActions: DiscordAwarenessReleaseAction[] }; cancelled?: { at: number; by?: string }; @@ -172,7 +187,7 @@ export interface DiscordAwarenessOp { cause: DiscordAwarenessOpCause; requestedAt: number; attempts: DiscordAwarenessAttempt[]; - cancelled?: { at: number; reason: 'batch-cancelled' | 'superseded' }; + cancelled?: { at: number; reason: 'batch-cancelled' | 'request-cancelled' | 'superseded' | 'imported' }; } /** One dispatch: the head operation for a reaction key, plus any consecutive @@ -185,7 +200,9 @@ export interface DiscordAwarenessDispatch { /** Receipt for cancel(). Nothing is ever removed from Discord by a cancel. */ export interface DiscordAwarenessCancelReceipt { - batchId: string; + /** The batch id or retract request id that was cancelled. */ + target: string; + kind: 'batch' | 'retract'; /** Requests that will now never be sent (for a batch not yet activated, * every mark it would have requested). */ cancelled: number; @@ -197,12 +214,19 @@ export interface DiscordAwarenessCancelReceipt { unknown: number; /** Operations Discord already confirmed (cancel does not undo them). */ confirmed: number; + /** Requests of this batch or retract whose outcome is unresolved (on the + * wire or unknown), counted per request sent and including earlier + * attempts of operations whose last attempt was answered: any may land. */ + unresolvedAttempts: number; + /** Imported (pre-journal) attempts on this batch whose outcome the old + * ledger does not establish: any may have landed. */ + legacyOutcomesUnrecorded: number; } /** Receipt for retract(). */ export interface DiscordAwarenessRetractReceipt { requestId: string; - /** Removals queued (one per reaction key). */ + /** Removals queued: one per selected reaction key, whatever history says. */ removalsQueued: number; /** Add requests for those keys that had not been sent, now superseded. */ addsSuperseded: number; @@ -212,6 +236,32 @@ export interface DiscordAwarenessRetractReceipt { /** Unresolved earlier add requests on those keys (one per request sent, * however many batches' operations it carried). */ unresolvedAddAttempts: number; + /** Keys whose imported (pre-journal) history leaves attempt outcomes + * unrecorded: any of those may have had an effect. */ + keysWithLegacyUncertainty: number; +} + +/** + * What a pre-journal ledger recorded about one batch ref, kept as facts. It + * is evidence for operators and receipts, never a claim about what is on + * Discord now, and it never causes a dispatch. + */ +export interface DiscordAwarenessLegacyEvidence { + batchId: string; + key: DiscordAwarenessRef & { emoji: string }; + /** Attempts the old ledger counted (it kept only the last one's details). */ + attempts: number; + lastAction: DiscordAwarenessAction; + /** The old ledger's status after the last attempt. */ + lastStatus: 'pending' | 'applied' | 'permanent-failure'; + lastError?: string; + /** What the last error was: AF's refusal before writing a request, a + * Discord answer (message/channel gone or inaccessible), or anything else. */ + lastErrorKind?: 'pre-write-refusal' | 'discord-answer' | 'other'; + /** Recorded attempts whose outcome the old ledger does not establish: + * every attempt before the last, and the last unless it was applied, + * refused before writing, or answered by Discord. */ + outcomesUnrecorded: number; } export interface DiscordAwarenessReleaseReceipt { @@ -222,6 +272,7 @@ export interface DiscordAwarenessReleaseReceipt { /** Per-batch view for operators: what was asked and what is known. */ export interface DiscordAwarenessBatchView { + kind: 'batch'; id: string; status: DiscordAwarenessBatchStatus; scope: DiscordAwarenessBatchRecord['scope']; @@ -240,10 +291,37 @@ export interface DiscordAwarenessBatchView { adds: Record; /** Removals requested (by any retract) on this batch's refs, by status. */ removals: Record; - /** Attempts on this batch's refs whose outcome is unknown or on the wire. */ + /** Requests on this batch's refs whose outcome is unknown or on the wire. */ unresolvedAttempts: number; + /** What a pre-journal ledger recorded about this batch's refs, if imported. */ + legacy?: { + entries: number; + /** Refs whose last recorded action was an add the old ledger saw applied. */ + lastAddConfirmed: number; + /** Refs whose last recorded action was a removal it saw applied. */ + lastRemoveConfirmed: number; + /** Recorded attempts whose outcome the old ledger does not establish. */ + outcomesUnrecorded: number; + }; } +/** One retract request: the removals it queued and what became of them. */ +export interface DiscordAwarenessRetractView { + kind: 'retract'; + /** The retract's request id (what cancel takes). */ + id: string; + /** The batch it retracted, or `all`. */ + target: string; + at: number; + by?: string; + cancelled?: { at: number; by?: string }; + removals: Record; + /** Removal requests whose outcome is unknown or on the wire. */ + unresolvedAttempts: number; +} + +export type DiscordAwarenessView = DiscordAwarenessBatchView | DiscordAwarenessRetractView; + /** Extract Discord addressing metadata without reading message content. */ export interface DiscordMessageMetadataCarrier { metadata?: Record; @@ -334,7 +412,7 @@ type JournalRecord = action: DiscordAwarenessAction; cause: DiscordAwarenessOpCause; } - | { t: 'op-cancelled'; at: number; opId: string; reason: 'batch-cancelled' | 'superseded' } + | { t: 'op-cancelled'; at: number; opId: string; reason: 'batch-cancelled' | 'request-cancelled' | 'superseded' | 'imported' } | { t: 'dispatching'; at: number; opId: string; attempt: number; dispatchId?: string } | { t: 'outcome'; @@ -346,49 +424,72 @@ type JournalRecord = error?: string; } | { t: 'retract'; at: number; requestId: string; target: string; by?: string } + | { t: 'request-cancelled'; at: number; requestId: string; by?: string } | { t: 'suppression-complete'; at: number; batchId: string } - | { t: 'import-complete'; at: number; source: 'v2'; sha256: string } - | { t: 'commit'; at: number; txn: string }; + | { t: 'legacy-evidence'; at: number; evidence: DiscordAwarenessLegacyEvidence } + | { t: 'import-complete'; at: number; source: 'v2'; sha256: string }; -/** - * Every multi-record append is one transaction: its records carry a `txn` - * id and only take effect when the closing `commit` record is present. A - * torn append (disk full, crash mid-write) therefore applies none of its - * records, never a prefix, so an activation can't half-queue its marks and a - * retract can't supersede adds without queuing its removals. - */ -type StoredRecord = JournalRecord & { txn?: string }; +/** One journal entry (one Chronicle record): records that apply together. */ +interface JournalEntryGroup { + records: JournalRecord[]; +} + +/** The reduced state as a checkpoint stores it. */ +interface JournalSnapshot { + batches: DiscordAwarenessBatch[]; + ops: DiscordAwarenessOp[]; + legacy: DiscordAwarenessLegacyEvidence[]; + retracts?: RetractRequest[]; + importedSources: string[]; +} + +export const DISCORD_AWARENESS_RECORD_TYPE = 'af:discord-awareness'; +/** Replay at open is bounded by a checkpoint every this many entries. */ +const CHECKPOINT_EVERY = 256; interface JournalState { batches: Map; /** Operations in request order. */ ops: Map; + /** Imported pre-journal evidence, by reaction key. */ + legacy: Map; + /** Retract requests, by request id. */ + retracts: Map; importedSources: Set; } +interface RetractRequest { + requestId: string; + target: string; + at: number; + by?: string; + cancelled?: { at: number; by?: string }; +} + /** - * The durable awareness-mark ledger: an append-only journal (one JSON record - * per line, each append fsynced) and the operations derived from it. + * The durable awareness-mark ledger: an append-only journal of typed records + * in the store, and the operations reduced from it. One instance per store is + * the journal's only writer (RecordJournal's single-writer rule): the running + * framework, or the offline CLI while the host is stopped. */ export class DiscordAwarenessOutbox { - readonly path: string; - private readonly legacyPath: string; + private readonly journal: RecordJournal; + private readonly legacyPath?: string; + private readonly checkpointEvery: number; + /** Reduced state; rebuilt from the journal when null. */ + private state: JournalState | null = null; /** Operations dispatched by this process and still awaiting an outcome. */ private readonly inFlight = new Set(); /** - * @param path the journal file. A path ending in `.json` names a pre-journal - * ledger (as an older `discordAwarenessOutboxPath` would): the journal then - * lives beside it as `.journal.jsonl` and the file itself is imported. + * @param store the Chronicle store the journal lives in. + * @param opts.legacyPath a pre-journal JSON ledger to import once, if present. + * @param opts.checkpointEvery entries between checkpoints (default 256). */ - constructor(path: string) { - if (path.endsWith('.json')) { - this.legacyPath = path; - this.path = `${path.slice(0, -'.json'.length)}.journal.jsonl`; - } else { - this.path = path; - this.legacyPath = join(dirname(path), LEGACY_LEDGER_NAME); - } + constructor(store: JsStore, opts: { legacyPath?: string; checkpointEvery?: number } = {}) { + this.journal = new RecordJournal(store, { type: DISCORD_AWARENESS_RECORD_TYPE }); + if (opts.legacyPath) this.legacyPath = opts.legacyPath; + this.checkpointEvery = Math.max(1, Math.floor(opts.checkpointEvery ?? CHECKPOINT_EVERY)); } // -- surgery side ----------------------------------------------------------- @@ -428,7 +529,8 @@ export class DiscordAwarenessOutbox { ? { suppressionIntervals: input.suppressionIntervals.map((interval) => ({ ...interval })) } : {}), }; - this.append([{ t: 'batch', at: batch.createdAt, batch }]); + // Durable before the surgery's switch, so startup can crash-complete it. + this.append([{ t: 'batch', at: batch.createdAt, batch }], { durable: true }); return { ...structuredClone(batch), status: 'prepared' }; } @@ -454,10 +556,45 @@ export class DiscordAwarenessOutbox { kind: 'batch', batchId, }, at)); - this.append(records); + // The activation asserts the body change just committed (the switch, or + // the last redaction): sync that first, so a crash can never leave marks + // active for a change the store lost. + this.append(records, { afterCommittedState: true, durable: true }); return batch.refs.length; } + /** + * Activate a batch after its body change landed, and if that cannot be + * recorded, retire it instead, so that what the caller reports is what + * the journal will do: + * - `queued`: the batch is active (including when the activation landed + * but its durability barrier failed: startup would crash-complete it); + * - `not-scheduled`: it was retired; none of its marks will be sent; + * - `unresolved`: neither could be recorded; a later startup that can read + * the batch may activate it. + * Never throws. + */ + settleActivation(batchId: string): + | { status: 'queued'; queued: number } + | { status: 'not-scheduled'; error: string } + | { status: 'unresolved'; error: string } { + let detail: string; + try { + this.activate(batchId); + return { status: 'queued', queued: this.load().batches.get(batchId)?.refs.length ?? 0 }; + } catch (error) { + detail = boundDiscordAwarenessText(error instanceof Error ? error.message : String(error)); + } + try { + const current = this.load().batches.get(batchId); + if (current?.status === 'active') return { status: 'queued', queued: current.refs.length }; + if (this.discard(batchId)) return { status: 'not-scheduled', error: detail }; + } catch (error) { + detail += `; retiring it also failed: ${boundDiscordAwarenessText(error instanceof Error ? error.message : String(error))}`; + } + return { status: 'unresolved', error: detail }; + } + /** * Retire a batch whose surgery did not complete. Only a prepared batch can * be discarded; returns false when nothing was retired. @@ -465,20 +602,26 @@ export class DiscordAwarenessOutbox { discard(batchId: string): boolean { const batch = this.load().batches.get(batchId); if (!batch || batch.status !== 'prepared') return false; - this.append([{ t: 'discarded', at: Date.now(), batchId }]); + // Retiring a batch retires its obligations (marks and any body resume): + // the branch state its surgery left (a restored source, or a completed + // body) is synced first, and the record is durable, so a crash can never + // keep the retirement while losing the state it relies on. + this.append([{ t: 'discarded', at: Date.now(), batchId }], { afterCommittedState: true, durable: true }); return true; } /** * Suppressions whose body may still need resuming on the active branch: - * explicit batches with intervals whose target is exactly that branch, - * prepared (crash completion: resume, then activate) or held at an earlier - * startup (resume the body only; marks stay held). Branch ancestry is never - * consulted. + * explicit batches with intervals whose target is exactly that branch and + * whose body was never recorded complete. Body recovery is independent of + * the marks: a prepared batch is crash-completed (resume, then activate); + * any other, held, released or cancelled, has only its body resumed. + * A batch its own surgery retired (discarded) is not resumed. Branch + * ancestry is never consulted. */ preparedSuppressionsForBranch(branchName: string): DiscordAwarenessBatch[] { return [...this.load().batches.values()] - .filter((batch) => (batch.status === 'prepared' || batch.status === 'held') + .filter((batch) => batch.status !== 'discarded' && !batch.suppressionComplete && batch.activationPolicy === 'explicit' && !!batch.suppressionIntervals?.length @@ -486,9 +629,11 @@ export class DiscordAwarenessOutbox { .map((batch) => structuredClone(batch)); } - /** Record that a held suppression's body was completed (its marks stay held). */ + /** Record that a suppression's body was completed by a resume that did not + * activate its marks (they were held, released or cancelled). */ recordSuppressionComplete(batchId: string): void { - this.append([{ t: 'suppression-complete', at: Date.now(), batchId }]); + // Certifies the redactions just made: only after they are synced. + this.append([{ t: 'suppression-complete', at: Date.now(), batchId }], { afterCommittedState: true, durable: true }); } /** @@ -552,7 +697,7 @@ export class DiscordAwarenessOutbox { held.push(batch.id); } } - if (records.length > 0) this.append(records); + if (records.length > 0) this.append(records, { durable: true }); return { activated, held, unknownAttempts }; } @@ -565,12 +710,15 @@ export class DiscordAwarenessOutbox { * dropped. Nothing is ever removed from Discord by a cancel: the receipt * reports what may already have landed, and retract removes it. */ - cancel(batchId: string, by?: string): DiscordAwarenessCancelReceipt { + cancel(target: string, by?: string): DiscordAwarenessCancelReceipt { const state = this.load(); + if (state.retracts.has(target)) return this.cancelRetract(state, target, by); + const batchId = target; const batch = state.batches.get(batchId); - if (!batch) throw new Error(`Discord awareness batch not found: ${batchId}`); + if (!batch) throw new Error(`Discord awareness batch or retract request not found: ${target}`); const receipt: DiscordAwarenessCancelReceipt = { - batchId, + target: batchId, + kind: 'batch', // A prepared batch (its surgery still running, or its bookkeeping // unresolved) has no requests yet: every one of its marks is stopped. cancelled: batch.status === 'prepared' && !batch.cancelled ? batch.refs.length : 0, @@ -578,6 +726,13 @@ export class DiscordAwarenessOutbox { inFlight: 0, unknown: 0, confirmed: 0, + unresolvedAttempts: unresolvedDispatches( + [...state.ops.values()].filter((op) => op.cause.batchId === batchId && op.action === 'add'), + (op, attempt) => this.attemptUnresolved(op, attempt), + ), + legacyOutcomesUnrecorded: [...state.legacy.values()].flat() + .filter((evidence) => evidence.batchId === batchId) + .reduce((sum, evidence) => sum + evidence.outcomesUnrecorded, 0), }; if (batch.cancelled) return receipt; const at = Date.now(); @@ -595,27 +750,67 @@ export class DiscordAwarenessOutbox { else continue; // failed or already cancelled records.push({ t: 'op-cancelled', at, opId: op.opId, reason: 'batch-cancelled' }); } - this.append(records); + this.append(records, { durable: true }); + return receipt; + } + + /** Stop a retract's removals that are still due; nothing sent is undone. */ + private cancelRetract(state: JournalState, requestId: string, by?: string): DiscordAwarenessCancelReceipt { + const retract = state.retracts.get(requestId)!; + const ops = [...state.ops.values()].filter((op) => op.cause.requestId === requestId); + const receipt: DiscordAwarenessCancelReceipt = { + target: requestId, + kind: 'retract', + cancelled: 0, + heldDropped: 0, + inFlight: 0, + unknown: 0, + confirmed: 0, + unresolvedAttempts: unresolvedDispatches(ops, (op, attempt) => this.attemptUnresolved(op, attempt)), + legacyOutcomesUnrecorded: 0, + }; + if (retract.cancelled) return receipt; + const at = Date.now(); + const records: JournalRecord[] = [{ t: 'request-cancelled', at, requestId, ...(by ? { by } : {}) }]; + for (const op of ops) { + const status = this.opStatus(op); + if (status === 'confirmed') { + receipt.confirmed++; + continue; + } + if (status === 'requested') receipt.cancelled++; + else if (status === 'dispatching') receipt.inFlight++; + else if (status === 'unknown') receipt.unknown++; + else continue; + records.push({ t: 'op-cancelled', at, opId: op.opId, reason: 'request-cancelled' }); + } + this.append(records, { durable: true }); return receipt; } /** * Remove this bot's mark (through the configured route) from a batch's refs, - * or from every ref any batch ever sent an add for. Acts across batches: a - * reaction is one per (route, message, emoji), whichever batches asked for - * it. Keys where no add attempt ever left the host are skipped. Add - * requests not yet sent for those keys are superseded. The removal is - * always sent, whatever earlier outcomes say. + * or from every key any batch requested or an imported ledger recorded an + * add for (`all`). A reaction is one per (route, message, emoji), whichever + * batches asked for it, so this acts across batches. A removal is queued for + * every selected key, whatever history says: history can't establish what + * is on Discord now, and removing an absent reaction does nothing. Add + * requests not yet sent for those keys are superseded. The receipt reports + * history only: requests whose outcome is unresolved, and imported history + * that leaves outcomes unrecorded, any of which may land or have landed. */ retract(target: string | 'all', by?: string): DiscordAwarenessRetractReceipt { const state = this.load(); let candidateKeys: Array; if (target === 'all') { - candidateKeys = uniqueKeys([...state.ops.values()].filter((op) => op.action === 'add').map((op) => op.key)); + candidateKeys = uniqueKeys([ + ...[...state.ops.values()].filter((op) => op.action === 'add').map((op) => op.key), + ...[...state.legacy.values()].flat().map((evidence) => evidence.key), + ]); } else { const batch = state.batches.get(target); if (!batch) throw new Error(`Discord awareness batch not found: ${target}`); - candidateKeys = batch.refs.map((ref) => ({ ...ref, emoji: batch.emoji })); + candidateKeys = uniqueKeys(batch.refs.map((ref) => ({ ...ref, emoji: batch.emoji }))); } const requestId = randomUUID(); const at = Date.now(); @@ -625,34 +820,27 @@ export class DiscordAwarenessOutbox { addsSuperseded: 0, keysWithUnresolvedAdds: 0, unresolvedAddAttempts: 0, + keysWithLegacyUncertainty: 0, }; - const toRemove: Array = []; const records: JournalRecord[] = [{ t: 'retract', at, requestId, target, ...(by ? { by } : {}) }]; + const toRemove: Array = []; for (const key of candidateKeys) { const adds = this.opsForKey(state, key).filter((op) => op.action === 'add'); - const left = adds.some((op) => op.attempts.some((attempt) => attempt.outcome !== 'not-sent')); - if (!left) { - // Nothing was ever sent for this key: stop its unsent adds, and - // there is no reaction of ours to remove. - for (const op of adds) { - if (this.opStatus(op) !== 'requested') continue; - records.push({ t: 'op-cancelled', at, opId: op.opId, reason: 'superseded' }); - receipt.addsSuperseded++; - } - continue; - } const unresolved = unresolvedDispatches(adds, (op, attempt) => this.attemptUnresolved(op, attempt)); if (unresolved > 0) { receipt.keysWithUnresolvedAdds++; receipt.unresolvedAddAttempts += unresolved; } + if ((state.legacy.get(markKey(key)) ?? []).some((evidence) => evidence.outcomesUnrecorded > 0)) { + receipt.keysWithLegacyUncertainty++; + } toRemove.push({ ...key, action: 'remove' }); } const requested = this.requestRecords(state, toRemove, undefined, { kind: 'retract', requestId }, at); - receipt.addsSuperseded += requested.filter((record) => record.t === 'op-cancelled').length; + receipt.addsSuperseded = requested.filter((record) => record.t === 'op-cancelled').length; receipt.removalsQueued = requested.filter((record) => record.t === 'requested').length; records.push(...requested); - this.append(records); + this.append(records, { durable: true }); return receipt; } @@ -671,7 +859,7 @@ export class DiscordAwarenessOutbox { kind: 'release', batchId, }, at)); - this.append(records); + this.append(records, { durable: true }); return { batchId, addsQueued: actions.filter((action) => action.action === 'add').length, @@ -700,49 +888,81 @@ export class DiscordAwarenessOutbox { } const dispatches: Array = []; for (const ops of byKey.values()) { - // One request per reaction key on the wire at a time, including a - // cancelled one still awaiting its outcome: opposite actions must not - // overtake each other. - if (ops.some((op) => op.attempts.at(-1)?.outcome === undefined && op.attempts.length > 0 - && this.inFlight.has(op.opId))) continue; - const live = ops.filter((op) => !op.cancelled); - let head: DiscordAwarenessOp | undefined; - for (let index = 0; index < live.length; index++) { - const op = live[index]; - const status = this.opStatus(op); - if (status === 'requested') { head = op; break; } - if (status === 'unknown' && index === live.length - 1) { head = op; break; } - } - if (!head) continue; - const opIds = [head.opId]; - const start = live.indexOf(head); - for (let index = start + 1; index < live.length; index++) { - const next = live[index]; - if (next.action !== head.action || this.opStatus(next) !== 'requested') break; - opIds.push(next.opId); - } - dispatches.push({ key: head.key, action: head.action, opIds, order: head.requestedAt }); + const head = this.headDispatch(ops); + if (head) dispatches.push(head); } return dispatches .sort((a, b) => a.order - b.order) .map(({ order: _order, ...dispatch }) => dispatch); } - /** Write-ahead: record that a dispatch is about to leave the host. */ + /** + * Choose a server's next due dispatch and write it ahead, in one step: + * eligibility is computed from the journal as it is now, so a cancel or + * retract made while an earlier request was on the wire is never + * overridden by an older list. `skip` excludes dispatches already tried + * in this pass. Returns null when nothing is due. + */ + claimDispatch( + serverId: string, + skip: (dispatch: DiscordAwarenessDispatch) => boolean = () => false, + ): { dispatch: DiscordAwarenessDispatch; attempts: Array<{ opId: string; attempt: number }> } | null { + for (const dispatch of this.pendingDispatches(serverId)) { + if (skip(dispatch)) continue; + const attempts = this.recordDispatching(dispatch); + if (attempts.length > 0) return { dispatch, attempts }; + } + return null; + } + + /** + * Write-ahead: record that a dispatch is about to leave the host. A + * dispatch computed earlier is narrowed to the operations still due for its + * key now; when none remain, nothing is written and no attempts are + * returned (do not send). + */ recordDispatching(dispatch: DiscordAwarenessDispatch): Array<{ opId: string; attempt: number }> { const state = this.load(); + const current = this.headDispatch(this.opsForKey(state, dispatch.key)); + if (!current || current.action !== dispatch.action) return []; + const opIds = dispatch.opIds.filter((opId) => current.opIds.includes(opId)); + if (opIds.length === 0) return []; const at = Date.now(); const dispatchId = randomUUID(); - const attempts = dispatch.opIds.map((opId) => { - const op = state.ops.get(opId); - if (!op) throw new Error(`Discord awareness operation not found: ${opId}`); - return { opId, attempt: op.attempts.length + 1 }; - }); - this.append(attempts.map(({ opId, attempt }) => ({ t: 'dispatching' as const, at, opId, attempt, dispatchId }))); + const attempts = opIds.map((opId) => ({ opId, attempt: state.ops.get(opId)!.attempts.length + 1 })); + this.append( + attempts.map(({ opId, attempt }) => ({ t: 'dispatching' as const, at, opId, attempt, dispatchId })), + { durable: true }, + ); for (const { opId } of attempts) this.inFlight.add(opId); return attempts; } + /** The due dispatch for one key's operations (in request order), if any. */ + private headDispatch(ops: DiscordAwarenessOp[]): (DiscordAwarenessDispatch & { order: number }) | null { + // One request per reaction key on the wire at a time, including a + // cancelled one still awaiting its outcome: opposite actions must not + // overtake each other. + if (ops.some((op) => op.attempts.length > 0 && op.attempts.at(-1)!.outcome === undefined + && this.inFlight.has(op.opId))) return null; + const live = ops.filter((op) => !op.cancelled); + let head: DiscordAwarenessOp | undefined; + for (let index = 0; index < live.length; index++) { + const op = live[index]; + const status = this.opStatus(op); + if (status === 'requested') { head = op; break; } + if (status === 'unknown' && index === live.length - 1) { head = op; break; } + } + if (!head) return null; + const opIds = [head.opId]; + for (let index = live.indexOf(head) + 1; index < live.length; index++) { + const next = live[index]; + if (next.action !== head.action || this.opStatus(next) !== 'requested') break; + opIds.push(next.opId); + } + return { key: { ...head.key }, action: head.action, opIds, order: head.requestedAt }; + } + /** Record the outcome of a dispatch's attempts. */ recordOutcome( attempts: Array<{ opId: string; attempt: number }>, @@ -782,9 +1002,9 @@ export class DiscordAwarenessOutbox { return this.opStatus(op); } - view(): DiscordAwarenessBatchView[] { + view(): DiscordAwarenessView[] { const state = this.load(); - const views: DiscordAwarenessBatchView[] = []; + const views: DiscordAwarenessView[] = []; for (const batch of state.batches.values()) { if (batch.status === 'discarded') continue; const keys = new Set(batch.refs.map((ref) => markKey({ ...ref, emoji: batch.emoji }))); @@ -798,7 +1018,9 @@ export class DiscordAwarenessOutbox { if (op.action === 'remove') removals[this.opStatus(op)]++; } const unresolvedAttempts = unresolvedDispatches(onKeys, (op, attempt) => this.attemptUnresolved(op, attempt)); + const evidence = [...state.legacy.values()].flat().filter((entry) => entry.batchId === batch.id); views.push({ + kind: 'batch', id: batch.id, status: batch.status, scope: batch.scope, @@ -818,6 +1040,31 @@ export class DiscordAwarenessOutbox { adds, removals, unresolvedAttempts, + ...(evidence.length > 0 + ? { + legacy: { + entries: evidence.length, + lastAddConfirmed: evidence.filter((entry) => entry.lastAction === 'add' && entry.lastStatus === 'applied').length, + lastRemoveConfirmed: evidence.filter((entry) => entry.lastAction === 'remove' && entry.lastStatus === 'applied').length, + outcomesUnrecorded: evidence.reduce((sum, entry) => sum + entry.outcomesUnrecorded, 0), + }, + } + : {}), + }); + } + for (const retract of state.retracts.values()) { + const ops = [...state.ops.values()].filter((op) => op.cause.requestId === retract.requestId); + const removals = emptyCounts(); + for (const op of ops) removals[this.opStatus(op)]++; + views.push({ + kind: 'retract', + id: retract.requestId, + target: retract.target, + at: retract.at, + ...(retract.by ? { by: retract.by } : {}), + ...(retract.cancelled ? { cancelled: retract.cancelled } : {}), + removals, + unresolvedAttempts: unresolvedDispatches(ops, (op, attempt) => this.attemptUnresolved(op, attempt)), }); } return views; @@ -853,10 +1100,11 @@ export class DiscordAwarenessOutbox { } /** - * Records for new requests. A request supersedes requests for the same - * key with the opposite action that were never sent (the newest explicit - * intent wins without churning the reaction); operations already sent are - * never touched. + * Records for new requests. A request supersedes the future of every + * request for the same key with the opposite action that is still due + * (never sent, refused before writing, or failed retryably): the newest + * explicit intent wins. Their attempts stay as history, and a request on + * the wire still settles before anything else for that key is sent. */ private requestRecords( state: JournalState, @@ -876,7 +1124,7 @@ export class DiscordAwarenessOutbox { }; for (const op of this.opsForKey(state, key)) { if (op.action === request.action || superseded.has(op.opId)) continue; - if (this.opStatus(op) !== 'requested' || op.attempts.some((attempt) => attempt.outcome !== 'not-sent')) continue; + if (this.opStatus(op) !== 'requested') continue; records.push({ t: 'op-cancelled', at, opId: op.opId, reason: 'superseded' }); superseded.add(op.opId); } @@ -885,146 +1133,130 @@ export class DiscordAwarenessOutbox { return records; } - private load(): JournalState { - const { records } = this.readJournal(); - const state = fold(records); - const legacy = this.pendingLegacyImport(state); - if (legacy?.records.length) return fold([...records, ...legacy.records]); - return state; - } - /** - * The journal's records, and how to repair an unterminated final line - * before the next append: a torn fragment (never applied) is truncated - * away; a complete record that only lost its newline is terminated. - * Without the repair, the next append would share that line and turn a - * harmless torn tail into corruption. + * The reduced state, rebuilt from the journal when there is none yet or an + * earlier write left the journal unreconciled (RecordJournal then requires + * a reload before writing again). The first load also imports a + * pre-journal ledger, once. */ - private readJournal(): { records: StoredRecord[]; repair?: { truncateTo: number } | { terminate: true } } { - let raw: Buffer; - try { - raw = readFileSync(this.path); - } catch (error) { - if ((error as NodeJS.ErrnoException).code === 'ENOENT') return { records: [] }; - throw new Error( - `Could not read Discord awareness journal ${this.path}: ${error instanceof Error ? error.message : String(error)}`, - ); - } - const terminated = raw.length === 0 || raw[raw.length - 1] === 0x0a; - const lastNewline = raw.lastIndexOf(0x0a); - const body = terminated ? raw : raw.subarray(0, lastNewline + 1); - const lines = body.toString('utf8').split('\n'); - const records: StoredRecord[] = []; - for (let index = 0; index < lines.length; index++) { - const line = lines[index]; - if (!line.trim()) continue; - try { - records.push(JSON.parse(line) as StoredRecord); - } catch { - throw new Error(`Corrupt Discord awareness journal ${this.path} at line ${index + 1}`); + private load(): JournalState { + if (this.state && !this.journal.needsReconcile) return this.state; + const { snapshot, entries } = this.journal.load(); + const state = snapshot ? restoreState(snapshot) : emptyState(); + for (const { id, entry } of entries) { + if (!entry || !Array.isArray(entry.records)) { + throw new Error(`Corrupt Discord awareness journal entry ${id}`); } + applyRecords(state, entry.records); } - if (terminated) return { records }; - const tail = raw.subarray(lastNewline + 1).toString('utf8'); - try { - records.push(JSON.parse(tail) as StoredRecord); - return { records, repair: { terminate: true } }; - } catch { - // A torn append: none of it ever applied. - return { records, repair: { truncateTo: lastNewline + 1 } }; - } + this.state = state; + this.importLegacyLedger(state); + return state; } - /** - * A pre-journal ledger beside the journal: null when there is none; - * otherwise its import records, empty when its hash is already imported - * (a crash between the import and the rename), so the file is only renamed. - */ - private pendingLegacyImport(state: JournalState): { records: JournalRecord[]; sha256: string } | null { + /** Import a pre-journal ledger beside the store once (by hash), then rename it. */ + private importLegacyLedger(state: JournalState): void { + if (!this.legacyPath) return; let raw: string; try { raw = readFileSync(this.legacyPath, 'utf8'); } catch (error) { - if ((error as NodeJS.ErrnoException).code === 'ENOENT') return null; + if ((error as NodeJS.ErrnoException).code === 'ENOENT') return; throw new Error( `Could not read Discord awareness ledger ${this.legacyPath}: ${error instanceof Error ? error.message : String(error)}`, ); } const sha256 = createHash('sha256').update(raw).digest('hex'); - if (state.importedSources.has(sha256)) return { records: [], sha256 }; - let parsed: unknown; - try { - parsed = JSON.parse(raw); - } catch { - throw new Error(`Invalid Discord awareness ledger ${this.legacyPath}`); - } - return { records: importLegacy(parsed, this.legacyPath, sha256), sha256 }; - } - - private append(records: JournalRecord[]): void { - if (records.length === 0) return; - const journal = this.readJournal(); - const state = fold(journal.records); - const legacy = this.pendingLegacyImport(state); - const all: StoredRecord[] = legacy ? [...legacy.records, ...records] : records; - let lines: StoredRecord[] = all; - if (all.length > 1) { - const txn = randomUUID(); - lines = [ - ...all.map((record) => ({ ...record, txn })), - { t: 'commit', at: Date.now(), txn }, - ]; - } - mkdirSync(dirname(this.path), { recursive: true }); - if (journal.repair && 'truncateTo' in journal.repair) truncateSync(this.path, journal.repair.truncateTo); - const prefix = journal.repair && 'terminate' in journal.repair ? '\n' : ''; - const fd = openSync(this.path, 'a', 0o600); - try { - writeSync(fd, prefix + lines.map((record) => JSON.stringify(record)).join('\n') + '\n'); - fsyncSync(fd); - } finally { - closeSync(fd); + if (!state.importedSources.has(sha256)) { + let parsed: unknown; + try { + parsed = JSON.parse(raw); + } catch { + throw new Error(`Invalid Discord awareness ledger ${this.legacyPath}`); + } + this.append(importLegacy(parsed, this.legacyPath, sha256), { durable: true }); } - if (legacy) this.retireLegacyFile(); - } - - private retireLegacyFile(): void { + // Imported (now or by an earlier process that stopped before renaming): + // the file is inert; move it out of the way. if (!existsSync(this.legacyPath)) return; try { renameSync(this.legacyPath, `${this.legacyPath}.migrated-v2`); } catch (error) { - // The import is already recorded (by hash), so the file is inert; the - // next write retries the rename. console.error( `[discord-awareness] imported ${basename(this.legacyPath)} but could not rename it: ` + `${error instanceof Error ? error.message : String(error)}`, ); } } + + /** + * Append one group of records as one journal entry and apply it. A failed + * append leaves the state to be rebuilt from the journal: whether it was + * written is unknown, and the next operation reloads before writing. + */ + private append(records: JournalRecord[], opts: AppendOptions = {}): void { + if (records.length === 0) return; + const state = this.load(); + try { + this.journal.append({ records }, opts); + } catch (error) { + this.state = null; + throw error; + } + applyRecords(state, records); + if (this.journal.entriesSinceCheckpoint >= this.checkpointEvery) { + try { + this.journal.checkpoint(snapshotState(state)); + } catch (error) { + // Replay just stays longer; the journal reloads before its next write. + this.state = null; + console.error( + `[discord-awareness] checkpoint failed: ${error instanceof Error ? error.message : String(error)}`, + ); + } + } + } } // --------------------------------------------------------------------------- // Fold and legacy import // --------------------------------------------------------------------------- -function fold(stored: StoredRecord[]): JournalState { - const state: JournalState = { batches: new Map(), ops: new Map(), importedSources: new Set() }; - // Records of a transaction apply, in order, at its commit; an uncommitted - // transaction (a torn or interrupted append) never applies. - const open = new Map(); - const records: JournalRecord[] = []; - for (const record of stored) { - if (record.t === 'commit') { - records.push(...(open.get(record.txn) ?? [])); - open.delete(record.txn); - } else if (record.txn) { - const buffered = open.get(record.txn); - if (buffered) buffered.push(record); - else open.set(record.txn, [record]); - } else { - records.push(record); - } - } +function emptyState(): JournalState { + return { batches: new Map(), ops: new Map(), legacy: new Map(), retracts: new Map(), importedSources: new Set() }; +} + +function snapshotState(state: JournalState): JournalSnapshot { + return { + batches: [...state.batches.values()].map((batch) => structuredClone(batch)), + ops: [...state.ops.values()].map((op) => structuredClone(op)), + legacy: [...state.legacy.values()].flat().map((evidence) => structuredClone(evidence)), + retracts: [...state.retracts.values()].map((retract) => structuredClone(retract)), + importedSources: [...state.importedSources], + }; +} + +function restoreState(snapshot: JournalSnapshot): JournalState { + const state: JournalState = { + batches: new Map(snapshot.batches.map((batch) => [batch.id, structuredClone(batch)])), + ops: new Map(snapshot.ops.map((op) => [op.opId, structuredClone(op)])), + legacy: new Map(), + retracts: new Map((snapshot.retracts ?? []).map((retract) => [retract.requestId, structuredClone(retract)])), + importedSources: new Set(snapshot.importedSources), + }; + for (const evidence of snapshot.legacy ?? []) addLegacyEvidence(state, structuredClone(evidence)); + return state; +} + +function addLegacyEvidence(state: JournalState, evidence: DiscordAwarenessLegacyEvidence): void { + const key = markKey(evidence.key); + const list = state.legacy.get(key); + if (list) list.push(evidence); + else state.legacy.set(key, [evidence]); +} + +/** Apply one entry's records, in order, to the reduced state. */ +function applyRecords(state: JournalState, records: JournalRecord[]): void { for (const record of records) { switch (record.t) { case 'batch': @@ -1034,7 +1266,13 @@ function fold(stored: StoredRecord[]): JournalState { break; case 'activated': { const batch = state.batches.get(record.batchId); - if (batch && batch.status !== 'discarded') batch.status = 'active'; + if (batch && batch.status !== 'discarded') { + batch.status = 'active'; + // Only a surgery (or its resume) activates a suppression, after its + // last redaction: activation records the body complete. Release + // does not. + if (batch.suppressionIntervals?.length) batch.suppressionComplete = true; + } break; } case 'discarded': { @@ -1114,6 +1352,26 @@ function fold(stored: StoredRecord[]): JournalState { if (batch) batch.suppressionComplete = true; break; } + case 'legacy-evidence': + addLegacyEvidence(state, structuredClone(record.evidence)); + break; + case 'retract': + if (!state.retracts.has(record.requestId)) { + state.retracts.set(record.requestId, { + requestId: record.requestId, + target: record.target, + at: record.at, + ...(record.by ? { by: record.by } : {}), + }); + } + break; + case 'request-cancelled': { + const retract = state.retracts.get(record.requestId); + if (retract && !retract.cancelled) { + retract.cancelled = { at: record.at, ...(record.by ? { by: record.by } : {}) }; + } + break; + } case 'import-complete': state.importedSources.add(record.sha256); break; @@ -1121,7 +1379,6 @@ function fold(stored: StoredRecord[]): JournalState { break; } } - return state; } interface LegacyEntry extends DiscordAwarenessRef { @@ -1148,11 +1405,13 @@ interface LegacyBatch { } /** - * Translate a version 1 or 2 ledger into journal records with deterministic - * identities, ending with an `import-complete` record carrying the file's - * hash. What the old ledger recorded is kept as it was recorded; nothing is - * inferred about Discord. Pending work, and batches still prepared, are held - * for an operator: they were queued before marks were an explicit choice. + * Translate a version 1 or 2 ledger into journal records, ending with an + * `import-complete` record carrying the file's hash. Batches keep their ids. + * What the old ledger recorded about each ref is kept as legacy evidence: + * facts (attempt count, last action, status and error), never synthesized + * attempts and never a judgment about what is on Discord. Pending work, and + * batches still prepared, are held for an operator: they were queued before + * marks were an explicit choice. Nothing imported is ever dispatched. */ function importLegacy(parsed: unknown, path: string, sha256: string): JournalRecord[] { const document = parsed as { version?: unknown; batches?: unknown }; @@ -1187,6 +1446,8 @@ function importLegacy(parsed: unknown, path: string, sha256: string): JournalRec ...(raw.suppressionIntervals?.length ? { suppressionIntervals: raw.suppressionIntervals } : {}), }, }); + // An active batch was activated by its surgery, after its body (an + // earlier release activated suppressions only after the last redaction). if (legacyActive) records.push({ t: 'activated', at: raw.createdAt, batchId: raw.id }); const releaseActions: DiscordAwarenessReleaseAction[] = []; for (const entry of raw.refs) { @@ -1200,27 +1461,13 @@ function importLegacy(parsed: unknown, path: string, sha256: string): JournalRec ...key, action: desired || !legacyActive ? 'add' : 'remove', ...(entry.attempts ? { priorAttempts: entry.attempts } : {}), - ...(entry.lastError ? { priorError: entry.lastError } : {}), + ...(entry.lastError ? { priorError: boundDiscordAwarenessText(entry.lastError) } : {}), }); } - continue; } - if (!entry.attempts || !entry.lastAction) continue; - const opId = `v2:${raw.id}:${markKey(key)}`; - const attemptAt = entry.lastAttemptAt ?? raw.createdAt; - records.push({ t: 'requested', at: raw.createdAt, opId, key, action: entry.lastAction, cause: { kind: 'import', batchId: raw.id } }); - records.push({ t: 'dispatching', at: attemptAt, opId, attempt: 1 }); - records.push(status === 'applied' - ? { t: 'outcome', at: attemptAt, opId, attempt: 1, outcome: 'confirmed' } - : { - t: 'outcome', - at: attemptAt, - opId, - attempt: 1, - outcome: 'failed', - permanent: true, - ...(entry.lastError ? { error: `recorded by an earlier ledger: ${entry.lastError}` } : {}), - }); + if (entry.attempts && entry.attempts > 0 && entry.lastAction) { + records.push({ t: 'legacy-evidence', at, evidence: legacyEvidence(raw.id, key, entry, status) }); + } } if (releaseActions.length > 0) { records.push({ @@ -1238,6 +1485,37 @@ function importLegacy(parsed: unknown, path: string, sha256: string): JournalRec return records; } +/** The facts a v2 entry recorded about its attempts, and what they leave open. */ +function legacyEvidence( + batchId: string, + key: DiscordAwarenessRef & { emoji: string }, + entry: LegacyEntry, + status: 'pending' | 'applied' | 'permanent-failure', +): DiscordAwarenessLegacyEvidence { + const attempts = entry.attempts!; + const lastError = entry.lastError; + const lastErrorKind = lastError === undefined ? undefined + : PRE_WRITE_REFUSAL.test(lastError) ? 'pre-write-refusal' + : isPermanentDiscordReactionFailure(lastError) ? 'discord-answer' + : 'other'; + // The last attempt's outcome is established when the ledger saw it + // applied, or its error proves it was refused before writing or answered + // by Discord. Earlier attempts' details were never kept. + const lastEstablished = status === 'applied' + || lastErrorKind === 'pre-write-refusal' + || (status === 'permanent-failure' && lastErrorKind === 'discord-answer'); + return { + batchId, + key: { ...key }, + attempts, + lastAction: entry.lastAction!, + lastStatus: status, + ...(lastError !== undefined ? { lastError: boundDiscordAwarenessText(lastError) } : {}), + ...(lastErrorKind ? { lastErrorKind } : {}), + outcomesUnrecorded: (attempts - 1) + (lastEstablished ? 0 : 1), + }; +} + // --------------------------------------------------------------------------- // Keys // --------------------------------------------------------------------------- diff --git a/src/recovery/offline-branch.ts b/src/recovery/offline-branch.ts index 7045d4392..26c467e49 100644 --- a/src/recovery/offline-branch.ts +++ b/src/recovery/offline-branch.ts @@ -1,4 +1,5 @@ import { JsStore } from '@animalabs/chronicle'; +import type { SurgeryMarkerReceipt } from '../operator-log.js'; import { ContextManager } from '@animalabs/context-manager'; import { DEFAULT_DISCORD_AWARENESS_EMOJI, @@ -29,6 +30,7 @@ export interface OfflineRecoveryBranchOptions { namespace?: string; /** Use for old message records that do not carry metadata.serverId. */ discordServerId?: string; + /** A pre-journal awareness ledger to import (default: under the store). */ outboxPath?: string; emoji?: string; /** @@ -49,15 +51,17 @@ export interface OfflineRecoveryBranchResult { discordAddressable: number; /** The publication scope recorded for this recovery. */ marksScope: 'none' | 'addressed' | 'all'; - /** Marks requested (queued once the branch is activated); not delivered. */ + /** Marks queued for delivery (planned, for a dry run); not delivered. */ discordMarkersQueued: number; + /** What became of the marks once the branch was made (absent on a dry + * run): scheduling only, never Discord acceptance. */ + markers?: SurgeryMarkerReceipt; /** The refs that will be marked (empty unless marks were chosen). */ refs: DiscordAwarenessRef[]; /** Removed addressable messages left unmarked. */ unmarked: number; /** Authorized refs (marks.refs) this recovery does not remove. */ notRemoved: number; - outboxPath: string; } /** @@ -176,8 +180,6 @@ export async function createOfflineRecoveryBranch( if (targetBranch === sourceBranch) { throw new Error('Recovery branch name must differ from the active source branch'); } - const outboxPath = options.outboxPath - ?? defaultDiscordAwarenessOutboxPath(options.storePath); const result: OfflineRecoveryBranchResult = { dryRun: options.dryRun === true, @@ -191,11 +193,14 @@ export async function createOfflineRecoveryBranch( refs, unmarked: selection.unmarked, notRemoved: selection.notRemoved, - outboxPath, }; if (options.dryRun) return result; - const outbox = new DiscordAwarenessOutbox(outboxPath); + // The journal lives in this store; the host is stopped, so this process + // is its only writer. + const outbox = new DiscordAwarenessOutbox(store, { + legacyPath: options.outboxPath ?? defaultDiscordAwarenessOutboxPath(options.storePath), + }); const batch = outbox.prepare({ agentName: options.agentName, sourceBranch, @@ -226,18 +231,36 @@ export async function createOfflineRecoveryBranch( if (interval.fromId === interval.toId) contextManager.removeMessage(interval.fromId); else contextManager.removeMessages(interval.fromId, interval.toId); } - if (batch) outbox.activate(batch.id); } catch (error) { // A partially suppressed branch is not safe to boot into. Preserve it // for diagnosis, but leave Chronicle on the untouched source branch and - // keep the explicit outbox batch non-deliverable. + // keep the explicit batch prepared: it is the unfinished body's record + // (startup holds its marks, and resumes the body only on its branch). if (store.currentBranch().name === createdBranch) { await contextManager.switchBranch(sourceBranch); } throw error; } - return result; + // The body change landed. Marker bookkeeping is reported apart from it + // and never undoes it. + const facts = { + scope: (marks === 'none' ? 'none' : marks.scope) as SurgeryMarkerReceipt['scope'], + unmarked: selection.unmarked, + notRemoved: selection.notRemoved, + }; + let markers: SurgeryMarkerReceipt = { ...facts, status: 'none', queued: 0 }; + if (batch) { + const settled = outbox.settleActivation(batch.id); + if (batch.refs.length > 0) { + markers = settled.status === 'queued' + ? { ...facts, status: 'queued', queued: settled.queued, batchId: batch.id } + : settled.status === 'not-scheduled' + ? { ...facts, status: 'not-scheduled', queued: 0, error: settled.error } + : { ...facts, status: 'unresolved', queued: 0, batchId: batch.id, error: settled.error }; + } + } + return { ...result, discordMarkersQueued: markers.queued, markers }; } finally { contextManager?.close(); store.close(); diff --git a/src/recovery/recover-cli.ts b/src/recovery/recover-cli.ts index 99c642c0e..9191cea2a 100644 --- a/src/recovery/recover-cli.ts +++ b/src/recovery/recover-cli.ts @@ -1,5 +1,6 @@ #!/usr/bin/env node +import { JsStore } from '@animalabs/chronicle'; import { createOfflineRecoveryBranch } from './offline-branch.js'; import { DiscordAwarenessOutbox, @@ -29,6 +30,7 @@ function usage(): string { return `Usage: agent-framework-recover --store --agent [options] agent-framework-recover --store --awareness + (cancel takes a batch id or a retract request id) Creates and activates a Chronicle recovery branch while the normal host is down. The recovery is local to the resident: by default no Discord reaction is @@ -49,14 +51,16 @@ Options: Awareness marks on removed Discord messages (default none; addressed = mentions, replies to the bot and DMs; all = every addressable message) - --outbox Override the awareness journal path + --outbox A pre-journal awareness ledger to import + (default: /recovery/discord-awareness-outbox.json) --emoji Marker reaction (default: 💤) --dry-run Inspect counts/addresses without writing --help Show this help Awareness journal (host stopped): --awareness list Each batch: scope, counts by status, holds - --awareness cancel Stop a batch's unsent marks (never removes any) + --awareness cancel Stop further sends of a batch's marks, or of a + retract's removals (never undoes anything) --awareness retract Queue removal of this bot's marks (through each ref's configured MCPL route) on a batch's refs, @@ -135,26 +139,33 @@ function parseArgs(args: string[]): CliOptions { } function awareness(options: CliOptions): void { - const outbox = new DiscordAwarenessOutbox( - options.outboxPath ?? defaultDiscordAwarenessOutboxPath(options.storePath!), - ); - const { verb, target } = options.awareness!; - const by = 'agent-framework-recover'; - let result: unknown; - switch (verb) { - case 'list': result = { journal: outbox.path, batches: outbox.view() }; break; - case 'cancel': result = outbox.cancel(target!, by); break; - case 'retract': result = outbox.retract(target!, by); break; - case 'release': result = outbox.release(target!, by); break; + // The journal lives in the store; with the host stopped, this process is + // its only writer. + const store = JsStore.openOrCreate({ path: options.storePath! }); + try { + const outbox = new DiscordAwarenessOutbox(store, { + legacyPath: options.outboxPath ?? defaultDiscordAwarenessOutboxPath(options.storePath!), + }); + const { verb, target } = options.awareness!; + const by = 'agent-framework-recover'; + let result: unknown; + switch (verb) { + case 'list': result = { store: options.storePath, batches: outbox.view() }; break; + case 'cancel': result = outbox.cancel(target!, by); break; + case 'retract': result = outbox.retract(target!, by); break; + case 'release': result = outbox.release(target!, by); break; + } + console.log(JSON.stringify(result, null, 2)); + } finally { + store.close(); } - console.log(JSON.stringify(result, null, 2)); } async function main(): Promise { const options = parseArgs(process.argv.slice(2)); if (options.awareness) { - if (!options.storePath && !options.outboxPath) { - throw new Error(`--awareness needs --store (or --outbox)\n\n${usage()}`); + if (!options.storePath) { + throw new Error(`--awareness needs --store\n\n${usage()}`); } awareness(options); return; diff --git a/test/discord-awareness-delivery.test.ts b/test/discord-awareness-delivery.test.ts index 137afb045..cdf39b48d 100644 --- a/test/discord-awareness-delivery.test.ts +++ b/test/discord-awareness-delivery.test.ts @@ -14,11 +14,12 @@ import { test } from 'node:test'; import assert from 'node:assert/strict'; import { fileURLToPath } from 'node:url'; -import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; +import { JsStore } from '@animalabs/chronicle'; import { AgentFramework } from '../src/framework.js'; -import { DiscordAwarenessOutbox } from '../src/recovery/discord-awareness-outbox.js'; +import { DISCORD_AWARENESS_RECORD_TYPE, DiscordAwarenessOutbox } from '../src/recovery/discord-awareness-outbox.js'; import { MockMembrane } from './helpers/mock-membrane.js'; const FIXTURE = fileURLToPath(new URL('./fixtures/surgery-marks-mcpl-server.mjs', import.meta.url)); @@ -40,35 +41,39 @@ async function waitFor(description: string, predicate: () => boolean, timeoutMs function setup() { const dir = mkdtempSync(join(tmpdir(), 'awareness-delivery-')); const storePath = join(dir, 'store'); - // Outside the store: the store directory must not exist before Chronicle - // creates it, and these tests seed the journal before the first start. - const journal = join(dir, 'awareness', 'discord-awareness-journal.jsonl'); const paths = { calls: join(dir, 'calls.jsonl'), events: join(dir, 'events.jsonl'), hold: join(dir, 'release'), probe: join(dir, 'probe'), }; + /** Queue marks in the store's journal while no host has it open. */ const queueMarks = (messageIds: string[]) => { - const outbox = new DiscordAwarenessOutbox(journal); - const batch = outbox.prepare({ - agentName: 'resident', - sourceBranch: 'source', - targetBranch: 'main', - refs: messageIds.map((messageId) => ({ serverId: 'discord', channelId: 'discord:g1:c1', messageId })), - scope: 'all', - activationPolicy: 'explicit', - })!; - outbox.activate(batch.id); - return outbox; + const store = JsStore.openOrCreate({ path: storePath }); + try { + const outbox = new DiscordAwarenessOutbox(store); + const batch = outbox.prepare({ + agentName: 'resident', + sourceBranch: 'source', + targetBranch: 'main', + refs: messageIds.map((messageId) => ({ serverId: 'discord', channelId: 'discord:g1:c1', messageId })), + scope: 'all', + activationPolicy: 'explicit', + })!; + outbox.activate(batch.id); + } finally { + store.close(); + } }; + /** The running framework's own journal instance. */ + const journal = (framework: AgentFramework): DiscordAwarenessOutbox => + (framework as unknown as { discordAwarenessOutbox: DiscordAwarenessOutbox }).discordAwarenessOutbox; const config = (opts: { discord?: boolean; requestTimeoutMs?: number; awarenessDeadlineMs?: number } = {}) => ({ storePath, membrane: new MockMembrane().asMembrane(), agents: [{ name: 'resident', model: 'test', systemPrompt: 'test' }], modules: [], maintenanceIntervalMs: 0, - discordAwarenessOutboxPath: journal, ...(opts.awarenessDeadlineMs ? { discordAwarenessDeadlineMs: opts.awarenessDeadlineMs } : {}), ...(opts.discord === false ? {} : { mcplServers: [{ @@ -101,7 +106,7 @@ test('startup does not wait for queued marks, and channel traffic flows while th ); writeFileSync(paths.hold, '1'); await waitFor('both adds confirmed', () => { - const outbox = new DiscordAwarenessOutbox(journal); + const outbox = journal(framework!); return outbox.operations().filter((op) => outbox.operationStatus(op) === 'confirmed').length === 2; }); } finally { @@ -117,12 +122,9 @@ test('the mandatory awareness deadline still bounds a hung reaction with request try { queueMarks(['m1']); framework = await AgentFramework.create(config({ requestTimeoutMs: 0, awarenessDeadlineMs: 150 })); - // Another reader sees an unanswered dispatch as unknown at once; wait - // for the drain's own outcome record. await waitFor('the hung add to be recorded unknown', () => - new DiscordAwarenessOutbox(journal).operations().some((op) => op.attempts[0]?.outcome === 'unknown')); - const outbox = new DiscordAwarenessOutbox(journal); - const [op] = outbox.operations(); + journal(framework!).operations().some((op) => op.attempts[0]?.outcome === 'unknown')); + const [op] = journal(framework).operations(); assert.equal(op.attempts[0].outcome, 'unknown'); assert.match(op.attempts[0].error ?? '', /did not respond/); } finally { @@ -137,17 +139,17 @@ test('a route that is not connected leaves marks queued; they go out once it is' let framework: AgentFramework | undefined; try { queueMarks(['m1']); - const before = readFileSync(journal, 'utf8'); framework = await AgentFramework.create(config({ discord: false })); + const before = JSON.stringify(journal(framework).operations()); await framework.syncDiscordAwarenessMarkers(); - assert.equal(readFileSync(journal, 'utf8'), before, 'nothing recorded for a route that is down'); + assert.equal(JSON.stringify(journal(framework).operations()), before, 'nothing recorded for a route that is down'); await framework.stop(); framework = undefined; writeFileSync(paths.hold, '1'); framework = await AgentFramework.create(config()); await waitFor('the queued add confirmed', () => { - const outbox = new DiscordAwarenessOutbox(journal); + const outbox = journal(framework!); return outbox.operations().every((op) => outbox.operationStatus(op) === 'confirmed'); }); assert.deepEqual(jsonl(paths.calls).map((call) => call.name), ['add_reaction']); @@ -159,13 +161,14 @@ test('a route that is not connected leaves marks queued; they go out once it is' }); test('a corrupt journal stops startup, since it may hold a suppression the body needs resumed', async () => { - const { dir, journal, config } = setup(); + const { dir, storePath, config } = setup(); try { - mkdirSync(join(journal, '..'), { recursive: true }); - writeFileSync(journal, '{"t":"batch",\nnot json at all\n{"t":"commit","at":1,"txn":"x"}\n'); + const store = JsStore.openOrCreate({ path: storePath }); + store.appendJson(DISCORD_AWARENESS_RECORD_TYPE, { records: 'not a record group' }); + store.close(); await assert.rejects( AgentFramework.create(config({ discord: false })), - /Discord awareness accounting failed during startup reconciliation: Corrupt Discord awareness journal/, + /Discord awareness accounting failed during startup reconciliation: Corrupt Discord awareness journal entry/, ); } finally { rmSync(dir, { recursive: true, force: true }); @@ -188,7 +191,7 @@ test('a branch switch neither adds nor removes marks', async () => { }); const result = await framework.rollbackToMessage('resident', { messageId: tail, marks: { scope: 'addressed' } }); await waitFor('the add confirmed', () => - new DiscordAwarenessOutbox(journal).operations().some((op) => op.attempts[0]?.outcome === 'confirmed')); + journal(framework!).operations().some((op) => op.attempts[0]?.outcome === 'confirmed')); // Switch back to the source and forward again: nothing is sent. await cm.switchBranch(result.sourceBranch); await framework.syncDiscordAwarenessMarkers(); @@ -198,7 +201,7 @@ test('a branch switch neither adds nor removes marks', async () => { framework = await AgentFramework.create(config()); await framework.syncDiscordAwarenessMarkers(); assert.deepEqual(jsonl(paths.calls).map((call) => call.name), ['add_reaction']); - assert.equal(new DiscordAwarenessOutbox(journal).operations().length, 1); + assert.equal(journal(framework).operations().length, 1); } finally { writeFileSync(paths.hold, '1'); await framework?.stop(); diff --git a/test/discord-awareness-outbox.test.ts b/test/discord-awareness-outbox.test.ts index 4ba916680..c3b4eb532 100644 --- a/test/discord-awareness-outbox.test.ts +++ b/test/discord-awareness-outbox.test.ts @@ -1,25 +1,30 @@ /** * The Discord awareness-mark journal: an append-only record of what was - * asked and what was answered, never a model of what is on Discord. + * asked and what was answered, kept as typed records in the Chronicle store + * (RecordJournal), never a model of what is on Discord. * * Contract (room-225's proposed upstream change, after a field incident in * which a web-UI rollback queued 918 💤 reactions on other people's messages): * marks are an explicit publication choice (default none); one-shot, so * branch moves and restarts never derive operations from branch state; and * cancel / retract / release are explicit operator acts whose receipts say - * what is known and what is not. + * what history establishes and what it leaves open. */ import { test } from 'node:test'; import assert from 'node:assert/strict'; -import { appendFileSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { existsSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; import { join } from 'node:path'; import { tmpdir } from 'node:os'; +import { JsStore } from '@animalabs/chronicle'; import { + DISCORD_AWARENESS_RECORD_TYPE, DiscordAwarenessOutbox, boundDiscordAwarenessText, extractDiscordAwarenessRefs, selectDiscordAwarenessRefs, + type DiscordAwarenessBatchView, type DiscordAwarenessRef, + type DiscordAwarenessRetractView, } from '../src/recovery/discord-awareness-outbox.js'; import { AgentFramework } from '../src/framework.js'; import { McplRequestError } from '../src/mcpl/server-connection.js'; @@ -28,12 +33,24 @@ import { OperatorLog } from '../src/operator-log.js'; const ref = (messageId: string, channel = 'c1'): DiscordAwarenessRef => ({ serverId: 'discord', channelId: `discord:g1:${channel}`, messageId }); -function withJournal(fn: (path: string, dir: string) => void | Promise): () => Promise { +interface Harness { + store: JsStore; + dir: string; + legacyPath: string; + /** A fresh instance over the same store: what a restarted process reads. */ + reopen(): DiscordAwarenessOutbox; +} + +function withJournal(fn: (outbox: DiscordAwarenessOutbox, h: Harness) => void | Promise): () => Promise { return async () => { const dir = mkdtempSync(join(tmpdir(), 'discord-awareness-')); + const store = JsStore.openOrCreate({ path: join(dir, 'store') }); + const legacyPath = join(dir, 'recovery', 'discord-awareness-outbox.json'); + const h: Harness = { store, dir, legacyPath, reopen: () => new DiscordAwarenessOutbox(store, { legacyPath }) }; try { - await fn(join(dir, 'recovery', 'discord-awareness-journal.jsonl'), dir); + await fn(h.reopen(), h); } finally { + if (!store.isClosed()) store.close(); rmSync(dir, { recursive: true, force: true }); } }; @@ -52,25 +69,24 @@ function activeBatch(outbox: DiscordAwarenessOutbox, refs: DiscordAwarenessRef[] return batch; } -/** Dispatch everything due, answering each dispatch with `answer`. */ -function drain( +/** Claim and answer one dispatch for a message. */ +function answer( outbox: DiscordAwarenessOutbox, - answer: (dispatch: { action: string; messageId: string }) => 'confirmed' | 'failed' | 'not-sent' | 'unknown', -): string[] { - const seen: string[] = []; - for (let pass = 0; pass < 10; pass++) { - const dispatches = outbox.pendingDispatches('discord'); - if (dispatches.length === 0) break; - for (const dispatch of dispatches) { - const attempts = outbox.recordDispatching(dispatch); - const outcome = answer({ action: dispatch.action, messageId: dispatch.key.messageId }); - seen.push(`${dispatch.action}:${dispatch.key.messageId}:${outcome}`); - outbox.recordOutcome(attempts, outcome, outcome === 'failed' ? { permanent: true, error: 'Unknown Message' } : {}); - } - } - return seen; + messageId: string, + outcome: 'confirmed' | 'failed' | 'not-sent' | 'unknown', + detail: { permanent?: boolean; error?: string } = {}, +) { + const claimed = outbox.claimDispatch('discord', (d) => d.key.messageId !== messageId); + assert.ok(claimed, `a dispatch for ${messageId} is due`); + outbox.recordOutcome(claimed.attempts, outcome, detail); + return claimed; } +const batches = (outbox: DiscordAwarenessOutbox) => + outbox.view().filter((v): v is DiscordAwarenessBatchView => v.kind === 'batch'); +const retracts = (outbox: DiscordAwarenessOutbox) => + outbox.view().filter((v): v is DiscordAwarenessRetractView => v.kind === 'retract'); + test('extractDiscordAwarenessRefs keeps only direct Discord addressing metadata', () => { const refs = extractDiscordAwarenessRefs([ { metadata: { serverId: 'discord', channelId: 'discord:g1:c1', messageId: 'm1' } }, @@ -88,68 +104,53 @@ test('selection: none marks nothing; addressed means the chat:addressed tag; ref { metadata: { ...ref('b2') } }, { metadata: { ...ref('a2', 'dm'), tags: ['chat:addressed', 'chat:dm'] } }, ]; - const none = selectDiscordAwarenessRefs(messages, 'none'); - assert.deepEqual(none, { refs: [], addressable: 4, unmarked: 4, notRemoved: 0 }); - + assert.deepEqual(selectDiscordAwarenessRefs(messages, 'none'), { refs: [], addressable: 4, unmarked: 4, notRemoved: 0 }); const addressed = selectDiscordAwarenessRefs(messages, { scope: 'addressed' }); assert.deepEqual(addressed.refs.map((r) => r.messageId), ['a1', 'a2']); assert.equal(addressed.unmarked, 2); - - const all = selectDiscordAwarenessRefs(messages, { scope: 'all' }); - assert.equal(all.refs.length, 4); - + assert.equal(selectDiscordAwarenessRefs(messages, { scope: 'all' }).refs.length, 4); // A frozen choice: only previewed refs, and never anything else removed. const frozen = selectDiscordAwarenessRefs(messages, { scope: 'all', refs: [ref('a1'), ref('gone')] }); assert.deepEqual(frozen.refs.map((r) => r.messageId), ['a1']); - assert.equal(frozen.notRemoved, 1, 'an authorized ref the surgery did not remove'); + assert.equal(frozen.notRemoved, 1); assert.equal(frozen.unmarked, 3); - // Scope still applies to authorized refs. const frozenAddressed = selectDiscordAwarenessRefs(messages, { scope: 'addressed', refs: [ref('b1'), ref('a2', 'dm')] }); assert.deepEqual(frozenAddressed.refs.map((r) => r.messageId), ['a2']); }); -test('a prepared batch delivers nothing; activation queues one add per ref, written ahead of dispatch', withJournal((path) => { - const outbox = new DiscordAwarenessOutbox(path); +test('the journal is typed records in the store: activation queues one add per ref, written ahead of dispatch', withJournal((outbox, h) => { const batch = outbox.prepare({ agentName: 'cairn', sourceBranch: 'main', targetBranch: 'rollback/cairn/1', refs: [ref('m1'), ref('m2')], scope: 'all', })!; assert.equal(outbox.pendingDispatches('discord').length, 0, 'prepared work is not deliverable'); assert.equal(outbox.activate(batch.id), 2); - const [first] = outbox.pendingDispatches('discord'); - assert.equal(first.action, 'add'); - assert.equal(first.key.emoji, '💤'); - - const attempts = outbox.recordDispatching(first); - // The write-ahead record is durable before the request could leave. - const journal = readFileSync(path, 'utf8'); - assert.match(journal, /"t":"dispatching"/); - // A fresh reader (another process) sees it as unknown, never as unsent. - const other = new DiscordAwarenessOutbox(path); - assert.equal(other.operations().find((op) => op.opId === first.opIds[0])!.attempts.length, 1); - assert.equal(other.operationStatus(other.operations().find((op) => op.opId === first.opIds[0])!), 'unknown'); - - outbox.recordOutcome(attempts, 'confirmed'); - assert.deepEqual(outbox.pendingDispatches('discord').map((d) => d.key.messageId), ['m2']); + assert.ok(h.store.getRecordIdsByType(DISCORD_AWARENESS_RECORD_TYPE).length >= 2); + + const claimed = outbox.claimDispatch('discord')!; + assert.equal(claimed.dispatch.action, 'add'); + // The write-ahead record is in the store before the request could leave: + // a fresh reader (a restarted process) sees the attempt, outcome unknown. + const other = h.reopen(); + const op = other.operations().find((candidate) => candidate.opId === claimed.attempts[0].opId)!; + assert.equal(op.attempts.length, 1); + assert.equal(other.operationStatus(op), 'unknown'); + + outbox.recordOutcome(claimed.attempts, 'confirmed'); + assert.deepEqual(outbox.pendingDispatches('discord').map((d) => d.key.messageId), [claimed.dispatch.key.messageId === 'm1' ? 'm2' : 'm1']); })); -test('startup derives nothing from ancestry: exact-target crash completion, everything else held', withJournal((path) => { - const outbox = new DiscordAwarenessOutbox(path); - const crashed = outbox.prepare({ - agentName: 'cairn', sourceBranch: 'main', targetBranch: 'rollback/cairn/1', refs: [ref('m1')], scope: 'all', - })!; - const elsewhere = outbox.prepare({ - agentName: 'cairn', sourceBranch: 'main', targetBranch: 'rollback/cairn/2', refs: [ref('m2')], scope: 'addressed', - })!; +test('startup derives nothing from ancestry: exact-target crash completion, everything else held', withJournal((outbox, h) => { + const crashed = outbox.prepare({ agentName: 'cairn', sourceBranch: 'main', targetBranch: 'rollback/cairn/1', refs: [ref('m1')], scope: 'all' })!; + const elsewhere = outbox.prepare({ agentName: 'cairn', sourceBranch: 'main', targetBranch: 'rollback/cairn/2', refs: [ref('m2')], scope: 'addressed' })!; const suppression = outbox.prepare({ agentName: 'cairn', sourceBranch: 'main', targetBranch: 'rollback/cairn/1', refs: [ref('m3')], scope: 'all', activationPolicy: 'explicit', suppressionIntervals: [{ fromId: 'i1', toId: 'i1' }], })!; const active = activeBatch(outbox, [ref('m4')]); - const [dispatch] = outbox.pendingDispatches('discord').filter((d) => d.key.messageId === 'm4'); - outbox.recordDispatching(dispatch); // the process dies with this request on the wire + outbox.claimDispatch('discord', (d) => d.key.messageId !== 'm4'); // dies with this request on the wire - const restarted = new DiscordAwarenessOutbox(path); + const restarted = h.reopen(); const recovered = restarted.recoverAtStartup('rollback/cairn/1'); assert.deepEqual(recovered.activated, [crashed.id]); assert.deepEqual(recovered.held, [elsewhere.id]); @@ -159,291 +160,324 @@ test('startup derives nothing from ancestry: exact-target crash completion, ever assert.equal(status.get(elsewhere.id), 'held'); assert.equal(status.get(suppression.id), 'prepared', 'left for resumePreparedDiscordSuppressions'); assert.equal(status.get(active.id), 'active'); - // The unanswered add is unknown, and as the newest op for its key it is - // retried (idempotent); the held batch's ref is not dispatched at all. - assert.deepEqual( - restarted.pendingDispatches('discord').map((d) => d.key.messageId).sort(), - ['m1', 'm4'], - ); + // The unanswered add is unknown and, still the newest for its key, retried. + assert.deepEqual(restarted.pendingDispatches('discord').map((d) => d.key.messageId).sort(), ['m1', 'm4']); + // A later restart on another branch changes nothing. + assert.deepEqual(h.reopen().recoverAtStartup('main'), { activated: [], held: [suppression.id], unknownAttempts: 0 }); +})); - // A later restart on a different branch changes nothing. - const again = new DiscordAwarenessOutbox(path).recoverAtStartup('main'); - assert.deepEqual(again, { activated: [], held: [suppression.id], unknownAttempts: 0 }); +test('a dispatch is admitted from the journal as it is now: a cancel during an in-flight request stops the rest', withJournal((outbox) => { + const batch = activeBatch(outbox, [ref('m1'), ref('m2'), ref('m3')]); + // A drain computed this list, then sent the first while holding the rest. + const stale = outbox.pendingDispatches('discord'); + assert.equal(stale.length, 3); + const first = outbox.recordDispatching(stale[0]); + assert.equal(first.length, 1); + const receipt = outbox.cancel(batch.id, 'operator'); + assert.equal(receipt.cancelled, 2); + assert.equal(receipt.inFlight, 1); + // The stale list can no longer authorize anything. + assert.deepEqual(outbox.recordDispatching(stale[1]), []); + assert.deepEqual(outbox.recordDispatching(stale[2]), []); + outbox.recordOutcome(first, 'confirmed'); + assert.equal(outbox.claimDispatch('discord'), null); })); -test('cancel stops every further send of a batch, never removes, and says what may still land', withJournal((path) => { - const outbox = new DiscordAwarenessOutbox(path); +test('cancel stops every further send of a batch, never removes, and reports what may still land', withJournal((outbox, h) => { const batch = activeBatch(outbox, [ref('m1'), ref('m2'), ref('m3'), ref('m4')]); - const dispatches = outbox.pendingDispatches('discord'); - const byId = new Map(dispatches.map((d) => [d.key.messageId, d])); - outbox.recordOutcome(outbox.recordDispatching(byId.get('m1')!), 'confirmed'); - outbox.recordOutcome(outbox.recordDispatching(byId.get('m2')!), 'unknown', { error: 'did not respond' }); - const m3Attempts = outbox.recordDispatching(byId.get('m3')!); // on the wire right now + answer(outbox, 'm1', 'confirmed'); + answer(outbox, 'm2', 'unknown', { error: 'did not respond' }); + const m3 = outbox.claimDispatch('discord', (d) => d.key.messageId !== 'm3')!; // on the wire const receipt = outbox.cancel(batch.id, 'operator'); assert.deepEqual( - { cancelled: receipt.cancelled, inFlight: receipt.inFlight, unknown: receipt.unknown, confirmed: receipt.confirmed }, - { cancelled: 1, inFlight: 1, unknown: 1, confirmed: 1 }, + { target: receipt.target, kind: receipt.kind, cancelled: receipt.cancelled, inFlight: receipt.inFlight, unknown: receipt.unknown, confirmed: receipt.confirmed, unresolvedAttempts: receipt.unresolvedAttempts }, + { target: batch.id, kind: 'batch', cancelled: 1, inFlight: 1, unknown: 1, confirmed: 1, unresolvedAttempts: 2 }, ); - // Nothing new is sent, no removal is queued, and the unknown add is not - // retried; the key with a request on the wire stays busy meanwhile. - assert.equal(outbox.pendingDispatches('discord').length, 0); - - // The in-flight request lands after the cancel: that is recorded as what - // happened, and it is still not followed by anything. - outbox.recordOutcome(m3Attempts, 'confirmed'); - const m3 = outbox.operations().find((op) => op.key.messageId === 'm3')!; - assert.equal(outbox.operationStatus(m3), 'confirmed'); + assert.equal(outbox.pendingDispatches('discord').length, 0, 'nothing new is sent, nothing is removed'); + // The in-flight request lands after the cancel: recorded, not followed up. + outbox.recordOutcome(m3.attempts, 'confirmed'); + assert.equal(outbox.operationStatus(outbox.operations().find((op) => op.key.messageId === 'm3')!), 'confirmed'); assert.equal(outbox.pendingDispatches('discord').length, 0); - - // Cancellation is durable: a restart or a branch move re-arms nothing. - const restarted = new DiscordAwarenessOutbox(path); + // Durable across a restart. + const restarted = h.reopen(); restarted.recoverAtStartup('some/other/branch'); assert.ok(restarted.batches().find((b) => b.id === batch.id)?.cancelled); assert.equal(restarted.pendingDispatches('discord').length, 0); - // Retract still reaches what may have landed: m1, m2 (unknown) and m3. - const retracted = restarted.retract(batch.id); - assert.equal(retracted.removalsQueued, 3); - assert.equal(retracted.keysWithUnresolvedAdds, 1); })); -test('cancelling a batch whose surgery has not activated it stops every mark, and its body can still resume', withJournal((path) => { - const outbox = new DiscordAwarenessOutbox(path); - const rollback = outbox.prepare({ - agentName: 'cairn', sourceBranch: 'main', targetBranch: 'rollback/cairn/1', refs: [ref('m1'), ref('m2')], scope: 'all', - })!; - const receipt = outbox.cancel(rollback.id, 'operator'); - assert.equal(receipt.cancelled, 2); +test('the cancel receipt keeps an earlier unresolved request after a later answer', withJournal((outbox) => { + const batch = activeBatch(outbox, [ref('m1'), ref('m2')]); + answer(outbox, 'm1', 'unknown', { error: 'timed out' }); + answer(outbox, 'm1', 'confirmed'); // the retry confirms + answer(outbox, 'm2', 'unknown', { error: 'timed out' }); + answer(outbox, 'm2', 'failed', { permanent: true, error: 'Unknown Message' }); + const receipt = outbox.cancel(batch.id); + assert.equal(receipt.confirmed, 1); + assert.equal(receipt.unknown, 0); + assert.equal(receipt.unresolvedAttempts, 2, 'both first attempts may still land'); +})); + +test('cancelling a batch its surgery has not activated stops every mark; its body can still resume', withJournal((outbox) => { + const rollback = outbox.prepare({ agentName: 'cairn', sourceBranch: 'main', targetBranch: 'rollback/cairn/1', refs: [ref('m1'), ref('m2')], scope: 'all' })!; + assert.equal(outbox.cancel(rollback.id, 'operator').cancelled, 2); assert.throws(() => outbox.activate(rollback.id), /cancelled before activation/); - // Not activated by startup's crash completion, and not held either. assert.deepEqual(outbox.recoverAtStartup('rollback/cairn/1'), { activated: [], held: [], unknownAttempts: 0 }); - assert.equal(outbox.pendingDispatches('discord').length, 0); const suppression = outbox.prepare({ agentName: 'cairn', sourceBranch: 'main', targetBranch: 'suppress/cairn/1', refs: [ref('m3')], scope: 'all', activationPolicy: 'explicit', suppressionIntervals: [{ fromId: 'i1', toId: 'i1' }], })!; outbox.cancel(suppression.id); - // The suppression's body still resumes on its branch; its marks never go. assert.deepEqual(outbox.preparedSuppressionsForBranch('suppress/cairn/1').map((b) => b.id), [suppression.id]); outbox.recordSuppressionComplete(suppression.id); assert.deepEqual(outbox.preparedSuppressionsForBranch('suppress/cairn/1'), []); assert.equal(outbox.pendingDispatches('discord').length, 0); })); -test('retract removes this bot\'s mark across batches, supersedes unsent adds, and discloses unresolved attempts', withJournal((path) => { - const outbox = new DiscordAwarenessOutbox(path); +test('releasing a held suppression\'s marks never certifies or forgets its unfinished body', withJournal((outbox) => { + const suppression = outbox.prepare({ + agentName: 'cairn', sourceBranch: 'main', targetBranch: 'partial', refs: [ref('m1')], scope: 'all', + activationPolicy: 'explicit', suppressionIntervals: [{ fromId: 'i1', toId: 'i2' }], + })!; + assert.deepEqual(outbox.recoverAtStartup('main').held, [suppression.id]); + assert.deepEqual(outbox.preparedSuppressionsForBranch('partial').map((b) => b.id), [suppression.id]); + const released = outbox.release(suppression.id, 'operator'); + assert.equal(released.addsQueued, 1); + assert.equal(outbox.batches()[0].status, 'active'); + assert.equal(outbox.batches()[0].suppressionComplete, undefined); + // Body recovery still owes the redactions on `partial`. + assert.deepEqual(outbox.preparedSuppressionsForBranch('partial').map((b) => b.id), [suppression.id]); +})); + +test('retract queues a removal for every selected key, whatever history says, and discloses unresolved requests', withJournal((outbox) => { const a = activeBatch(outbox, [ref('m1'), ref('m2'), ref('m3')]); const b = activeBatch(outbox, [ref('m1')]); // another batch asked for the same reaction - // m1's adds share one dispatch (same key, same action, both requested). - const m1 = outbox.pendingDispatches('discord').find((d) => d.key.messageId === 'm1')!; - assert.equal(m1.opIds.length, 2); - const m1Attempts = outbox.recordDispatching(m1); - outbox.recordOutcome(m1Attempts, 'unknown', { error: 'did not respond within 75ms' }); - // m2's add never left the host; m3's was confirmed. - const m2 = outbox.pendingDispatches('discord').find((d) => d.key.messageId === 'm2')!; - outbox.recordOutcome(outbox.recordDispatching(m2), 'not-sent'); - const m3 = outbox.pendingDispatches('discord').find((d) => d.key.messageId === 'm3')!; - outbox.recordOutcome(outbox.recordDispatching(m3), 'confirmed'); + const m1 = outbox.claimDispatch('discord', (d) => d.key.messageId !== 'm1')!; + assert.equal(m1.attempts.length, 2, "both batches' adds ride one request"); + outbox.recordOutcome(m1.attempts, 'unknown', { error: 'did not respond within 75ms' }); + answer(outbox, 'm2', 'not-sent'); + answer(outbox, 'm3', 'confirmed'); const receipt = outbox.retract(a.id, 'operator'); - // m1 and m3 had adds leave the host; m2's never did, so nothing to remove. - assert.equal(receipt.removalsQueued, 2); - assert.equal(receipt.addsSuperseded, 1, "m2's unsent add is superseded"); - assert.equal(receipt.keysWithUnresolvedAdds, 1, 'm1: a timed-out add may land after the removal'); - assert.equal(receipt.unresolvedAddAttempts, 1, "one request carried both batches' adds"); - - const dispatches = outbox.pendingDispatches('discord'); - assert.deepEqual(dispatches.map((d) => `${d.action}:${d.key.messageId}`).sort(), ['remove:m1', 'remove:m3']); - - // The removal is confirmed, and the earlier unknown add stays unresolved: - // a confirmation of a different request never resolves it. - drain(outbox, () => 'confirmed'); - const views = new Map(outbox.view().map((v) => [v.id, v])); - assert.equal(views.get(a.id)!.adds.unknown, 1); - assert.equal(views.get(a.id)!.removals.confirmed, 2); - assert.equal(views.get(b.id)!.unresolvedAttempts, 1); + assert.equal(receipt.removalsQueued, 3, 'm2 too: history does not decide whether the act is carried out'); + assert.equal(receipt.addsSuperseded, 1, "m2's due add stops"); + assert.equal(receipt.keysWithUnresolvedAdds, 1); + assert.equal(receipt.unresolvedAddAttempts, 1, 'one physical request carried both adds'); + assert.equal(receipt.keysWithLegacyUncertainty, 0); + assert.deepEqual(outbox.pendingDispatches('discord').map((d) => `${d.action}:${d.key.messageId}`).sort(), + ['remove:m1', 'remove:m2', 'remove:m3']); + for (const id of ['m1', 'm2', 'm3']) answer(outbox, id, 'confirmed'); + const views = new Map(batches(outbox).map((v) => [v.id, v])); + assert.equal(views.get(a.id)!.removals.confirmed, 3); + assert.equal(views.get(b.id)!.unresolvedAttempts, 1, 'the unknown add stays unresolved history'); + const [retractView] = retracts(outbox); + assert.equal(retractView.target, a.id); + assert.equal(retractView.removals.confirmed, 3); })); -test('a retry confirming an unknown add never erases the unresolved first attempt', withJournal((path) => { - const outbox = new DiscordAwarenessOutbox(path); +test('a newer opposite request stops the retries of a retryably failed one, in both directions', withJournal((outbox) => { const batch = activeBatch(outbox, [ref('m1')]); - const first = outbox.pendingDispatches('discord')[0]; - outbox.recordOutcome(outbox.recordDispatching(first), 'unknown'); - // Still the newest op for its key: retried (idempotent), as attempt 2. - const retry = outbox.pendingDispatches('discord')[0]; - assert.deepEqual(retry.opIds, first.opIds); - const retryAttempts = outbox.recordDispatching(retry); - assert.equal(retryAttempts[0].attempt, 2); - outbox.recordOutcome(retryAttempts, 'confirmed'); - assert.equal(outbox.pendingDispatches('discord').length, 0); - assert.equal(outbox.view()[0].unresolvedAttempts, 1, 'attempt 1 is still unknown'); - + answer(outbox, 'm1', 'failed', { error: 'Discord 503' }); // retryable const receipt = outbox.retract(batch.id); - assert.equal(receipt.keysWithUnresolvedAdds, 1); - assert.equal(receipt.unresolvedAddAttempts, 1); + assert.equal(receipt.addsSuperseded, 1); + for (let i = 0; i < 3; i++) { + const claimed = outbox.claimDispatch('discord')!; + assert.equal(claimed.dispatch.action, 'remove'); + outbox.recordOutcome(claimed.attempts, 'failed', { error: 'Discord 503' }); + } + const add = outbox.operations().find((op) => op.action === 'add')!; + assert.equal(add.attempts.length, 1, 'its history stays'); + assert.equal(add.cancelled?.reason, 'superseded'); + // Mirror: a new add supersedes the retryably failing remove. + activeBatch(outbox, [ref('m1')]); + const next = outbox.claimDispatch('discord')!; + assert.equal(next.dispatch.action, 'add'); + assert.equal(outbox.operations().find((op) => op.action === 'remove')!.cancelled?.reason, 'superseded'); })); -test('a new request reaches Discord even when an earlier one got the same answer', withJournal((path) => { - const outbox = new DiscordAwarenessOutbox(path); - const first = activeBatch(outbox, [ref('m1')]); - drain(outbox, () => 'confirmed'); - outbox.retract(first.id); - drain(outbox, () => 'confirmed'); - // Someone else's tool may have reacted again since; a fresh, explicit - // placement is sent rather than skipped on the strength of old receipts. - activeBatch(outbox, [ref('m1')]); - assert.deepEqual(outbox.pendingDispatches('discord').map((d) => d.action), ['add']); +test('a retract request can be cancelled by the id its receipt returned, even after a restart', withJournal((outbox, h) => { + const batch = activeBatch(outbox, [ref('m1'), ref('m2'), ref('m3')]); + for (const id of ['m1', 'm2', 'm3']) answer(outbox, id, 'confirmed'); + const { requestId } = outbox.retract(batch.id); + const onWire = outbox.claimDispatch('discord')!; + const restarted = h.reopen(); + // The retract is listed, so its id survives the original receipt. + assert.equal(retracts(restarted)[0].id, requestId); + const receipt = outbox.cancel(requestId, 'operator'); + assert.deepEqual( + { kind: receipt.kind, cancelled: receipt.cancelled, inFlight: receipt.inFlight }, + { kind: 'retract', cancelled: 2, inFlight: 1 }, + ); + outbox.recordOutcome(onWire.attempts, 'confirmed'); + assert.equal(outbox.claimDispatch('discord'), null); + assert.ok(retracts(outbox)[0].cancelled); })); -test('retract supersedes an add that never left the host instead of churning the reaction', withJournal((path) => { - const outbox = new DiscordAwarenessOutbox(path); - const batch = activeBatch(outbox, [ref('m2'), ref('m3')]); - // m3's add was sent and confirmed; m2's has not been sent yet. - const m3 = outbox.pendingDispatches('discord').find((d) => d.key.messageId === 'm3')!; - outbox.recordOutcome(outbox.recordDispatching(m3), 'confirmed'); +test('a retry confirming an unknown add never erases the unresolved first attempt', withJournal((outbox) => { + const batch = activeBatch(outbox, [ref('m1')]); + const first = answer(outbox, 'm1', 'unknown'); + const retry = outbox.claimDispatch('discord')!; + assert.deepEqual(retry.dispatch.opIds, first.dispatch.opIds); + assert.equal(retry.attempts[0].attempt, 2); + outbox.recordOutcome(retry.attempts, 'confirmed'); + assert.equal(batches(outbox)[0].unresolvedAttempts, 1); const receipt = outbox.retract(batch.id); - assert.equal(receipt.addsSuperseded, 1); - assert.equal(receipt.removalsQueued, 1, 'nothing was ever sent for m2, so there is nothing to remove'); - assert.deepEqual(outbox.pendingDispatches('discord').map((d) => `${d.action}:${d.key.messageId}`), ['remove:m3']); - const m2 = outbox.operations().find((op) => op.key.messageId === 'm2')!; - assert.equal(m2.cancelled?.reason, 'superseded'); + assert.equal(receipt.unresolvedAddAttempts, 1); })); -test('release queues a held batch\'s recorded operations, explicitly', withJournal((path) => { - const outbox = new DiscordAwarenessOutbox(path); - const batch = outbox.prepare({ - agentName: 'cairn', sourceBranch: 'main', targetBranch: 'rollback/cairn/9', refs: [ref('m1'), ref('m2')], scope: 'addressed', - })!; +test('release queues a held batch\'s recorded operations, explicitly', withJournal((outbox) => { + const batch = outbox.prepare({ agentName: 'cairn', sourceBranch: 'main', targetBranch: 'rollback/cairn/9', refs: [ref('m1'), ref('m2')], scope: 'addressed' })!; outbox.recoverAtStartup('main'); assert.equal(outbox.batches()[0].status, 'held'); assert.equal(outbox.pendingDispatches('discord').length, 0); const receipt = outbox.release(batch.id, 'operator'); assert.deepEqual({ adds: receipt.addsQueued, removals: receipt.removalsQueued }, { adds: 2, removals: 0 }); - assert.equal(outbox.batches()[0].status, 'active'); assert.equal(outbox.pendingDispatches('discord').length, 2); assert.throws(() => outbox.release(batch.id), /not held/); })); -test('outside error text is bounded before it is journaled', withJournal((path) => { +test('settleActivation says what the journal will do when activation cannot be recorded', withJournal((outbox) => { + const ok = outbox.prepare({ agentName: 'cairn', sourceBranch: 'main', targetBranch: 'b1', refs: [ref('m1')], scope: 'all' })!; + assert.deepEqual(outbox.settleActivation(ok.id), { status: 'queued', queued: 1 }); + const cancelled = outbox.prepare({ agentName: 'cairn', sourceBranch: 'main', targetBranch: 'b2', refs: [ref('m2')], scope: 'all' })!; + outbox.cancel(cancelled.id); + const settled = outbox.settleActivation(cancelled.id); + assert.equal(settled.status, 'not-scheduled'); + assert.equal(outbox.batches().find((b) => b.id === cancelled.id)?.status, 'discarded'); +})); + +test('outside error text is bounded before it is journaled', withJournal((outbox) => { assert.equal(boundDiscordAwarenessText('short'), 'short'); const huge = `${'x'.repeat(300)}${'界'.repeat(500_000)}😀tail`; const bounded = boundDiscordAwarenessText(huge); assert.ok(bounded.length <= 500, `bounded to ${bounded.length}`); assert.match(bounded, /chars omitted/); assert.ok(!/[\uD800-\uDBFF](?![\uDC00-\uDFFF])|(? { - const outbox = new DiscordAwarenessOutbox(path); - const batch = outbox.prepare({ - agentName: 'cairn', sourceBranch: 'main', targetBranch: 'rollback/cairn/1', refs: [ref('m1'), ref('m2')], scope: 'all', - })!; - outbox.activate(batch.id); - const committed = readFileSync(path, 'utf8'); - // Simulate an activation whose append was cut off before its commit. - const lines = committed.trimEnd().split('\n'); - const activation = lines.filter((line) => line.includes('"txn"')); - assert.ok(activation.length >= 3, 'activation is a transaction'); - writeFileSync(path, lines.filter((line) => !line.includes('"txn"') && !line.includes('"t":"commit"')).join('\n') + '\n'); - appendFileSync(path, activation.slice(0, -1).join('\n') + '\n' + '{"t":"requested","at":1,"op'); - const reread = new DiscordAwarenessOutbox(path); - assert.equal(reread.batches()[0].status, 'prepared', 'the activation never committed'); - assert.equal(reread.pendingDispatches('discord').length, 0); - assert.equal(reread.operations().length, 0); +test('a corrupt journal entry stops the reader rather than being skipped', withJournal((_outbox, h) => { + h.store.appendJson(DISCORD_AWARENESS_RECORD_TYPE, { records: 'nope' }); + assert.throws(() => h.reopen().batches(), /Corrupt Discord awareness journal entry/); })); -test('an append after a torn tail repairs it instead of corrupting the journal', withJournal((path) => { - const outbox = new DiscordAwarenessOutbox(path); - const batch = activeBatch(outbox, [ref('m1'), ref('m2')]); - // A single-record append cut off mid-line (the process died writing it). - appendFileSync(path, '{"t":"dispatching","at":1,"opId":"x","att'); - const reread = new DiscordAwarenessOutbox(path); - assert.equal(reread.pendingDispatches('discord').length, 2, 'the torn record never applied'); - // The next append must not share the torn line. - const dispatch = reread.pendingDispatches('discord')[0]; - reread.recordOutcome(reread.recordDispatching(dispatch), 'confirmed'); - const again = new DiscordAwarenessOutbox(path); - assert.equal(again.pendingDispatches('discord').length, 1); - assert.ok(!readFileSync(path, 'utf8').includes('"att{'), 'the fragment was truncated away'); - - // A complete record that only lost its newline is kept, then terminated. - const lines = readFileSync(path, 'utf8'); - writeFileSync(path, lines.trimEnd()); - const kept = new DiscordAwarenessOutbox(path); - assert.equal(kept.pendingDispatches('discord').length, 1); - kept.cancel(batch.id); - assert.equal(new DiscordAwarenessOutbox(path).pendingDispatches('discord').length, 0); - assert.equal(new DiscordAwarenessOutbox(path).batches()[0].cancelled !== undefined, true); +test('checkpoints bound replay and reproduce the reduced state', withJournal((_outbox, h) => { + const outbox = new DiscordAwarenessOutbox(h.store, { checkpointEvery: 8 }); + const batch = activeBatch(outbox, Array.from({ length: 5 }, (_, i) => ref(`m${i}`))); + for (let round = 0; round < 3; round++) { + for (let i = 0; i < 5; i++) { + const claimed = outbox.claimDispatch('discord', (d) => d.key.messageId !== `m${i}`); + if (claimed) outbox.recordOutcome(claimed.attempts, round === 2 ? 'confirmed' : 'failed', { error: 'Discord 503' }); + } + } + outbox.retract(batch.id); + assert.ok(h.store.getRecordIdsByType(`${DISCORD_AWARENESS_RECORD_TYPE}/checkpoint`).length >= 2, 'checkpoints were written'); + const reread = new DiscordAwarenessOutbox(h.store, { checkpointEvery: 8 }); + assert.deepEqual(reread.view(), outbox.view()); + assert.deepEqual(reread.operations(), outbox.operations()); + assert.equal(batches(reread).find((v) => v.id === batch.id)!.adds.confirmed, 5); + assert.equal(retracts(reread).length, 1); })); -test('v2 import keeps what the old ledger recorded, holds pending work, and happens once', withJournal((path, dir) => { - const legacyPath = join(dir, 'recovery', 'discord-awareness-outbox.json'); - const entry = (messageId: string, fields: Record) => ({ ...ref(messageId), attempts: 1, lastAction: 'add', ...fields }); +test('v2 import keeps what the old ledger recorded as evidence, holds pending work, and happens once', withJournal((_outbox, h) => { + const entry = (messageId: string, fields: Record) => ({ ...ref(messageId), lastAction: 'add', ...fields }); const legacy = { version: 2, batches: [ { // The household's shape: an active target-branch batch, some marks - // applied, the rest marked permanent-failure by hand after shutdown. + // applied, the rest marked permanent-failure by hand after shutdown, + // one request interrupted on the wire. id: 'dd34b043', status: 'active', agentName: 'resident', sourceBranch: 'main', targetBranch: 'rollback/resident/1791358473307', emoji: '💤', createdAt: 1791358473400, activationPolicy: 'target-branch', refs: [ - entry('a1', { desired: true, markerPresent: true, deliveryStatus: 'applied' }), - entry('a2', { desired: true, markerPresent: true, deliveryStatus: 'applied' }), - entry('p1', { desired: true, markerPresent: false, deliveryStatus: 'permanent-failure', lastError: 'Cannot send request: connection to "discord" is closed' }), + entry('a1', { desired: true, markerPresent: true, deliveryStatus: 'applied', attempts: 1 }), + entry('n1', { desired: true, markerPresent: false, deliveryStatus: 'permanent-failure', attempts: 1, lastError: 'Cannot send request: connection to "discord" is closed' }), + entry('u1', { desired: true, markerPresent: false, deliveryStatus: 'permanent-failure', attempts: 1, lastError: 'Connection to MCPL server "discord" closed while awaiting response for tools/call (id=4)' }), + entry('g1', { desired: true, markerPresent: false, deliveryStatus: 'permanent-failure', attempts: 1, lastError: 'Unknown Message' }), + entry('g3', { desired: true, markerPresent: false, deliveryStatus: 'permanent-failure', attempts: 3, lastError: 'Unknown Message' }), ], }, { - // An interrupted drain that nobody edited: its pending work is held. + // June's probe: a pending entry whose one attempt's outcome is unknown. id: 'pending-batch', status: 'active', agentName: 'resident', sourceBranch: 'main', targetBranch: 'rollback/resident/2', emoji: '💤', createdAt: 1791358473500, - refs: [ - entry('q1', { desired: true, markerPresent: false, deliveryStatus: 'pending', lastError: 'Cannot send request' }), - entry('q2', { desired: true, markerPresent: true, deliveryStatus: 'applied' }), - ], + refs: [entry('p1', { desired: true, markerPresent: false, deliveryStatus: 'pending', attempts: 1, lastError: 'did not respond within 10000ms' })], }, ], }; - mkdirSync(join(dir, 'recovery'), { recursive: true }); - writeFileSync(legacyPath, JSON.stringify(legacy)); - - const outbox = new DiscordAwarenessOutbox(path); - // Reading imports in memory only; nothing is dispatched from an import. - assert.equal(outbox.pendingDispatches('discord').length, 0); - const views = new Map(outbox.view().map((v) => [v.id, v])); - assert.equal(views.get('dd34b043')!.adds.confirmed, 2); - assert.equal(views.get('dd34b043')!.adds.failed, 1); + mkdirSync(join(h.dir, 'recovery'), { recursive: true }); + writeFileSync(h.legacyPath, JSON.stringify(legacy)); + + const outbox = h.reopen(); + assert.equal(outbox.pendingDispatches('discord').length, 0, 'nothing imported is dispatched'); + assert.equal(outbox.operations().length, 0, 'no synthesized attempts'); + assert.equal(existsSync(h.legacyPath), false); + assert.equal(existsSync(`${h.legacyPath}.migrated-v2`), true); + const views = new Map(batches(outbox).map((v) => [v.id, v])); + assert.deepEqual(views.get('dd34b043')!.legacy, { + entries: 5, + lastAddConfirmed: 1, + lastRemoveConfirmed: 0, + // u1: its one outcome unrecorded; g3: two earlier attempts unrecorded. + outcomesUnrecorded: 3, + }); assert.equal(views.get('pending-batch')!.status, 'held'); assert.equal(views.get('pending-batch')!.held!.releaseActions, 1); - // The first write persists the import (with its hash) and retires the file. + // Retract carries out the act for every ref and discloses what history leaves open. const receipt = outbox.retract('dd34b043', 'operator'); - assert.equal(receipt.removalsQueued, 3, 'every add that left the host is removed; the import cannot tell which were sent'); - assert.equal(existsSync(legacyPath), false); - assert.equal(existsSync(`${legacyPath}.migrated-v2`), true); - - // A crash between the import append and the rename re-reads the same - // file: its hash is recorded, so nothing is imported or queued twice. - writeFileSync(legacyPath, JSON.stringify(legacy)); - const again = new DiscordAwarenessOutbox(path); + assert.equal(receipt.removalsQueued, 5); + assert.equal(receipt.unresolvedAddAttempts, 0, 'no physical request of this journal'); + assert.equal(receipt.keysWithLegacyUncertainty, 2, 'u1 and g3'); + const probe = outbox.retract('pending-batch'); + assert.equal(probe.removalsQueued, 1); + assert.equal(probe.keysWithLegacyUncertainty, 1); + assert.equal(outbox.cancel('pending-batch').legacyOutcomesUnrecorded, 1); + + // A crash between the import and the rename: the hash is recorded, so the + // file is only renamed, never imported twice. + writeFileSync(h.legacyPath, JSON.stringify(legacy)); + const again = h.reopen(); assert.equal(again.batches().filter((b) => b.id === 'dd34b043').length, 1); - assert.equal(again.pendingDispatches('discord').length, 3); - again.cancel('pending-batch'); - assert.equal(existsSync(legacyPath), false, 'the next write finishes the rename'); + assert.equal(batches(again).find((v) => v.id === 'dd34b043')!.legacy!.entries, 5); + assert.equal(existsSync(h.legacyPath), false); })); +// --------------------------------------------------------------------------- +// Framework seams, with stubbed collaborators +// --------------------------------------------------------------------------- + +function storeHarness() { + const dir = mkdtempSync(join(tmpdir(), 'discord-awareness-fw-')); + const store = JsStore.openOrCreate({ path: join(dir, 'store') }); + return { + dir, + store, + outbox: new DiscordAwarenessOutbox(store), + cleanup: () => { if (!store.isClosed()) store.close(); rmSync(dir, { recursive: true, force: true }); }, + }; +} + +async function muted(fn: () => Promise): Promise { + const original = console.error; + console.error = () => {}; + try { + return await fn(); + } finally { + console.error = original; + } +} + test('framework drain classifies outcomes from what reached the server', async () => { - const dir = mkdtempSync(join(tmpdir(), 'discord-awareness-drain-')); + const h = storeHarness(); try { - const outbox = new DiscordAwarenessOutbox(join(dir, 'journal.jsonl')); - activeBatch(outbox, [ref('m1'), ref('m2'), ref('m3'), ref('m4'), ref('m5')]); + activeBatch(h.outbox, [ref('m1'), ref('m2'), ref('m3'), ref('m4'), ref('m5')]); const calls: string[] = []; let connected = true; const connection = { @@ -457,218 +491,206 @@ test('framework drain classifies outcomes from what reached the server', async ( }, }; const framework = Object.create(AgentFramework.prototype) as any; - framework.discordAwarenessOutbox = outbox; + framework.discordAwarenessOutbox = h.outbox; framework.discordAwarenessDrains = new Map(); framework.discordAwarenessDeadlineMs = 1000; framework.mcplServerRegistry = { getServer: () => connection }; - const originalError = console.error; - console.error = () => {}; - try { - await framework.drainDiscordAwarenessOutbox('discord'); - } finally { - console.error = originalError; - } - assert.deepEqual(calls, ['add_reaction:m1', 'add_reaction:m2', 'add_reaction:m3', 'add_reaction:m4', 'add_reaction:m5']); - const status = new Map(outbox.operations().map((op) => [op.key.messageId, outbox.operationStatus(op)])); + await muted(() => framework.drainDiscordAwarenessOutbox('discord')); + assert.deepEqual(calls.sort(), ['add_reaction:m1', 'add_reaction:m2', 'add_reaction:m3', 'add_reaction:m4', 'add_reaction:m5']); + const status = new Map(h.outbox.operations().map((op) => [op.key.messageId, h.outbox.operationStatus(op)])); assert.equal(status.get('m1'), 'confirmed'); assert.equal(status.get('m2'), 'failed', 'Discord said the message is gone: final'); assert.equal(status.get('m3'), 'requested', 'never written: still queued'); assert.equal(status.get('m4'), 'unknown', 'written, no answer: unknown'); - assert.equal(status.get('m5'), 'confirmed'); // A disconnected route leaves its work queued without writing anything. connected = false; - const before = readFileSync(join(dir, 'journal.jsonl'), 'utf8'); - console.error = () => {}; - try { - await framework.drainDiscordAwarenessOutbox('discord'); - } finally { - console.error = originalError; - } - assert.equal(readFileSync(join(dir, 'journal.jsonl'), 'utf8'), before); + const records = h.store.getRecordIdsByType(DISCORD_AWARENESS_RECORD_TYPE).length; + await muted(() => framework.drainDiscordAwarenessOutbox('discord')); + assert.equal(h.store.getRecordIdsByType(DISCORD_AWARENESS_RECORD_TYPE).length, records); // Back online: the unsent and the unknown (newest for their keys) go again. connected = true; calls.length = 0; - await framework.drainDiscordAwarenessOutbox('discord'); + await muted(() => framework.drainDiscordAwarenessOutbox('discord')); assert.deepEqual(calls.sort(), ['add_reaction:m3', 'add_reaction:m4']); } finally { - rmSync(dir, { recursive: true, force: true }); + h.cleanup(); } }); test('framework resumes an interrupted suppression before activating its marks', async () => { - const dir = mkdtempSync(join(tmpdir(), 'discord-awareness-resume-')); + const h = storeHarness(); try { - const outbox = new DiscordAwarenessOutbox(join(dir, 'journal.jsonl')); - const batch = outbox.prepare({ - agentName: 'cairn', - sourceBranch: 'main', - targetBranch: 'recovery/cairn/suppressed', - activationPolicy: 'explicit', - suppressionIntervals: [{ fromId: 'i1', toId: 'i2' }, { fromId: 'i3', toId: 'i3' }], - refs: [ref('m1')], - scope: 'all', + const batch = h.outbox.prepare({ + agentName: 'cairn', sourceBranch: 'main', targetBranch: 'recovery/cairn/suppressed', activationPolicy: 'explicit', + suppressionIntervals: [{ fromId: 'i1', toId: 'i2' }, { fromId: 'i3', toId: 'i3' }], refs: [ref('m1')], scope: 'all', })!; const present = new Set(['i1', 'i2']); // i3 interval committed before crash. const removals: string[] = []; const cm = { getMessage: (id: string) => present.has(id) ? { id } : null, removeMessage: (id: string) => { present.delete(id); removals.push(id); }, - removeMessages: (from: string, to: string) => { - present.delete(from); present.delete(to); removals.push(`${from}..${to}`); - }, + removeMessages: (from: string, to: string) => { present.delete(from); present.delete(to); removals.push(`${from}..${to}`); }, }; const framework = Object.create(AgentFramework.prototype) as any; - framework.discordAwarenessOutbox = outbox; + framework.discordAwarenessOutbox = h.outbox; framework.store = { currentBranch: () => ({ name: 'recovery/cairn/suppressed' }) }; framework.agents = new Map([['cairn', { getContextManager: () => cm }]]); - - await framework.resumePreparedDiscordSuppressions(); - + await muted(() => framework.resumePreparedDiscordSuppressions()); assert.deepEqual(removals, ['i1..i2']); - assert.equal(outbox.batches().find((candidate) => candidate.id === batch.id)?.status, 'active'); - assert.deepEqual(outbox.pendingDispatches('discord').map((d) => d.key.messageId), ['m1']); + assert.equal(h.outbox.batches().find((candidate) => candidate.id === batch.id)?.status, 'active'); + assert.deepEqual(h.outbox.pendingDispatches('discord').map((d) => d.key.messageId), ['m1']); } finally { - rmSync(dir, { recursive: true, force: true }); + h.cleanup(); } }); -test('message-granular host/command undo places no marks unless the command asks', async () => { - const dir = mkdtempSync(join(tmpdir(), 'discord-awareness-undo-')); +/** A framework stub whose agent context is a plain list (ids i0..), for host/command. */ +function hostCommandFramework(h: ReturnType, messages: Array<{ id: string; metadata: Record }>) { + let currentBranch = 'main'; + const removed = new Set(); + const live = () => messages.filter((m) => !removed.has(m.id)); + const contextManager = { + getAllMessages: () => live(), + getMessageCount: () => live().length, + getMessageWindow: (offset: number, limit: number) => ({ + messages: live().slice(offset, offset + limit), + startIndex: offset, + totalCount: live().length, + }), + branchAt: (_id: string, name: string) => name, + switchBranch: async (name: string) => { currentBranch = name; }, + removeMessage: (id: string) => { removed.add(id); }, + removeMessages: (from: string, to: string) => { + const ids = messages.map((m) => m.id); + for (const id of ids.slice(ids.indexOf(from), ids.indexOf(to) + 1)) removed.add(id); + }, + }; + const framework = Object.create(AgentFramework.prototype) as any; + framework.agents = new Map([['cairn', { state: { status: 'idle' }, getContextManager: () => contextManager }]]); + framework.store = { currentBranch: () => ({ name: currentBranch }), sync: () => h.store.sync() }; + framework.discordAwarenessOutbox = h.outbox; + framework.discordAwarenessEmoji = '🫥'; + framework.discordAwarenessDrains = new Map(); + framework.mcplServerRegistry = null; + framework.moduleRegistry = { getModule: () => null }; + framework.lastVisiblePreview = async () => null; + framework.activeTurnTokens = new Map(); + framework.nextTurnToken = 1; + framework.deferredMessages = []; + framework.pendingAssistantBlocks = new Map(); + framework.operatorLog = new OperatorLog(undefined); + framework.opsAlert = () => {}; + return { framework, removed, setBranch: (name: string) => { currentBranch = name; } }; +} + +test('message-granular host/command undo places no marks unless asked; the marks verb reaches the controls', async () => { + const h = storeHarness(); try { - const messages = [ + const { framework, setBranch } = hostCommandFramework(h, [ { id: 'i0', metadata: { ...ref('m0') } }, { id: 'i1', metadata: { ...ref('m1'), tags: ['chat:addressed'] } }, { id: 'i2', metadata: { ...ref('m2'), tags: ['chat:ambient'] } }, { id: 'i3', metadata: { ...ref('m3') } }, - ]; - let currentBranch = 'main'; - const contextManager = { - getAllMessages: () => messages, - getMessageCount: () => messages.length, - getMessageWindow: (offset: number, limit: number) => ({ - messages: messages.slice(offset, offset + limit), - startIndex: offset, - totalCount: messages.length, - }), - branchAt: (_id: string, name: string) => name, - switchBranch: async (name: string) => { currentBranch = name; }, - }; - const framework = Object.create(AgentFramework.prototype) as any; - framework.agents = new Map([['cairn', { state: { status: 'idle' }, getContextManager: () => contextManager }]]); - framework.store = { currentBranch: () => ({ name: currentBranch }) }; - framework.discordAwarenessOutbox = new DiscordAwarenessOutbox(join(dir, 'journal.jsonl')); - framework.discordAwarenessEmoji = '🫥'; - framework.discordAwarenessDrains = new Map(); - framework.mcplServerRegistry = null; - framework.moduleRegistry = { getModule: () => null }; - framework.lastVisiblePreview = async () => null; - framework.activeTurnTokens = new Map(); - framework.nextTurnToken = 1; - framework.deferredMessages = []; - framework.pendingAssistantBlocks = new Map(); - framework.operatorLog = new OperatorLog(undefined); - const originalError = console.error; - console.error = () => {}; - try { + ]); + await muted(async () => { const silent = await framework.handleHostCommand('discord', { command: 'undo', agentName: 'cairn', messages: 1 }); assert.equal(silent.ok, true); assert.deepEqual(silent.markers, { scope: 'none', unmarked: 1, notRemoved: 0, status: 'none', queued: 0 }); - assert.equal(framework.discordAwarenessOutbox.pendingDispatches('discord').length, 0); + assert.equal(h.outbox.pendingDispatches('discord').length, 0); - currentBranch = 'main'; - const marked = await framework.handleHostCommand('discord', { - command: 'undo', agentName: 'cairn', messages: 3, marks: 'addressed', - }); + setBranch('main'); + const marked = await framework.handleHostCommand('discord', { command: 'undo', agentName: 'cairn', messages: 3, marks: 'addressed' }); assert.equal(marked.markers.status, 'queued'); assert.equal(marked.markers.queued, 1); assert.equal(marked.markers.unmarked, 2); - const dispatches = framework.discordAwarenessOutbox.pendingDispatches('discord'); - assert.deepEqual(dispatches.map((d: { key: { messageId: string; emoji: string } }) => `${d.key.messageId}${d.key.emoji}`), ['m1🫥']); + assert.deepEqual(h.outbox.pendingDispatches('discord').map((d) => `${d.key.messageId}${d.key.emoji}`), ['m1🫥']); - // A mistyped scope is refused, never guessed. - currentBranch = 'main'; - const typo = await framework.handleHostCommand('discord', { - command: 'undo', agentName: 'cairn', messages: 1, marks: 'adressed', - }); + const typo = await framework.handleHostCommand('discord', { command: 'undo', agentName: 'cairn', messages: 1, marks: 'adressed' }); assert.equal(typo.ok, false); assert.match(typo.error, /marks must be none, addressed or all/); - // The journal's controls are reachable through host/command too. const listed = await framework.handleHostCommand('discord', { command: 'marks', action: 'list' }); assert.equal(listed.ok, true); - assert.equal(listed.awareness.length, 1); - const cancelled = await framework.handleHostCommand('discord', { - command: 'marks', action: 'cancel', batchId: listed.awareness[0].id, requesterName: 'op', - }); + const batch = listed.awareness.find((v: { kind: string }) => v.kind === 'batch'); + const cancelled = await framework.handleHostCommand('discord', { command: 'marks', action: 'cancel', target: batch.id, requesterName: 'op' }); assert.equal(cancelled.ok, true); assert.equal(cancelled.awareness.cancelled, 1); - assert.equal(framework.discordAwarenessOutbox.pendingDispatches('discord').length, 0); + assert.equal(h.outbox.pendingDispatches('discord').length, 0); const missing = await framework.handleHostCommand('discord', { command: 'marks', action: 'retract' }); assert.equal(missing.ok, false); - } finally { - console.error = originalError; - } + }); } finally { - rmSync(dir, { recursive: true, force: true }); + h.cleanup(); } }); -test('host/command hide marks only on request, through the journal, routed by the issuing server for old records', async () => { - const dir = mkdtempSync(join(tmpdir(), 'discord-awareness-hide-')); +test('turn-based host/command undo honours the marks choice for the messages its turns removed', async () => { + const h = storeHarness(); try { const messages = [ + { id: 'i0', metadata: { ...ref('m0') } }, + { id: 'i1', metadata: { ...ref('m1'), tags: ['chat:addressed'] } }, // arrived during the turn + { id: 'i2', metadata: {} }, // the agent's reply + ]; + const { framework, removed } = hostCommandFramework(h, messages); + let turns = 1; + framework.undoLastTurn = () => { + if (turns === 0) return { undone: false }; + turns--; + removed.add('i1'); + removed.add('i2'); + return { undone: true }; + }; + await muted(async () => { + const result = await framework.handleHostCommand('discord', { command: 'undo', agentName: 'cairn', turns: 2, marks: 'addressed' }); + assert.equal(result.ok, true); + assert.equal(result.undone, 1); + assert.equal(result.markers.status, 'queued'); + assert.equal(result.markers.queued, 1); + assert.deepEqual(h.outbox.pendingDispatches('discord').map((d) => d.key.messageId), ['m1']); + }); + } finally { + h.cleanup(); + } +}); + +test('host/command hide takes the store reservation and marks only on request, through the journal', async () => { + const h = storeHarness(); + try { + const { framework } = hostCommandFramework(h, [ { id: 'i0', metadata: { ...ref('m0') } }, { id: 'i1', metadata: { channelId: 'discord:g1:c1', messageId: 'm1', tags: ['chat:addressed'] } }, // pre-serverId record { id: 'i2', metadata: { ...ref('m2'), tags: ['chat:ambient'] } }, { id: 'i3', metadata: { ...ref('m3') } }, - ]; - const removed: string[] = []; - const contextManager = { - getAllMessages: () => messages.filter((m) => !removed.includes(m.id)), - removeMessage: (id: string) => { removed.push(id); }, - removeMessages: (from: string, to: string) => { - const ids = messages.map((m) => m.id); - for (const id of ids.slice(ids.indexOf(from), ids.indexOf(to) + 1)) removed.push(id); - }, - }; - const framework = Object.create(AgentFramework.prototype) as any; - framework.agents = new Map([['cairn', { state: { status: 'idle' }, getContextManager: () => contextManager }]]); - framework.store = { currentBranch: () => ({ name: 'main' }) }; - framework.discordAwarenessOutbox = new DiscordAwarenessOutbox(join(dir, 'journal.jsonl')); - framework.discordAwarenessEmoji = '💤'; - framework.discordAwarenessDrains = new Map(); - framework.mcplServerRegistry = null; - framework.lastVisiblePreview = async () => null; - framework.operatorLog = new OperatorLog(undefined); - const originalError = console.error; - console.error = () => {}; - try { + ]); + await muted(async () => { + // Another agent sharing the store is mid-turn: hide refuses, removes nothing. + framework.activeTurnTokens.set('other', 99); + const busy = await framework.handleHostCommand('discord', { command: 'hide', agentName: 'cairn', fromMessageId: 'm3' }); + assert.equal(busy.ok, false); + assert.equal(busy.code, 'agent-busy'); + framework.activeTurnTokens.delete('other'); + const silent = await framework.handleHostCommand('discord', { command: 'hide', agentName: 'cairn', fromMessageId: 'm3' }); assert.equal(silent.ok, true); assert.equal(silent.markers.status, 'none'); assert.equal(silent.markers.unmarked, 1); - assert.equal(framework.discordAwarenessOutbox.batches().length, 0); + assert.equal(h.outbox.batches().length, 0); + assert.equal(framework.surgeryHold, null, 'the reservation was released'); const marked = await framework.handleHostCommand('discord', { command: 'hide', agentName: 'cairn', fromMessageId: 'm1', toMessageId: 'm2', marks: 'addressed', }); - assert.equal(marked.ok, true); assert.equal(marked.hidden, 2); assert.equal(marked.markers.status, 'queued'); assert.equal(marked.markers.queued, 1); - assert.equal(marked.markers.unmarked, 1); - const [dispatch] = framework.discordAwarenessOutbox.pendingDispatches('discord'); - assert.deepEqual(dispatch.key, { serverId: 'discord', channelId: 'discord:g1:c1', messageId: 'm1', emoji: '💤' }); - // Explicit and already applied: a restart on this branch changes nothing. - const restarted = new DiscordAwarenessOutbox(join(dir, 'journal.jsonl')); - assert.deepEqual(restarted.recoverAtStartup('main'), { activated: [], held: [], unknownAttempts: 0 }); - assert.equal(restarted.batches()[0].status, 'active'); - } finally { - console.error = originalError; - } + const [dispatch] = h.outbox.pendingDispatches('discord'); + assert.deepEqual(dispatch.key, { serverId: 'discord', channelId: 'discord:g1:c1', messageId: 'm1', emoji: '🫥' }); + assert.deepEqual(new DiscordAwarenessOutbox(h.store).recoverAtStartup('main'), { activated: [], held: [], unknownAttempts: 0 }); + }); } finally { - rmSync(dir, { recursive: true, force: true }); + h.cleanup(); } }); diff --git a/test/offline-recovery-branch.test.ts b/test/offline-recovery-branch.test.ts index 8b34e9aff..0ee20e99f 100644 --- a/test/offline-recovery-branch.test.ts +++ b/test/offline-recovery-branch.test.ts @@ -7,10 +7,20 @@ import { JsStore } from '@animalabs/chronicle'; import { ContextManager } from '@animalabs/context-manager'; import { createOfflineRecoveryBranch } from '../src/recovery/offline-branch.js'; import { + DISCORD_AWARENESS_RECORD_TYPE, DiscordAwarenessOutbox, - defaultDiscordAwarenessOutboxPath, } from '../src/recovery/discord-awareness-outbox.js'; +/** Read the store's awareness journal with the host (and CLI) stopped. */ +function readJournal(storePath: string, read: (outbox: DiscordAwarenessOutbox, store: JsStore) => T): T { + const store = JsStore.openOrCreate({ path: storePath }); + try { + return read(new DiscordAwarenessOutbox(store), store); + } finally { + store.close(); + } +} + test('offline recovery branches without compiling and queues discarded Discord refs', async () => { const dir = mkdtempSync(join(tmpdir(), 'offline-recovery-')); const storePath = join(dir, 'agent.chronicle'); @@ -53,17 +63,17 @@ test('offline recovery branches without compiling and queues discarded Discord r recoveredCm.close(); recoveredStore.close(); - const batches = new DiscordAwarenessOutbox( - defaultDiscordAwarenessOutboxPath(storePath), - ).pendingDispatches('discord'); + const batches = readJournal(storePath, (outbox) => outbox.pendingDispatches('discord')); assert.equal(batches.length, 2); assert.deepEqual(batches.map((dispatch) => dispatch.key.messageId), ['m-toxic-1', 'm-toxic-2']); // The outbox is metadata-only: quarantined text must never leak to it. - const rawOutbox = await import('node:fs').then(({ readFileSync }) => - readFileSync(defaultDiscordAwarenessOutboxPath(storePath), 'utf8')); - assert.ok(!rawOutbox.includes('never echo this one')); - assert.ok(!rawOutbox.includes('or this one')); + const rawJournal = readJournal(storePath, (_outbox, store) => store.getRecordIdsByType(DISCORD_AWARENESS_RECORD_TYPE) + .map((id) => Buffer.from(store.getRecord(id)!.payload).toString('utf-8')) + .join('\n')); + assert.ok(rawJournal.length > 0); + assert.ok(!rawJournal.includes('never echo this one')); + assert.ok(!rawJournal.includes('or this one')); } finally { rmSync(dir, { recursive: true, force: true }); } @@ -180,9 +190,7 @@ test('offline recovery suppresses inclusive ranges in context order', async () = recoveredCm.close(); recoveredStore.close(); - const batches = new DiscordAwarenessOutbox( - defaultDiscordAwarenessOutboxPath(storePath), - ).pendingDispatches('discord'); + const batches = readJournal(storePath, (outbox) => outbox.pendingDispatches('discord')); assert.equal(batches.length, 3); assert.deepEqual( batches.map((dispatch) => dispatch.key.messageId), @@ -300,9 +308,11 @@ test('offline recovery is local by default; addressed marks only messages that a assert.equal(local.discordAddressable, 2); assert.equal(local.discordMarkersQueued, 0); assert.equal(local.unmarked, 2); - const outbox = new DiscordAwarenessOutbox(defaultDiscordAwarenessOutboxPath(storePath)); - assert.equal(outbox.pendingDispatches('discord').length, 0, 'nothing is queued for Discord'); - assert.equal(outbox.batches().length, 0); + assert.deepEqual(local.markers, { scope: 'none', unmarked: 2, notRemoved: 0, status: 'none', queued: 0 }); + readJournal(storePath, (outbox) => { + assert.equal(outbox.pendingDispatches('discord').length, 0, 'nothing is queued for Discord'); + assert.equal(outbox.batches().length, 0); + }); // A dry run with addressed shows exactly what would be marked. rmSync(storePath, { recursive: true, force: true }); @@ -316,12 +326,49 @@ test('offline recovery is local by default; addressed marks only messages that a }); assert.equal(marked.discordMarkersQueued, 1); assert.equal(marked.unmarked, 1); + assert.equal(marked.markers?.status, 'queued'); assert.deepEqual( - new DiscordAwarenessOutbox(defaultDiscordAwarenessOutboxPath(storePath)).pendingDispatches('discord') - .map((dispatch) => dispatch.key.messageId), + readJournal(storePath, (outbox) => outbox.pendingDispatches('discord').map((dispatch) => dispatch.key.messageId)), ['m-addressed'], ); } finally { rmSync(dir, { recursive: true, force: true }); } }); + +test('offline recovery reports a marker bookkeeping failure apart from the body, which stands', async () => { + const dir = mkdtempSync(join(tmpdir(), 'offline-recovery-settle-')); + const storePath = join(dir, 'agent.chronicle'); + try { + const store = JsStore.openOrCreate({ path: storePath }); + const cm = await ContextManager.open({ store, namespace: 'agents/cairn' }); + cm.addMessage('user', [{ type: 'text', text: 'safe' }], { serverId: 'discord', channelId: 'discord:g1:c1', messageId: 'm-safe' }); + cm.addMessage('user', [{ type: 'text', text: 'hey' }], { serverId: 'discord', channelId: 'discord:g1:c1', messageId: 'm-gone', tags: ['chat:addressed'] }); + cm.close(); + store.close(); + const original = DiscordAwarenessOutbox.prototype.activate; + DiscordAwarenessOutbox.prototype.activate = function () { throw new Error('injected activation failure'); }; + let result: Awaited>; + try { + result = await createOfflineRecoveryBranch({ + storePath, agentName: 'cairn', messageId: 'm-safe', marks: { scope: 'all' }, branchName: 'recovery/cairn/settle', + }); + } finally { + DiscordAwarenessOutbox.prototype.activate = original; + } + assert.equal(result.markers?.status, 'not-scheduled'); + assert.match(result.markers?.status === 'not-scheduled' ? result.markers.error : '', /injected activation failure/); + assert.equal(result.discordMarkersQueued, 0); + const reopened = JsStore.openOrCreate({ path: storePath }); + try { + assert.equal(reopened.currentBranch().name, 'recovery/cairn/settle', 'the recovery branch stands'); + const outbox = new DiscordAwarenessOutbox(reopened); + assert.equal(outbox.batches()[0].status, 'discarded'); + assert.equal(outbox.pendingDispatches('discord').length, 0); + } finally { + reopened.close(); + } + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); diff --git a/test/surgery-marks-receipt.test.ts b/test/surgery-marks-receipt.test.ts index 674d08f90..523782f29 100644 --- a/test/surgery-marks-receipt.test.ts +++ b/test/surgery-marks-receipt.test.ts @@ -70,7 +70,8 @@ describe('surgery and awareness marks', () => { let framework: AgentFramework; const cm = () => framework.getAgent('resident')!.getContextManager(); - const outbox = () => new DiscordAwarenessOutbox(join(storePath, 'recovery', 'discord-awareness-journal.jsonl')); + /** The running framework's own journal instance (the journal's one writer). */ + const outbox = () => (framework as unknown as { discordAwarenessOutbox: DiscordAwarenessOutbox }).discordAwarenessOutbox; const answered = () => jsonl(eventsPath).filter((e) => e.event === 'reaction-answered').length; const calls = () => jsonl<{ name: string; args: { messageId: string; emoji: string } }>(callsPath); @@ -347,8 +348,7 @@ describe('surgery and awareness marks', () => { assert.ok(kinds.includes('awareness-cancel') && kinds.includes('awareness-retract')); }); - it('marks chosen where no awareness journal exists are reported as not scheduled, not as unwanted', async () => { - // A caller-supplied store and no storePath: no default journal location. + it('a caller-supplied store keeps its awareness journal in the store too', async () => { framework = await AgentFramework.create({ store: JsStore.openOrCreate({ path: storePath }), membrane: new MockMembrane().asMembrane(), @@ -359,12 +359,109 @@ describe('surgery and awareness marks', () => { }); const { tail } = seed(2, [0]); const result = await framework.rollbackToMessage('resident', { messageId: tail, marks: { scope: 'addressed' } }); - assert.equal(result.messagesRemoved, 2); - assert.equal(result.markers.status, 'not-scheduled'); - assert.match(result.markers.status === 'not-scheduled' ? result.markers.error : '', /no awareness journal/); + assert.equal(result.markers.status, 'queued'); + assert.equal(result.markers.queued, 1); assert.equal(result.markers.unmarked, 1); - const local = await framework.rollbackToMessage('resident', { messageId: String(cm().getAllMessages()[0].id) }); - assert.equal(local.markers.status, 'none', 'nothing chosen is still none'); + }); + + it('cancel during delivery stops the marks not yet sent: exactly one request reaches the server', async () => { + await start(true); + const { tail } = seed(3); + const result = await framework.rollbackToMessage('resident', { messageId: tail, marks: { scope: 'all' } }); + const batchId = result.markers.status === 'queued' ? result.markers.batchId : ''; + await waitFor('the first add on the wire', () => calls().length === 1); + const receipt = framework.cancelDiscordAwareness(batchId); + assert.deepEqual({ cancelled: receipt.cancelled, inFlight: receipt.inFlight }, { cancelled: 2, inFlight: 1 }); + writeFileSync(holdPath, '1'); + await waitFor('the held add answered and recorded', () => + outbox().operations().some((op) => op.attempts[0]?.outcome === 'confirmed')); + await framework.syncDiscordAwarenessMarkers(); + await new Promise((resolve) => setTimeout(resolve, 100)); + assert.equal(calls().length, 1, 'the cancelled marks were never sent'); + }); + + it('a failed rollback keeps its batch when the source cannot be confirmed, and says so', async () => { + await start(false); + const { tail } = seed(2); + const live = cm(); + const realSwitch = live.switchBranch.bind(live); + // The switch lands, then fails; the restore does nothing: the store is + // left on the fork, so the batch is the unfinished change's record. + live.switchBranch = async (name: string) => { await realSwitch(name); throw new Error('injected switch failure'); }; + const realRestore = (framework as any).restoreSourceBranch; + (framework as any).restoreSourceBranch = async () => 'source not restored (injected)'; + try { + await assert.rejects( + framework.rollbackToMessage('resident', { messageId: tail, marks: { scope: 'all' } }), + /awareness batch .* kept, since the source branch could not be confirmed/, + ); + } finally { + live.switchBranch = realSwitch; + (framework as any).restoreSourceBranch = realRestore; + } + const [batch] = outbox().batches(); + assert.equal(batch.status, 'prepared', 'not retired while the store sits on the fork'); + }); + + it('a failed suppression retires its batch only after the source is back, and reports a failed retire', async () => { + await start(false); + const { removed } = seed(3); + const live = cm(); + const realFork = live.fork.bind(live); + live.fork = async () => { throw new Error('injected fork failure'); }; + const discard = DiscordAwarenessOutbox.prototype.discard; + DiscordAwarenessOutbox.prototype.discard = function () { throw new Error('injected retire failure'); }; + try { + await assert.rejects( + framework.suppressMessages('resident', { messageIds: [removed[0]], marks: { scope: 'all' } }), + /could not be retired \(injected retire failure\)/, + ); + } finally { + live.fork = realFork; + DiscordAwarenessOutbox.prototype.discard = discard; + } + assert.equal(outbox().batches()[0].status, 'prepared'); + // With retirement working, the same failure retires it (source confirmed). + live.fork = async () => { throw new Error('injected fork failure'); }; + try { + await assert.rejects(framework.suppressMessages('resident', { messageIds: [removed[1]], marks: { scope: 'all' } })); + } finally { + live.fork = realFork; + } + assert.equal(outbox().batches().at(-1)!.status, 'discarded'); + }); + + it('an interrupted suppression: held at startup, its marks released, then its body resumed on its branch', async () => { + await start(false); + const { removed } = seed(3); + const live = cm(); + const sourceBranch = live.currentBranch().name; + // An interrupted suppression: the fork exists, nothing was redacted yet. + await live.fork('partial'); + const [target] = live.getAllMessages().filter((m) => String(m.id) === removed[1]); + const batch = outbox().prepare({ + agentName: 'resident', sourceBranch, targetBranch: 'partial', + refs: [{ serverId: 'discord', channelId: 'discord:g1:c1', messageId: 'amb-1' }], scope: 'all', + activationPolicy: 'explicit', suppressionIntervals: [{ fromId: String(target.id), toId: String(target.id) }], + })!; + await live.switchBranch(sourceBranch); + await framework.stop(); + + await start(false); // on the source branch: the batch is held + assert.equal(outbox().batches()[0].status, 'held'); + const released = framework.releaseDiscordAwareness(batch.id); + assert.equal(released.addsQueued, 1); + await framework.stop(); + + const store = JsStore.openOrCreate({ path: storePath }); + store.switchBranch('partial'); + store.close(); + await start(false); // on the target: the body is completed, the marks untouched + assert.ok(!cm().getAllMessages().some((m) => String(m.id) === removed[1]), 'the saved redaction landed'); + const [after] = outbox().batches(); + assert.equal(after.suppressionComplete, true); + assert.equal(after.status, 'active'); + assert.equal(outbox().operations().filter((op) => op.action === 'add').length, 1, 'only the release queued marks'); }); it('a surgery that removes nothing addressable schedules no marks even when marks are chosen', async () => { From c0d2d902da4458821082693446041d329e84694a Mon Sep 17 00:00:00 2001 From: Ada-1017 Date: Wed, 7 Oct 2026 10:01:42 -0400 Subject: [PATCH 16/73] fix(awareness): a newer opposite intent retires in-flight and unknown requests' retries; drains stop with their route June #45033 (probe /tmp/june-1016-799-opposite-probe.ts), two lifecycle gaps at 799a16b: - an add on the wire when a retract arrived was not superseded (only `requested` ops were), so its later retryable failure put it back ahead of the remove; - an unknown add superseded by nothing (unknown ops were left alone), so cancelling the retract made it the newest live op again and revived its retries. requestRecords now retires the future retries of every opposite-action op that has not ended (requested, on the wire, or unknown): op-cancelled 'superseded'. Attempts stay as history; unresolved ones are still disclosed (keysWithUnresolvedAdds); the all-ops in-flight fence still makes the key wait for the request on the wire; a confirmed late answer still reads as confirmed. Cancelling the newer request stops it without reviving anything. Shutdown churn (found re-reading stop()): after closeAll, a running drain claimed every remaining op once, writing a durable dispatching record and a not-sent outcome for each (one store sync apiece) while the host stopped. The drain now stops claiming when its connection is gone (work stays queued; reconnect starts a new drain), and stop() awaits running drains after closing MCPL so the in-flight outcome lands before the store closes. Tests: in-flight add -> retract -> retryable failure -> next claim is the remove; unknown add -> retract -> cancel retract -> nothing due; the mirror (remove in flight, new add, retryable failure -> add); all across a restart; a drain whose route goes away writes only the in-flight attempt. Awareness set + record-journal 78/78. --- src/framework.ts | 8 +++ src/recovery/discord-awareness-outbox.ts | 16 +++--- test/discord-awareness-outbox.test.ts | 64 +++++++++++++++++++++++- 3 files changed, 80 insertions(+), 8 deletions(-) diff --git a/src/framework.ts b/src/framework.ts index fb58eea51..fe44344bf 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -2028,6 +2028,10 @@ export class AgentFramework { shutdownPromises.push(this.mcplServerRegistry.closeAll()); } await Promise.all(shutdownPromises); + // An awareness drain still recording the outcome of the request that was + // on the wire when its connection closed finishes before the store does; + // it claims nothing more once the route is gone. + await Promise.allSettled([...this.discordAwarenessDrains.values()]); // Streams may queue storage repairs while being cancelled above. Retry // after their teardown, while the store is still open, before final sync. @@ -13715,6 +13719,10 @@ export class AgentFramework { `${dispatch.opIds.join(',')}\0${dispatch.action}`; let announced = false; while (true) { + // A route that went away (shutdown, or a disconnect that will + // reconnect) leaves the rest queued: claiming would only write + // attempts that cannot be sent. + if (!connection.isConnected) break; // One dispatch at a time, chosen and written ahead in one step from // the journal as it is now: a cancel or retract made while the // previous request was on the wire takes effect here. diff --git a/src/recovery/discord-awareness-outbox.ts b/src/recovery/discord-awareness-outbox.ts index f7e6592d7..167ce380e 100644 --- a/src/recovery/discord-awareness-outbox.ts +++ b/src/recovery/discord-awareness-outbox.ts @@ -1100,11 +1100,12 @@ export class DiscordAwarenessOutbox { } /** - * Records for new requests. A request supersedes the future of every - * request for the same key with the opposite action that is still due - * (never sent, refused before writing, or failed retryably): the newest - * explicit intent wins. Their attempts stay as history, and a request on - * the wire still settles before anything else for that key is sent. + * Records for new requests. A request retires the future retries of every + * request for the same key with the opposite action that has not ended + * (due, on the wire, or unknown): the newest explicit intent wins, and + * cancelling it later never revives what it superseded. Their attempts + * stay as history (an unresolved one is still disclosed), and a request + * on the wire still settles before anything else for that key is sent. */ private requestRecords( state: JournalState, @@ -1123,8 +1124,9 @@ export class DiscordAwarenessOutbox { emoji: request.emoji ?? emoji ?? DEFAULT_DISCORD_AWARENESS_EMOJI, }; for (const op of this.opsForKey(state, key)) { - if (op.action === request.action || superseded.has(op.opId)) continue; - if (this.opStatus(op) !== 'requested') continue; + if (op.action === request.action || superseded.has(op.opId) || op.cancelled) continue; + const status = this.opStatus(op); + if (status !== 'requested' && status !== 'dispatching' && status !== 'unknown') continue; records.push({ t: 'op-cancelled', at, opId: op.opId, reason: 'superseded' }); superseded.add(op.opId); } diff --git a/test/discord-awareness-outbox.test.ts b/test/discord-awareness-outbox.test.ts index c3b4eb532..d0d46fab6 100644 --- a/test/discord-awareness-outbox.test.ts +++ b/test/discord-awareness-outbox.test.ts @@ -261,7 +261,7 @@ test('retract queues a removal for every selected key, whatever history says, an const receipt = outbox.retract(a.id, 'operator'); assert.equal(receipt.removalsQueued, 3, 'm2 too: history does not decide whether the act is carried out'); - assert.equal(receipt.addsSuperseded, 1, "m2's due add stops"); + assert.equal(receipt.addsSuperseded, 3, "m2's due add stops, and m1's unknown ones are no longer retried"); assert.equal(receipt.keysWithUnresolvedAdds, 1); assert.equal(receipt.unresolvedAddAttempts, 1, 'one physical request carried both adds'); assert.equal(receipt.keysWithLegacyUncertainty, 0); @@ -296,6 +296,39 @@ test('a newer opposite request stops the retries of a retryably failed one, in b assert.equal(outbox.operations().find((op) => op.action === 'remove')!.cancelled?.reason, 'superseded'); })); +test('a newer opposite intent retires the old request\'s retries even when it was on the wire or unknown', withJournal((outbox, h) => { + // In flight when the retract arrives, then a retryable failure: remove next. + const a = activeBatch(outbox, [ref('m1')]); + const onWire = outbox.claimDispatch('discord')!; + const retract = outbox.retract(a.id); + assert.equal(retract.addsSuperseded, 1); + assert.equal(retract.keysWithUnresolvedAdds, 1, 'the request on the wire is disclosed'); + assert.equal(outbox.claimDispatch('discord'), null, 'the key waits for the request on the wire'); + outbox.recordOutcome(onWire.attempts, 'failed', { error: 'Discord 503' }); + assert.equal(outbox.claimDispatch('discord', () => false)!.dispatch.action, 'remove'); + // Unknown, then retract, then the retract cancelled: nothing is revived. + const b = activeBatch(outbox, [ref('m2')]); + answer(outbox, 'm2', 'unknown'); + const { requestId } = outbox.retract(b.id); + outbox.cancel(requestId); + assert.ok(!outbox.pendingDispatches('discord').some((d) => d.key.messageId === 'm2')); + // Mirror: a remove in flight when a new add arrives, then a retryable failure. + const c = activeBatch(outbox, [ref('m3')]); + answer(outbox, 'm3', 'confirmed'); + outbox.retract(c.id); + const removeOnWire = outbox.claimDispatch('discord', (d) => d.key.messageId !== 'm3')!; + activeBatch(outbox, [ref('m3')]); + outbox.recordOutcome(removeOnWire.attempts, 'failed', { error: 'Discord 503' }); + const next = outbox.claimDispatch('discord', (d) => d.key.messageId !== 'm3')!; + assert.equal(next.dispatch.action, 'add'); + outbox.recordOutcome(next.attempts, 'confirmed'); + // All of it holds across a restart. + const restarted = h.reopen(); + restarted.recoverAtStartup('rollback/cairn/1'); + const due = restarted.pendingDispatches('discord').map((d) => `${d.action}:${d.key.messageId}`); + assert.ok(!due.includes('add:m1') && !due.includes('add:m2') && !due.includes('remove:m3'), due.join(',')); +})); + test('a retract request can be cancelled by the id its receipt returned, even after a restart', withJournal((outbox, h) => { const batch = activeBatch(outbox, [ref('m1'), ref('m2'), ref('m3')]); for (const id of ['m1', 'm2', 'm3']) answer(outbox, id, 'confirmed'); @@ -519,6 +552,35 @@ test('framework drain classifies outcomes from what reached the server', async ( } }); +test('a drain whose route goes away stops claiming: the rest stays queued, with no attempts written', async () => { + const h = storeHarness(); + try { + activeBatch(h.outbox, [ref('m1'), ref('m2'), ref('m3')]); + let connected = true; + const calls: string[] = []; + const connection = { + get isConnected() { return connected; }, + sendToolsCallWithDeadline: async (_name: string, args: Record) => { + calls.push(String(args.messageId)); + connected = false; // the host is shutting down + throw new McplRequestError('Connection to MCPL server "discord" closed while awaiting response', 'no-response'); + }, + }; + const framework = Object.create(AgentFramework.prototype) as any; + framework.discordAwarenessOutbox = h.outbox; + framework.discordAwarenessDrains = new Map(); + framework.discordAwarenessDeadlineMs = 1000; + framework.mcplServerRegistry = { getServer: () => connection }; + await muted(() => framework.drainDiscordAwarenessOutbox('discord')); + assert.equal(calls.length, 1); + const attempts = h.outbox.operations().reduce((sum, op) => sum + op.attempts.length, 0); + assert.equal(attempts, 1, 'only the request that was on the wire'); + assert.equal(h.outbox.pendingDispatches('discord').length, 3, 'its unknown and the two unsent stay due'); + } finally { + h.cleanup(); + } +}); + test('framework resumes an interrupted suppression before activating its marks', async () => { const h = storeHarness(); try { From d44826bdf6faa86b7a3636770dbfe0a268d7f3aa Mon Sep 17 00:00:00 2001 From: Ada-1017 Date: Wed, 7 Oct 2026 10:26:05 -0400 Subject: [PATCH 17/73] fix(awareness): authorization order decides opposite requests; imported active is not body proof; startup and offline body barriers; reserved turn undo June #45105 (probe /tmp/june-1016-799-prepared-retract-probe.ts): prepare -> retract -> activate let activation's adds cancel the later retract's remove. Activation completes the surgery's earlier choice; it is not a new one. Every op now carries `authorization`, the journal position (state.position, one per applied record, checkpointed) of the act it carries out: a batch's prepare for its activation's adds (activate and startup crash completion), the retract record for its removals, the release record for a release's requests. requestRecords takes the authorization (default: a new act, later than everything): an op with an opposite op authorized after it, in any state, even cancelled, is recorded superseded at birth (cancelling the later act never revives it); otherwise it retires older-authorized opposite ops not yet ended. retract('all') now covers every key of every batch not discarded (prepared, held, active). activate()/settleActivation report the adds actually queued. A release stays a new act (wins over a retract made while held). The body obligation is untouched: a suppression whose marks were retracted before activation still records suppressionComplete on activation. June #45108 (probe /tmp/june-1016-799-legacy-body-probe.ts): importLegacy emitted `activated` for v2 active batches, which certified a suppression body the old JSONL/Chronicle window could have lost. It now emits `legacy-activated` (marks history only); the resume selector picks the imported explicit suppression on its exact target, resumes intervals idempotently, and records suppression-complete after the sync. June #45172: - recoverAtStartup's crash-completion activations append with afterCommittedState (sync first; a failed sync appends nothing), as the store may be caller-supplied and already open. - offline body failure: restore the source, then retire the batch (discard, after-sync) once the source is confirmed; if not confirmed, keep it prepared and say so; a failed retire is reported. The error composes failure; restore failure; retire clause, with the original as cause. June room-203 #45217 / Agnes #45253 split: turn-based host/command undo takes reserveStoreForSurgery around the whole act (before snapshot, the undoLastTurn loop, after snapshot, marks scheduling), refusing agent-busy while any agent sharing the store is mid-turn; released in finally. Tests: authorization order (prepare->retract->activate; overlapping prepared batches; retract('all') over prepared; cancelled retract does not revive; later choice wins; release wins over retract-while-held; body completes with retracted marks; all across restart); startup sync-first and nothing on failed sync; imported active suppression not certified (outbox) and resumed on real startup (receipt test, idempotent second startup); offline failure retire/keep; turn undo refused while another agent is mid-turn. Awareness set + record-journal: 83/83. --- src/framework.ts | 82 +++++++----- src/recovery/discord-awareness-outbox.ts | 152 ++++++++++++++++++----- src/recovery/offline-branch.ts | 37 +++++- test/discord-awareness-outbox.test.ts | 113 +++++++++++++++++ test/offline-recovery-branch.test.ts | 59 +++++++++ test/surgery-marks-receipt.test.ts | 41 +++++- 6 files changed, 411 insertions(+), 73 deletions(-) diff --git a/src/framework.ts b/src/framework.ts index fe44344bf..66ce75541 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -5422,44 +5422,62 @@ export class AgentFramework { const turnMarks: DiscordAwarenessMarks = params.marks === 'addressed' || params.marks === 'all' ? { scope: params.marks } : 'none'; - // What the undone turns remove from this agent's context is whatever is - // in it now and not after: addressing metadata only, never content. - const before = this.addressingSnapshot(agentName); - let undone = 0; + // Each undone turn switches the whole store: hold it, like every surgery, + // for the whole act, so no agent sharing it can start a turn between the + // address snapshots and the switches they bracket. + let release: () => void; try { - for (let i = 0; i < requested; i++) { - const r = this.undoLastTurn(agentName, hostCommandRequester(serverId, params)); - if (!r.undone) break; - undone++; - } + release = this.reserveStoreForSurgery('undo turns', agentName); } catch (error) { - // e.g. "Cannot undo while agent is streaming" — report what happened, - // including any turns already undone before the failure. - const msg = error instanceof Error ? error.message : String(error); - if (undone === 0) return { ok: false, error: msg }; - console.error(`[host-command] undo partially failed after ${undone}/${requested}: ${msg}`); - } - - console.error( - `[host-command] undo agent=${agentName} requested=${requested} undone=${undone}` + - ` by=${params.requesterName ?? params.requesterId ?? 'unknown'} (server=${serverId})`, - ); - - if (undone === 0) { return { - ok: true, - undone: 0, - requested, - markers: { scope: marksScope(turnMarks) as SurgeryMarkerReceipt['scope'], unmarked: 0, notRemoved: 0, status: 'none', queued: 0 }, - lastVisible: null, + ok: false, + error: error instanceof Error ? error.message : String(error), + ...(error instanceof OperatorActionError ? { code: error.code } : {}), }; } + let undone = 0; + let markers: SurgeryMarkerReceipt; + try { + // What the undone turns remove from this agent's context is whatever is + // in it now and not after: addressing metadata only, never content. + const before = this.addressingSnapshot(agentName); + try { + for (let i = 0; i < requested; i++) { + const r = this.undoLastTurn(agentName, hostCommandRequester(serverId, params)); + if (!r.undone) break; + undone++; + } + } catch (error) { + // e.g. "Cannot undo while agent is streaming" — report what happened, + // including any turns already undone before the failure. + const msg = error instanceof Error ? error.message : String(error); + if (undone === 0) return { ok: false, error: msg }; + console.error(`[host-command] undo partially failed after ${undone}/${requested}: ${msg}`); + } + + console.error( + `[host-command] undo agent=${agentName} requested=${requested} undone=${undone}` + + ` by=${params.requesterName ?? params.requesterId ?? 'unknown'} (server=${serverId})`, + ); + + if (undone === 0) { + return { + ok: true, + undone: 0, + requested, + markers: { scope: marksScope(turnMarks) as SurgeryMarkerReceipt['scope'], unmarked: 0, notRemoved: 0, status: 'none', queued: 0 }, + lastVisible: null, + }; + } - // Marks are a one-shot choice about the messages these turns removed; - // the branch move itself never adds or removes any. - const after = new Set(this.addressingSnapshot(agentName).keys()); - const removed = [...before].filter(([id]) => !after.has(id)).map(([, carrier]) => carrier); - const markers = this.scheduleAppliedMarks('undo', agentName, serverId, removed, turnMarks); + // Marks are a one-shot choice about the messages these turns removed; + // the branch move itself never adds or removes any. + const after = new Set(this.addressingSnapshot(agentName).keys()); + const removed = [...before].filter(([id]) => !after.has(id)).map(([, carrier]) => carrier); + markers = this.scheduleAppliedMarks('undo', agentName, serverId, removed, turnMarks); + } finally { + release(); + } return { ok: true, undone, requested, markers, lastVisible: await this.lastVisiblePreview(agentName) }; } diff --git a/src/recovery/discord-awareness-outbox.ts b/src/recovery/discord-awareness-outbox.ts index 167ce380e..96c1e661f 100644 --- a/src/recovery/discord-awareness-outbox.ts +++ b/src/recovery/discord-awareness-outbox.ts @@ -146,6 +146,12 @@ export interface DiscordAwarenessBatch extends DiscordAwarenessBatchRecord { /** A suppression whose body is recorded complete (by its activation, or by * a later startup's resume). */ suppressionComplete?: boolean; + /** Journal position of the surgery's marks choice (its prepare): the + * authorization of the adds its activation requests. */ + authorization?: number; + /** Journal position of the operator's release: the authorization of the + * requests the release queues. */ + releaseAuthorization?: number; held?: { reason: string; at: number; releaseActions: DiscordAwarenessReleaseAction[] }; cancelled?: { at: number; by?: string }; released?: { at: number; by?: string }; @@ -185,6 +191,13 @@ export interface DiscordAwarenessOp { key: DiscordAwarenessRef & { emoji: string }; action: DiscordAwarenessAction; cause: DiscordAwarenessOpCause; + /** + * Journal position of the act that authorized this request: a surgery's + * marks choice (its prepare, even when activation comes later), a retract, + * or a release. Between opposite requests for one reaction, the later + * authorization wins, whenever each request was created. + */ + authorization: number; requestedAt: number; attempts: DiscordAwarenessAttempt[]; cancelled?: { at: number; reason: 'batch-cancelled' | 'request-cancelled' | 'superseded' | 'imported' }; @@ -400,6 +413,9 @@ function isAddressed(message: DiscordMessageMetadataCarrier): boolean { type JournalRecord = | { t: 'batch'; at: number; batch: DiscordAwarenessBatchRecord } | { t: 'activated'; at: number; batchId: string } + /** An imported ledger's `active`: a historical fact about its marks, never + * a certificate that a suppression's body completed. */ + | { t: 'legacy-activated'; at: number; batchId: string } | { t: 'discarded'; at: number; batchId: string } | { t: 'held'; at: number; batchId: string; reason: string; releaseActions: DiscordAwarenessReleaseAction[] } | { t: 'released'; at: number; batchId: string; by?: string } @@ -441,6 +457,7 @@ interface JournalSnapshot { legacy: DiscordAwarenessLegacyEvidence[]; retracts?: RetractRequest[]; importedSources: string[]; + position?: number; } export const DISCORD_AWARENESS_RECORD_TYPE = 'af:discord-awareness'; @@ -456,6 +473,8 @@ interface JournalState { /** Retract requests, by request id. */ retracts: Map; importedSources: Set; + /** Records applied so far: each record's journal position. */ + position: number; } interface RetractRequest { @@ -464,6 +483,8 @@ interface RetractRequest { at: number; by?: string; cancelled?: { at: number; by?: string }; + /** Journal position of the retract: its removals' authorization. */ + authorization: number; } /** @@ -536,7 +557,10 @@ export class DiscordAwarenessOutbox { /** * Activate a prepared batch after its body change landed: one `add` request - * per ref, durable before this returns. Returns how many were queued. + * per ref, durable before this returns. Activation completes the surgery's + * earlier choice; it is not a new one, so an add for a reaction that a + * later act (a retract, or another batch's request) already decided is + * recorded superseded and never sent. Returns how many were queued. */ activate(batchId: string): number { const state = this.load(); @@ -552,15 +576,16 @@ export class DiscordAwarenessOutbox { } const at = Date.now(); const records: JournalRecord[] = [{ t: 'activated', at, batchId }]; - records.push(...this.requestRecords(state, batch.refs.map((ref) => ({ ...ref, action: 'add' as const })), batch.emoji, { + const requested = this.requestRecords(state, batch.refs.map((ref) => ({ ...ref, action: 'add' as const })), batch.emoji, { kind: 'batch', batchId, - }, at)); + }, at, batch.authorization); + records.push(...requested); // The activation asserts the body change just committed (the switch, or // the last redaction): sync that first, so a crash can never leave marks // active for a change the store lost. this.append(records, { afterCommittedState: true, durable: true }); - return batch.refs.length; + return liveRequests(requested); } /** @@ -580,14 +605,19 @@ export class DiscordAwarenessOutbox { | { status: 'unresolved'; error: string } { let detail: string; try { - this.activate(batchId); - return { status: 'queued', queued: this.load().batches.get(batchId)?.refs.length ?? 0 }; + return { status: 'queued', queued: this.activate(batchId) }; } catch (error) { detail = boundDiscordAwarenessText(error instanceof Error ? error.message : String(error)); } try { - const current = this.load().batches.get(batchId); - if (current?.status === 'active') return { status: 'queued', queued: current.refs.length }; + const state = this.load(); + const current = state.batches.get(batchId); + if (current?.status === 'active') { + // The activation landed but its barrier failed: report what it queued. + const queued = [...state.ops.values()] + .filter((op) => op.cause.kind === 'batch' && op.cause.batchId === batchId && !op.cancelled).length; + return { status: 'queued', queued }; + } if (this.discard(batchId)) return { status: 'not-scheduled', error: detail }; } catch (error) { detail += `; retiring it also failed: ${boundDiscordAwarenessText(error instanceof Error ? error.message : String(error))}`; @@ -682,6 +712,7 @@ export class DiscordAwarenessOutbox { batch.emoji, { kind: 'batch', batchId: batch.id }, at, + batch.authorization, )); activated.push(batch.id); } else { @@ -697,7 +728,11 @@ export class DiscordAwarenessOutbox { held.push(batch.id); } } - if (records.length > 0) this.append(records, { durable: true }); + // A crash-completion activation asserts the target's state like any + // activation: the store's current state is synced before it is recorded + // (the store may be caller-supplied and already open), and a failed sync + // records nothing. + if (records.length > 0) this.append(records, { afterCommittedState: activated.length > 0, durable: true }); return { activated, held, unknownAttempts }; } @@ -790,20 +825,27 @@ export class DiscordAwarenessOutbox { /** * Remove this bot's mark (through the configured route) from a batch's refs, - * or from every key any batch requested or an imported ledger recorded an - * add for (`all`). A reaction is one per (route, message, emoji), whichever - * batches asked for it, so this acts across batches. A removal is queued for - * every selected key, whatever history says: history can't establish what - * is on Discord now, and removing an absent reaction does nothing. Add - * requests not yet sent for those keys are superseded. The receipt reports - * history only: requests whose outcome is unresolved, and imported history - * that leaves outcomes unrecorded, any of which may land or have landed. + * or (`all`) from every key of every batch its own surgery did not retire, + * whether prepared, held or active, every key an add was requested for, and + * every key an imported ledger recorded. A reaction is one per (route, + * message, emoji), whichever batches asked for it, so this acts across + * batches. A removal is queued for every selected key, whatever history + * says: history can't establish what is on Discord now, and removing an + * absent reaction does nothing. It is the latest authorization for those + * keys: adds not yet ended are superseded, and adds an earlier choice + * requests later (a prepared batch's activation) are recorded superseded. + * The receipt reports history only: requests whose outcome is unresolved, + * and imported history that leaves outcomes unrecorded, any of which may + * land or have landed. */ retract(target: string | 'all', by?: string): DiscordAwarenessRetractReceipt { const state = this.load(); let candidateKeys: Array; if (target === 'all') { candidateKeys = uniqueKeys([ + ...[...state.batches.values()] + .filter((batch) => batch.status !== 'discarded') + .flatMap((batch) => batch.refs.map((ref) => ({ ...ref, emoji: batch.emoji }))), ...[...state.ops.values()].filter((op) => op.action === 'add').map((op) => op.key), ...[...state.legacy.values()].flat().map((evidence) => evidence.key), ]); @@ -1100,12 +1142,19 @@ export class DiscordAwarenessOutbox { } /** - * Records for new requests. A request retires the future retries of every - * request for the same key with the opposite action that has not ended - * (due, on the wire, or unknown): the newest explicit intent wins, and - * cancelling it later never revives what it superseded. Their attempts - * stay as history (an unresolved one is still disclosed), and a request - * on the wire still settles before anything else for that key is sent. + * Records for new requests, ordered by authorization: between opposite + * requests for one key, the later authorization wins, whenever each + * request was created. `authorization` is the journal position of the act + * the requests carry out (a prepared batch's choice, when its activation + * comes later); left out, they are a new act, later than everything + * recorded (a retract, a release). + * - Each older-authorized opposite request that has not ended (due, on + * the wire, or unknown) has its future retries retired. Its attempts + * stay as history (an unresolved one is still disclosed), and a request + * on the wire still settles before anything else for that key is sent. + * - A request with an opposite request authorized after it, in any state, + * even cancelled, is recorded superseded and never sent: cancelling the + * later act stops it without reviving what it decided against. */ private requestRecords( state: JournalState, @@ -1113,6 +1162,7 @@ export class DiscordAwarenessOutbox { emoji: string | undefined, cause: DiscordAwarenessOpCause, at: number, + authorization: number = Number.POSITIVE_INFINITY, ): JournalRecord[] { const records: JournalRecord[] = []; const superseded = new Set(); @@ -1123,14 +1173,20 @@ export class DiscordAwarenessOutbox { messageId: request.messageId, emoji: request.emoji ?? emoji ?? DEFAULT_DISCORD_AWARENESS_EMOJI, }; - for (const op of this.opsForKey(state, key)) { - if (op.action === request.action || superseded.has(op.opId) || op.cancelled) continue; + const opposite = this.opsForKey(state, key).filter((op) => op.action !== request.action); + const opId = randomUUID(); + records.push({ t: 'requested', at, opId, key, action: request.action, cause }); + if (opposite.some((op) => op.authorization > authorization)) { + records.push({ t: 'op-cancelled', at, opId, reason: 'superseded' }); + continue; + } + for (const op of opposite) { + if (superseded.has(op.opId) || op.cancelled || op.authorization >= authorization) continue; const status = this.opStatus(op); if (status !== 'requested' && status !== 'dispatching' && status !== 'unknown') continue; records.push({ t: 'op-cancelled', at, opId: op.opId, reason: 'superseded' }); superseded.add(op.opId); } - records.push({ t: 'requested', at, opId: randomUUID(), key, action: request.action, cause }); } return records; } @@ -1225,7 +1281,13 @@ export class DiscordAwarenessOutbox { // --------------------------------------------------------------------------- function emptyState(): JournalState { - return { batches: new Map(), ops: new Map(), legacy: new Map(), retracts: new Map(), importedSources: new Set() }; + return { batches: new Map(), ops: new Map(), legacy: new Map(), retracts: new Map(), importedSources: new Set(), position: 0 }; +} + +/** Requests in these records that are not recorded superseded with them. */ +function liveRequests(records: JournalRecord[]): number { + const cancelled = new Set(records.filter((record) => record.t === 'op-cancelled').map((record) => (record as { opId: string }).opId)); + return records.filter((record) => record.t === 'requested' && !cancelled.has(record.opId)).length; } function snapshotState(state: JournalState): JournalSnapshot { @@ -1235,6 +1297,7 @@ function snapshotState(state: JournalState): JournalSnapshot { legacy: [...state.legacy.values()].flat().map((evidence) => structuredClone(evidence)), retracts: [...state.retracts.values()].map((retract) => structuredClone(retract)), importedSources: [...state.importedSources], + position: state.position, }; } @@ -1245,6 +1308,7 @@ function restoreState(snapshot: JournalSnapshot): JournalState { legacy: new Map(), retracts: new Map((snapshot.retracts ?? []).map((retract) => [retract.requestId, structuredClone(retract)])), importedSources: new Set(snapshot.importedSources), + position: snapshot.position ?? 0, }; for (const evidence of snapshot.legacy ?? []) addLegacyEvidence(state, structuredClone(evidence)); return state; @@ -1257,13 +1321,15 @@ function addLegacyEvidence(state: JournalState, evidence: DiscordAwarenessLegacy else state.legacy.set(key, [evidence]); } -/** Apply one entry's records, in order, to the reduced state. */ +/** Apply one entry's records, in order, to the reduced state. Each record + * takes the next journal position. */ function applyRecords(state: JournalState, records: JournalRecord[]): void { for (const record of records) { + const position = ++state.position; switch (record.t) { case 'batch': if (!state.batches.has(record.batch.id)) { - state.batches.set(record.batch.id, { ...structuredClone(record.batch), status: 'prepared' }); + state.batches.set(record.batch.id, { ...structuredClone(record.batch), status: 'prepared', authorization: position }); } break; case 'activated': { @@ -1271,12 +1337,17 @@ function applyRecords(state: JournalState, records: JournalRecord[]): void { if (batch && batch.status !== 'discarded') { batch.status = 'active'; // Only a surgery (or its resume) activates a suppression, after its - // last redaction: activation records the body complete. Release - // does not. + // last redaction synced: activation records the body complete. + // Release does not, and neither does an imported `active`. if (batch.suppressionIntervals?.length) batch.suppressionComplete = true; } break; } + case 'legacy-activated': { + const batch = state.batches.get(record.batchId); + if (batch && batch.status !== 'discarded') batch.status = 'active'; + break; + } case 'discarded': { const batch = state.batches.get(record.batchId); if (batch && batch.status === 'prepared') batch.status = 'discarded'; @@ -1296,6 +1367,7 @@ function applyRecords(state: JournalState, records: JournalRecord[]): void { batch.status = 'active'; delete batch.held; batch.released = { at: record.at, ...(record.by ? { by: record.by } : {}) }; + batch.releaseAuthorization = position; } break; } @@ -1317,6 +1389,7 @@ function applyRecords(state: JournalState, records: JournalRecord[]): void { key: { ...record.key }, action: record.action, cause: { ...record.cause }, + authorization: authorizationOf(state, record.cause) ?? position, requestedAt: record.at, attempts: [], }); @@ -1364,6 +1437,7 @@ function applyRecords(state: JournalState, records: JournalRecord[]): void { target: record.target, at: record.at, ...(record.by ? { by: record.by } : {}), + authorization: position, }); } break; @@ -1383,6 +1457,14 @@ function applyRecords(state: JournalState, records: JournalRecord[]): void { } } +/** The journal position of the act a request carries out. */ +function authorizationOf(state: JournalState, cause: DiscordAwarenessOpCause): number | undefined { + if (cause.kind === 'batch' && cause.batchId) return state.batches.get(cause.batchId)?.authorization; + if (cause.kind === 'release' && cause.batchId) return state.batches.get(cause.batchId)?.releaseAuthorization; + if (cause.kind === 'retract' && cause.requestId) return state.retracts.get(cause.requestId)?.authorization; + return undefined; +} + interface LegacyEntry extends DiscordAwarenessRef { desired?: boolean; markerPresent?: boolean; @@ -1448,9 +1530,11 @@ function importLegacy(parsed: unknown, path: string, sha256: string): JournalRec ...(raw.suppressionIntervals?.length ? { suppressionIntervals: raw.suppressionIntervals } : {}), }, }); - // An active batch was activated by its surgery, after its body (an - // earlier release activated suppressions only after the last redaction). - if (legacyActive) records.push({ t: 'activated', at: raw.createdAt, batchId: raw.id }); + // Its marks were active. That is history, not proof of a suppression's + // body: the old ledger could record activation before the redactions + // were synced, so a startup on the target still verifies and resumes + // the intervals before recording the body complete. + if (legacyActive) records.push({ t: 'legacy-activated', at: raw.createdAt, batchId: raw.id }); const releaseActions: DiscordAwarenessReleaseAction[] = []; for (const entry of raw.refs) { const key = { serverId: entry.serverId, channelId: entry.channelId, messageId: entry.messageId, emoji: raw.emoji }; diff --git a/src/recovery/offline-branch.ts b/src/recovery/offline-branch.ts index 26c467e49..c00a5a50d 100644 --- a/src/recovery/offline-branch.ts +++ b/src/recovery/offline-branch.ts @@ -4,6 +4,7 @@ import { ContextManager } from '@animalabs/context-manager'; import { DEFAULT_DISCORD_AWARENESS_EMOJI, DiscordAwarenessOutbox, + boundDiscordAwarenessText, defaultDiscordAwarenessOutboxPath, extractDiscordAwarenessRefs, selectDiscordAwarenessRefs, @@ -233,13 +234,37 @@ export async function createOfflineRecoveryBranch( } } catch (error) { // A partially suppressed branch is not safe to boot into. Preserve it - // for diagnosis, but leave Chronicle on the untouched source branch and - // keep the explicit batch prepared: it is the unfinished body's record - // (startup holds its marks, and resumes the body only on its branch). - if (store.currentBranch().name === createdBranch) { - await contextManager.switchBranch(sourceBranch); + // for diagnosis, but put Chronicle back on the untouched source branch. + // Once the source is confirmed, the batch is retired (after the + // restored state is synced): its surgery never happened. If it can't + // be confirmed, the batch stays prepared as the unfinished body's + // record (startup holds its marks, and resumes the body only on its + // branch). The error says which. + const failure = error instanceof Error ? error.message : String(error); + let restore = ''; + try { + if (store.currentBranch().name !== sourceBranch) await contextManager.switchBranch(sourceBranch); + } catch (restoreError) { + restore = `; restoring ${sourceBranch} failed: ${boundDiscordAwarenessText( + restoreError instanceof Error ? restoreError.message : String(restoreError), + )}`; } - throw error; + let retire = ''; + if (batch) { + if (store.currentBranch().name !== sourceBranch) { + retire = `; awareness batch ${batch.id} kept, since the source branch could not be confirmed`; + } else { + try { + outbox.discard(batch.id); + } catch (discardError) { + retire = `; awareness batch ${batch.id} could not be retired (${boundDiscordAwarenessText( + discardError instanceof Error ? discardError.message : String(discardError), + )}); startup will hold it`; + } + } + } + if (!restore && !retire) throw error; + throw new Error(`${failure}${restore}${retire}`, { cause: error }); } // The body change landed. Marker bookkeeping is reported apart from it diff --git a/test/discord-awareness-outbox.test.ts b/test/discord-awareness-outbox.test.ts index d0d46fab6..e90dcb183 100644 --- a/test/discord-awareness-outbox.test.ts +++ b/test/discord-awareness-outbox.test.ts @@ -329,6 +329,92 @@ test('a newer opposite intent retires the old request\'s retries even when it wa assert.ok(!due.includes('add:m1') && !due.includes('add:m2') && !due.includes('remove:m3'), due.join(',')); })); +test('authorization order: a retract after a surgery\'s choice wins over that choice\'s later activation', withJournal((outbox, h) => { + const due = (o: DiscordAwarenessOutbox) => o.pendingDispatches('discord').map((d) => `${d.action}:${d.key.messageId}`).sort(); + const prepare = (target: string, refs: DiscordAwarenessRef[], extra: Record = {}) => outbox.prepare({ + agentName: 'cairn', sourceBranch: 'main', targetBranch: target, refs, scope: 'all', ...extra, + })!; + + // prepare -> retract -> activate: the remove goes, the add never does. + const a = prepare('t/a', [ref('m1')]); + outbox.retract(a.id); + assert.equal(outbox.activate(a.id), 0, 'nothing queued'); + assert.deepEqual(outbox.settleActivation(a.id), { status: 'queued', queued: 0 }); + assert.deepEqual(due(outbox), ['remove:m1']); + + // Overlapping prepared batches: a retract of one decides the shared key + // for the other, which was chosen earlier; its other key is unaffected. + const b = prepare('t/b', [ref('m2')]); + const c = prepare('t/c', [ref('m2'), ref('m3')]); + outbox.retract(b.id); + assert.equal(outbox.activate(c.id), 1); + assert.equal(outbox.activate(b.id), 0); + assert.deepEqual(due(outbox), ['add:m3', 'remove:m1', 'remove:m2']); + + // retract('all') covers batches not yet activated. + const d = prepare('t/d', [ref('m4')]); + const all = outbox.retract('all'); + assert.ok(all.removalsQueued >= 4); + assert.equal(outbox.activate(d.id), 0); + assert.ok(!due(outbox).includes('add:m4')); + + // Cancelling the retract stops it without reviving the choice it overrode. + const e = prepare('t/e', [ref('m5')]); + const { requestId } = outbox.retract(e.id); + outbox.cancel(requestId); + assert.equal(outbox.activate(e.id), 0); + assert.ok(!due(outbox).some((d) => d.endsWith(':m5')), 'neither the add nor the cancelled remove'); + + // A later choice wins over an earlier retract. + const f = prepare('t/f', [ref('m1')]); + assert.equal(outbox.activate(f.id), 1); + assert.ok(due(outbox).includes('add:m1') && !due(outbox).includes('remove:m1')); + + // A release is a new act: it wins over a retract made while the batch was held. + const g = prepare('t/g', [ref('m6')]); + h.reopen().recoverAtStartup('elsewhere'); // holds g (and nothing else is prepared) + const held = h.reopen(); + held.retract(g.id); + assert.equal(held.release(g.id).addsQueued, 1); + assert.ok(due(held).includes('add:m6') && !due(held).includes('remove:m6')); + + // The body's obligation is not the marks': a suppression whose marks were + // retracted before activation still records its body complete. + const sup = held.prepare({ + agentName: 'cairn', sourceBranch: 'main', targetBranch: 't/s', refs: [ref('m7')], scope: 'all', + activationPolicy: 'explicit', suppressionIntervals: [{ fromId: 'i1', toId: 'i1' }], + })!; + held.retract(sup.id); + assert.equal(held.activate(sup.id), 0); + const settled = held.batches().find((x) => x.id === sup.id)!; + assert.equal(settled.status, 'active'); + assert.equal(settled.suppressionComplete, true); + assert.deepEqual(held.preparedSuppressionsForBranch('t/s'), []); + + // Everything above holds across a restart. + assert.deepEqual(due(h.reopen()), due(held)); +})); + +test('startup records a crash-completion activation only after syncing the store, and nothing when the sync fails', withJournal((outbox, h) => { + const batch = outbox.prepare({ agentName: 'cairn', sourceBranch: 'main', targetBranch: 'rollback/cairn/1', refs: [ref('m1')], scope: 'all' })!; + const restarted = h.reopen(); + const sync = h.store.sync.bind(h.store); + const calls: string[] = []; + const append = h.store.appendJson.bind(h.store); + (h.store as any).appendJson = (type: string, payload: unknown) => { calls.push('append'); return append(type, payload); }; + (h.store as any).sync = () => { calls.push('sync'); throw new Error('injected sync failure'); }; + try { + assert.throws(() => restarted.recoverAtStartup('rollback/cairn/1'), /injected sync failure/); + } finally { + (h.store as any).sync = sync; + (h.store as any).appendJson = append; + } + assert.deepEqual(calls, ['sync'], 'the sync came first, and nothing was appended'); + assert.equal(h.reopen().batches().find((b) => b.id === batch.id)?.status, 'prepared'); + // With a working store, the next startup completes it. + assert.deepEqual(h.reopen().recoverAtStartup('rollback/cairn/1').activated, [batch.id]); +})); + test('a retract request can be cancelled by the id its receipt returned, even after a restart', withJournal((outbox, h) => { const batch = activeBatch(outbox, [ref('m1'), ref('m2'), ref('m3')]); for (const id of ['m1', 'm2', 'm3']) answer(outbox, id, 'confirmed'); @@ -482,6 +568,24 @@ test('v2 import keeps what the old ledger recorded as evidence, holds pending wo assert.equal(existsSync(h.legacyPath), false); })); +test('an imported active suppression is not proof its body completed: startup still verifies its intervals', withJournal((_outbox, h) => { + mkdirSync(join(h.dir, 'recovery'), { recursive: true }); + writeFileSync(h.legacyPath, JSON.stringify({ + version: 2, + batches: [{ + id: 'legacy-sup', status: 'active', agentName: 'cairn', sourceBranch: 'main', targetBranch: 'partial', + emoji: '💤', createdAt: 1, activationPolicy: 'explicit', suppressionIntervals: [{ fromId: 'i1', toId: 'i1' }], + refs: [{ ...ref('m1'), desired: true, markerPresent: true, deliveryStatus: 'applied', attempts: 1, lastAction: 'add' }], + }], + })); + const outbox = h.reopen(); + const [batch] = outbox.batches(); + assert.equal(batch.status, 'active', 'its marks were active: history'); + assert.equal(batch.suppressionComplete, undefined, 'its body is not certified by the old ledger'); + assert.deepEqual(outbox.preparedSuppressionsForBranch('partial').map((b) => b.id), ['legacy-sup']); + assert.equal(outbox.pendingDispatches('discord').length, 0); +})); + // --------------------------------------------------------------------------- // Framework seams, with stubbed collaborators // --------------------------------------------------------------------------- @@ -706,7 +810,16 @@ test('turn-based host/command undo honours the marks choice for the messages its return { undone: true }; }; await muted(async () => { + // Another agent sharing the store is mid-turn: refused, nothing undone. + framework.activeTurnTokens.set('other', 99); + const busy = await framework.handleHostCommand('discord', { command: 'undo', agentName: 'cairn', turns: 2, marks: 'addressed' }); + assert.equal(busy.ok, false); + assert.equal(busy.code, 'agent-busy'); + assert.equal(turns, 1); + framework.activeTurnTokens.delete('other'); + const result = await framework.handleHostCommand('discord', { command: 'undo', agentName: 'cairn', turns: 2, marks: 'addressed' }); + assert.equal(framework.surgeryHold, null, 'the reservation was released'); assert.equal(result.ok, true); assert.equal(result.undone, 1); assert.equal(result.markers.status, 'queued'); diff --git a/test/offline-recovery-branch.test.ts b/test/offline-recovery-branch.test.ts index 0ee20e99f..4fcaa8e7b 100644 --- a/test/offline-recovery-branch.test.ts +++ b/test/offline-recovery-branch.test.ts @@ -372,3 +372,62 @@ test('offline recovery reports a marker bookkeeping failure apart from the body, rmSync(dir, { recursive: true, force: true }); } }); + +test('a failed offline suppression retires its batch once the source is confirmed, and keeps it when it is not', async () => { + const dir = mkdtempSync(join(tmpdir(), 'offline-recovery-fail-')); + const storePath = join(dir, 'agent.chronicle'); + const seed = async () => { + rmSync(storePath, { recursive: true, force: true }); + const store = JsStore.openOrCreate({ path: storePath }); + const cm = await ContextManager.open({ store, namespace: 'agents/cairn' }); + cm.addMessage('user', [{ type: 'text', text: 'keep' }], { serverId: 'discord', channelId: 'discord:g1:c1', messageId: 'm-keep' }); + cm.addMessage('user', [{ type: 'text', text: 'hide me' }], { serverId: 'discord', channelId: 'discord:g1:c1', messageId: 'm-a' }); + cm.addMessage('user', [{ type: 'text', text: 'anchor' }], { serverId: 'discord', channelId: 'discord:g1:c1', messageId: 'm-current' }); + cm.close(); + store.close(); + }; + const inspect = () => { + const store = JsStore.openOrCreate({ path: storePath }); + try { + const outbox = new DiscordAwarenessOutbox(store); + return { branch: store.currentBranch().name, statuses: outbox.batches().map((b) => b.status), due: outbox.pendingDispatches().length }; + } finally { + store.close(); + } + }; + const removeMessage = ContextManager.prototype.removeMessage; + const switchBranch = ContextManager.prototype.switchBranch; + try { + ContextManager.prototype.removeMessage = function () { throw new Error('injected redaction failure'); }; + + // The source comes back: the batch is retired; nothing will ever resume or mark. + await seed(); + await assert.rejects( + createOfflineRecoveryBranch({ + storePath, agentName: 'cairn', messageId: 'm-current', suppressMessageIds: ['m-a'], + marks: { scope: 'all' }, branchName: 'recovery/cairn/fail', + }), + (error: Error) => /injected redaction failure/.test(error.message) && !/kept/.test(error.message), + ); + assert.deepEqual(inspect(), { branch: 'main', statuses: ['discarded'], due: 0 }); + + // The source can't be confirmed: the batch stays as the unfinished body's record. + await seed(); + ContextManager.prototype.switchBranch = async function (this: ContextManager, name: string) { + if (name === 'main') throw new Error('injected restore failure'); + return switchBranch.call(this, name); + }; + await assert.rejects( + createOfflineRecoveryBranch({ + storePath, agentName: 'cairn', messageId: 'm-current', suppressMessageIds: ['m-a'], + marks: { scope: 'all' }, branchName: 'recovery/cairn/fail2', + }), + /injected redaction failure; restoring main failed: injected restore failure; awareness batch .* kept, since the source branch could not be confirmed/, + ); + assert.deepEqual(inspect(), { branch: 'recovery/cairn/fail2', statuses: ['prepared'], due: 0 }); + } finally { + ContextManager.prototype.removeMessage = removeMessage; + ContextManager.prototype.switchBranch = switchBranch; + rmSync(dir, { recursive: true, force: true }); + } +}); diff --git a/test/surgery-marks-receipt.test.ts b/test/surgery-marks-receipt.test.ts index 523782f29..14448a8cd 100644 --- a/test/surgery-marks-receipt.test.ts +++ b/test/surgery-marks-receipt.test.ts @@ -21,7 +21,7 @@ import { describe, it, beforeEach, afterEach } from 'node:test'; import assert from 'node:assert/strict'; import { fileURLToPath } from 'node:url'; -import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { JsStore } from '@animalabs/chronicle'; @@ -464,6 +464,45 @@ describe('surgery and awareness marks', () => { assert.equal(outbox().operations().filter((op) => op.action === 'add').length, 1, 'only the release queued marks'); }); + it('an imported active suppression is verified on its target at startup: the old ledger does not prove its body', async () => { + await start(false); + const { removed } = seed(3); + const live = cm(); + const sourceBranch = live.currentBranch().name; + // The old ledger's window: its batch says active, but the fork still + // carries the message its suppression was to remove. + await live.fork('partial'); + const [target] = live.getAllMessages().filter((m) => String(m.id) === removed[1]); + await framework.stop(); + mkdirSync(join(storePath, 'recovery'), { recursive: true }); + writeFileSync(join(storePath, 'recovery', 'discord-awareness-outbox.json'), JSON.stringify({ + version: 2, + batches: [{ + id: 'legacy-sup', status: 'active', agentName: 'resident', sourceBranch, targetBranch: 'partial', + emoji: '💤', createdAt: 1, activationPolicy: 'explicit', + suppressionIntervals: [{ fromId: String(target.id), toId: String(target.id) }], + refs: [{ + serverId: 'discord', channelId: 'discord:g1:c1', messageId: 'amb-1', + desired: true, markerPresent: true, deliveryStatus: 'applied', attempts: 1, lastAction: 'add', + }], + }], + })); + + await start(false); // on 'partial' + assert.equal(cm().currentBranch().name, 'partial'); + assert.ok(!cm().getAllMessages().some((m) => String(m.id) === removed[1]), 'the interval was resumed'); + const [batch] = outbox().batches(); + assert.equal(batch.suppressionComplete, true); + assert.equal(batch.status, 'active'); + assert.equal(outbox().operations().length, 0, 'its marks stay history: nothing is sent'); + + // Idempotent: a later startup on the target changes nothing. + await framework.stop(); + await start(false); + assert.equal(outbox().batches()[0].suppressionComplete, true); + assert.equal(outbox().operations().length, 0); + }); + it('a surgery that removes nothing addressable schedules no marks even when marks are chosen', async () => { await start(false); const keep = String(cm().addMessage('Operator', [{ type: 'text', text: 'kept' }], {})); From c19d5e6c01be22a32e444835ec0178bb5d5115a7 Mon Sep 17 00:00:00 2001 From: Ada-1017 Date: Wed, 7 Oct 2026 10:31:25 -0400 Subject: [PATCH 18/73] docs(awareness): authorization order, reserved turn undo, imported active is not body proof README live-surgery section and the three changelog fragments describe d44826b's semantics: between opposite requests for one reaction the later authorization wins (a surgery's adds carry its choice's moment; release is a new act); retract('all') covers prepared/held/active batches and cancelling it never revives what it overrode; host/command turn undo holds the store like hide; an imported active suppression is verified and resumed rather than taken as body proof. --- README.md | 39 ++++++++++++------- changelog.d/surgery-marks-controls.added.md | 9 +++-- changelog.d/surgery-marks-one-shot.changed.md | 10 +++-- changelog.d/surgery-marks-opt-in.breaking.md | 10 +++-- 4 files changed, 42 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index 1cba189ae..464882dec 100644 --- a/README.md +++ b/README.md @@ -368,12 +368,16 @@ const result = await framework.rollbackToMessage('cairn', { from the journal as it is at that moment, so a cancel takes effect even while an earlier request is on the wire. A request written and never answered is recorded as `unknown`, and a later confirmation of a different - attempt never resolves it. A record that certifies a body change (an - activation, a completed suppression, a retirement) is written only after - that change is synced. A batch whose surgery was interrupted before its - branch switch was recorded is held at startup until an operator releases - it; an interrupted suppression's redactions are resumed whenever its branch - is active at startup, whatever its marks' state. + attempt never resolves it. Between opposite requests for one reaction, + the later *authorization* wins, whenever each request was created: a + surgery's adds carry the moment of its marks choice, even when activation + comes later, so a retract made in between prevails; a release is a new + act. A record that certifies a body change (an activation, a completed + suppression, a retirement) is written only after that change is synced. + A batch whose surgery was interrupted before its branch switch was + recorded is held at startup until an operator releases it; an interrupted + suppression's redactions are resumed whenever its branch is active at + startup, whatever its marks' state. - **Operator controls,** each recorded in the operator log: - `listDiscordAwareness()` lists batches and retract requests; - `cancelDiscordAwareness(batch | retractRequest)` stops all further sends @@ -382,16 +386,19 @@ const result = await framework.rollbackToMessage('cairn', { including earlier attempts later answered, any of which may still land; - `retractDiscordAwareness(batch | 'all')` queues removal of this bot's reaction, through each address's configured MCPL route, for every message - of the batch (or every message any batch or imported ledger recorded an - add for), whatever history says: removing an absent reaction does - nothing. It supersedes adds not yet sent, and its receipt discloses - earlier requests whose outcome is unknown and imported history that - leaves outcomes unrecorded; + of the batch (or of every batch not retired by its own surgery, prepared, + held or active, and every message an imported ledger recorded), whatever + history says: removing an absent reaction does nothing. As the latest + authorization for those messages, it stops adds that haven't ended and + any an earlier choice would request later; cancelling it never revives + them. Its receipt discloses earlier requests whose outcome is unknown and + imported history that leaves outcomes unrecorded; - `releaseDiscordAwareness(batch)` queues a held batch. - **Over `host/command`** (servers granted `allowHostCommands`): `undo` (by messages or by turns) and `hide` take `marks: 'none' | 'addressed' | 'all'` - (default `none`; any other value is refused) and return `markers`. `hide` - holds the store like every surgery. The `marks` verb takes `action: 'list' + (default `none`; any other value is refused) and return `markers`. Each + holds the store like every surgery, so it is refused while any agent + sharing the store is mid-turn. The `marks` verb takes `action: 'list' | 'cancel' | 'retract' | 'release'` with a `target` (a batch id; for cancel also a retract request id; for retract also `all`). @@ -399,8 +406,10 @@ A pre-journal awareness ledger (`discord-awareness-outbox.json` under the store, or `discordAwarenessOutboxPath`) is imported once on first use and the file renamed `.migrated-v2`. What it recorded about each message is kept as evidence (attempt count, last action, status and error, and how many attempt -outcomes it leaves unrecorded), never as a claim about what is on Discord. -Its undelivered work is held for an explicit release. +outcomes it leaves unrecorded), never as a claim about what is on Discord, +and an imported `active` suppression is not taken as proof that its body +completed: startup verifies and resumes its intervals on its branch. Its +undelivered work is held for an explicit release. ## Observability diff --git a/changelog.d/surgery-marks-controls.added.md b/changelog.d/surgery-marks-controls.added.md index 28be5e31f..175fb7539 100644 --- a/changelog.d/surgery-marks-controls.added.md +++ b/changelog.d/surgery-marks-controls.added.md @@ -10,9 +10,12 @@ unknown, including earlier attempts later answered, which may still land. - `retractDiscordAwareness(batch | 'all')` queues removal of this bot's reaction, through each message's configured MCPL route, for every - selected message, whatever history says. It supersedes adds not yet - sent, and its receipt discloses requests whose outcome is unknown and - imported history that leaves outcomes unrecorded. + selected message, whatever history says. As the latest authorization + for those messages, it stops adds that haven't ended and any an earlier + choice would request later (a prepared surgery's activation), and + cancelling it never revives them. Its receipt discloses requests whose + outcome is unknown and imported history that leaves outcomes + unrecorded. - `releaseDiscordAwareness(batch)` queues a held batch. Each control is recorded in the operator log. The same controls are diff --git a/changelog.d/surgery-marks-one-shot.changed.md b/changelog.d/surgery-marks-one-shot.changed.md index 043a702d6..942aedaea 100644 --- a/changelog.d/surgery-marks-one-shot.changed.md +++ b/changelog.d/surgery-marks-one-shot.changed.md @@ -4,10 +4,12 @@ did. Each request is written ahead of its dispatch, and a dispatch is admitted from the journal as it is at that moment. A request that went out and got no answer is recorded as `unknown`, and a later confirmation of a - different attempt never resolves it. A batch whose surgery was interrupted - before its branch switch was recorded is held at startup until an operator - releases it. An interrupted suppression's redactions are resumed whenever - its branch is active at startup, independent of what happened to its marks. + different attempt never resolves it. Between opposite requests for one + reaction, the later authorization wins, whenever each was created. A batch + whose surgery was interrupted before its branch switch was recorded is held + at startup until an operator releases it. An interrupted suppression's + redactions are resumed whenever its branch is active at startup, + independent of what happened to its marks. - A journal record that certifies a body change (a marks activation, a completed suppression, a retired batch) is now written only after that change is synced to the store. Previously a crash could keep marks active diff --git a/changelog.d/surgery-marks-opt-in.breaking.md b/changelog.d/surgery-marks-opt-in.breaking.md index 0b403f5ff..a0648b270 100644 --- a/changelog.d/surgery-marks-opt-in.breaking.md +++ b/changelog.d/surgery-marks-opt-in.breaking.md @@ -21,8 +21,9 @@ reaction deadline (`discordAwarenessDeadlineMs`). Marks already on Discord are left alone; nothing is removed automatically. - Receipts (`markers`) now also carry `scope`, `unmarked` and `notRemoved`. - - `host/command` `hide` now holds the store like every surgery, so it is - refused (`agent-busy`) while any agent sharing the store is mid-turn. + - `host/command` `hide` and turn-based `undo` now hold the store like + every surgery, so they are refused (`agent-busy`) while any agent + sharing the store is mid-turn. - No sibling version is required: a host that ignores the new option simply places no marks. The connectome-host and discord-mcpl companions add the choice to their surfaces. @@ -36,5 +37,6 @@ `discordAwarenessOutboxPath` now only names a previous JSON ledger to import. Such a file (by default `/recovery/discord-awareness-outbox.json`) is imported once on first use and renamed `.migrated-v2`. What it recorded - is kept as evidence, never as synthesized attempts, and its undelivered - work is held for an explicit release. + is kept as evidence, never as synthesized attempts or as proof that a + suppression's body completed, and its undelivered work is held for an + explicit release. From 85481af8f5c6a89414b1842641cd9f8db4250db5 Mon Sep 17 00:00:00 2001 From: Ada-1017 Date: Wed, 7 Oct 2026 10:45:29 -0400 Subject: [PATCH 19/73] fix(awareness): legacy outcomes come from the old writer's lastError, not its reconciliation state; cancel discloses released removals June #45470 (probe /tmp/june-1016-799-legacy-status-probe.ts, with the actual 64c480b writer): v2 deliveryStatus is the old writer's reconciliation state (setDesired rewrites it from the cached markerPresent without any request), not the last attempt's outcome. An unknown add reconciled onto the source read as applied -> lastAddConfirmed:1, outcomesUnrecorded:0; a confirmed add reconciled onto the source read as pending -> its attempt unrecorded. The 64c480b writer recorded a success by deleting lastError (recordSuccess) and a failure by setting it (recordFailure); nothing else touches it. Evidence now carries lastOutcome derived only from that: confirmed (no error after an attempt), not-sent (pre-write refusal), refused (Discord's definitive answer), unrecorded (anything else). outcomesUnrecorded = earlier attempts + the last when unrecorded. The old status is kept as oldDeliveryStatus, documented as reconciliation state; the view's lastAdd/RemoveConfirmed count lastOutcome confirmed. Held release actions still follow the old writer's pending work (that is what its reconciliation state legitimately says). Same probe: cancel(batch).unresolvedAttempts filtered adds only, so a released removal's unknown attempt went undisclosed while the view showed it. Cancel now counts every request the batch made (adds and released removals). Test: entries exactly as 64c480b leaves them (unknown add then source reconcile; confirmed add then source reconcile) give lastAddConfirmed 1 and outcomesUnrecorded 1; the pending removal is held; release, unknown then confirmed removal, cancel -> unresolvedAttempts 1. Outbox 30/30. --- src/recovery/discord-awareness-outbox.ts | 56 +++++++++++++++++------- test/discord-awareness-outbox.test.ts | 37 ++++++++++++++++ 2 files changed, 76 insertions(+), 17 deletions(-) diff --git a/src/recovery/discord-awareness-outbox.ts b/src/recovery/discord-awareness-outbox.ts index 96c1e661f..21ab972bd 100644 --- a/src/recovery/discord-awareness-outbox.ts +++ b/src/recovery/discord-awareness-outbox.ts @@ -265,15 +265,30 @@ export interface DiscordAwarenessLegacyEvidence { /** Attempts the old ledger counted (it kept only the last one's details). */ attempts: number; lastAction: DiscordAwarenessAction; - /** The old ledger's status after the last attempt. */ - lastStatus: 'pending' | 'applied' | 'permanent-failure'; + /** + * The last attempt's outcome, as far as the old writer's fields establish + * it: it recorded a success by clearing `lastError` and a failure by + * setting it, and nothing else touched that field. `confirmed`: no error + * after an attempt. `not-sent`: AF refused before writing the request. + * `refused`: Discord answered that the message or channel is gone or + * inaccessible. `unrecorded`: any other error, so the request may have + * landed. + */ + lastOutcome: 'confirmed' | 'not-sent' | 'refused' | 'unrecorded'; + /** + * The old writer's reconciliation state at shutdown (`deliveryStatus`): + * whether its cached belief about the marker matched what it then wanted. + * Branch reconciliation rewrote it without any request, so it says + * nothing about any attempt's outcome. + */ + oldDeliveryStatus: 'pending' | 'applied' | 'permanent-failure'; lastError?: string; /** What the last error was: AF's refusal before writing a request, a * Discord answer (message/channel gone or inaccessible), or anything else. */ lastErrorKind?: 'pre-write-refusal' | 'discord-answer' | 'other'; /** Recorded attempts whose outcome the old ledger does not establish: - * every attempt before the last, and the last unless it was applied, - * refused before writing, or answered by Discord. */ + * every attempt before the last (only the last one's details were kept), + * and the last when its outcome is `unrecorded`. */ outcomesUnrecorded: number; } @@ -761,8 +776,10 @@ export class DiscordAwarenessOutbox { inFlight: 0, unknown: 0, confirmed: 0, + // Every request the batch itself made: its adds, and any removals its + // release queued. unresolvedAttempts: unresolvedDispatches( - [...state.ops.values()].filter((op) => op.cause.batchId === batchId && op.action === 'add'), + [...state.ops.values()].filter((op) => op.cause.batchId === batchId), (op, attempt) => this.attemptUnresolved(op, attempt), ), legacyOutcomesUnrecorded: [...state.legacy.values()].flat() @@ -1086,8 +1103,8 @@ export class DiscordAwarenessOutbox { ? { legacy: { entries: evidence.length, - lastAddConfirmed: evidence.filter((entry) => entry.lastAction === 'add' && entry.lastStatus === 'applied').length, - lastRemoveConfirmed: evidence.filter((entry) => entry.lastAction === 'remove' && entry.lastStatus === 'applied').length, + lastAddConfirmed: evidence.filter((entry) => entry.lastAction === 'add' && entry.lastOutcome === 'confirmed').length, + lastRemoveConfirmed: evidence.filter((entry) => entry.lastAction === 'remove' && entry.lastOutcome === 'confirmed').length, outcomesUnrecorded: evidence.reduce((sum, entry) => sum + entry.outcomesUnrecorded, 0), }, } @@ -1571,12 +1588,18 @@ function importLegacy(parsed: unknown, path: string, sha256: string): JournalRec return records; } -/** The facts a v2 entry recorded about its attempts, and what they leave open. */ +/** + * The facts a v2 entry recorded about its attempts, and what they leave open. + * Only `lastError` speaks to the last attempt's outcome (the old writer + * cleared it on success and set it on failure); `deliveryStatus` was its + * reconciliation state, rewritten by branch switches without any request, + * and is kept as that and nothing more. + */ function legacyEvidence( batchId: string, key: DiscordAwarenessRef & { emoji: string }, entry: LegacyEntry, - status: 'pending' | 'applied' | 'permanent-failure', + oldDeliveryStatus: 'pending' | 'applied' | 'permanent-failure', ): DiscordAwarenessLegacyEvidence { const attempts = entry.attempts!; const lastError = entry.lastError; @@ -1584,21 +1607,20 @@ function legacyEvidence( : PRE_WRITE_REFUSAL.test(lastError) ? 'pre-write-refusal' : isPermanentDiscordReactionFailure(lastError) ? 'discord-answer' : 'other'; - // The last attempt's outcome is established when the ledger saw it - // applied, or its error proves it was refused before writing or answered - // by Discord. Earlier attempts' details were never kept. - const lastEstablished = status === 'applied' - || lastErrorKind === 'pre-write-refusal' - || (status === 'permanent-failure' && lastErrorKind === 'discord-answer'); + const lastOutcome = lastErrorKind === undefined ? 'confirmed' + : lastErrorKind === 'pre-write-refusal' ? 'not-sent' + : lastErrorKind === 'discord-answer' ? 'refused' + : 'unrecorded'; return { batchId, key: { ...key }, attempts, lastAction: entry.lastAction!, - lastStatus: status, + lastOutcome, + oldDeliveryStatus, ...(lastError !== undefined ? { lastError: boundDiscordAwarenessText(lastError) } : {}), ...(lastErrorKind ? { lastErrorKind } : {}), - outcomesUnrecorded: (attempts - 1) + (lastEstablished ? 0 : 1), + outcomesUnrecorded: (attempts - 1) + (lastOutcome === 'unrecorded' ? 1 : 0), }; } diff --git a/test/discord-awareness-outbox.test.ts b/test/discord-awareness-outbox.test.ts index e90dcb183..dba3803e1 100644 --- a/test/discord-awareness-outbox.test.ts +++ b/test/discord-awareness-outbox.test.ts @@ -568,6 +568,43 @@ test('v2 import keeps what the old ledger recorded as evidence, holds pending wo assert.equal(existsSync(h.legacyPath), false); })); +test('legacy evidence reads outcomes from the old writer\'s lastError, never from its reconciliation state', withJournal((_outbox, h) => { + // Entries exactly as the 64c480b writer leaves them (recordSuccess clears + // lastError, recordFailure sets it; setDesired rewrites deliveryStatus + // from the cached markerPresent without any request). + const legacy = { + version: 2, + batches: [{ + id: 'reconciled', status: 'active', agentName: 'resident', sourceBranch: 'main', targetBranch: 'rollback/resident/9', + emoji: '💤', createdAt: 1, + refs: [ + // An unknown add, then a switch to the source: "applied" only because + // the cache (false) matched desired (false). The add may have landed. + { ...ref('u1'), desired: false, markerPresent: false, deliveryStatus: 'applied', attempts: 1, lastAction: 'add', + lastError: 'MCPL server "discord" did not respond within 10000ms' }, + // A confirmed add, then a switch to the source: "pending" is the + // removal the old writer wanted; the add really succeeded. + { ...ref('c1'), desired: false, markerPresent: true, deliveryStatus: 'pending', attempts: 1, lastAction: 'add' }, + ], + }], + }; + mkdirSync(join(h.dir, 'recovery'), { recursive: true }); + writeFileSync(h.legacyPath, JSON.stringify(legacy)); + const outbox = h.reopen(); + const [view] = batches(outbox); + assert.deepEqual(view.legacy, { entries: 2, lastAddConfirmed: 1, lastRemoveConfirmed: 0, outcomesUnrecorded: 1 }); + assert.equal(view.status, 'held', "the old writer's pending removal is held for release"); + + // The released removal's own requests are disclosed by cancel, like adds. + assert.equal(outbox.release('reconciled').removalsQueued, 1); + answer(outbox, 'c1', 'unknown'); + answer(outbox, 'c1', 'confirmed'); + const receipt = outbox.cancel('reconciled'); + assert.equal(receipt.unresolvedAttempts, 1, 'the unknown removal attempt may still land'); + assert.equal(receipt.confirmed, 1); + assert.equal(receipt.legacyOutcomesUnrecorded, 1); +})); + test('an imported active suppression is not proof its body completed: startup still verifies its intervals', withJournal((_outbox, h) => { mkdirSync(join(h.dir, 'recovery'), { recursive: true }); writeFileSync(h.legacyPath, JSON.stringify({ From 9baaeb17f7035eab5843da37f4a8b9b41e8c83d8 Mon Sep 17 00:00:00 2001 From: Ada-1017 Date: Wed, 7 Oct 2026 10:52:52 -0400 Subject: [PATCH 20/73] fix(awareness): a repeated cancel writes nothing and still reports truthful history June #45522: cancel() and cancelRetract() returned early with zero counters once the group was cancelled, so a confirmation landing after the first cancel read as confirmed:0 on a repeat while the view showed it. The history projection (confirmed; in flight; unknown, including attempts left without an outcome by a previous process) is now computed for every call, whatever each request's cancel state, in one helper (stopRequests); only the stopping records (and `cancelled`, newly stopped requests) depend on the group not being cancelled already, and a repeat writes nothing. Test: batch with one add on the wire and one due -> cancel (cancelled 1, inFlight 1) -> the in-flight add confirms -> repeat cancel (cancelled 0, inFlight 0, confirmed 1, unresolvedAttempts 0, no new journal records); the same for a retract request. --- src/recovery/discord-awareness-outbox.ts | 63 ++++++++++++++---------- test/discord-awareness-outbox.test.ts | 27 ++++++++++ 2 files changed, 65 insertions(+), 25 deletions(-) diff --git a/src/recovery/discord-awareness-outbox.ts b/src/recovery/discord-awareness-outbox.ts index 21ab972bd..98768be62 100644 --- a/src/recovery/discord-awareness-outbox.ts +++ b/src/recovery/discord-awareness-outbox.ts @@ -786,24 +786,49 @@ export class DiscordAwarenessOutbox { .filter((evidence) => evidence.batchId === batchId) .reduce((sum, evidence) => sum + evidence.outcomesUnrecorded, 0), }; - if (batch.cancelled) return receipt; const at = Date.now(); const records: JournalRecord[] = [{ t: 'cancelled', at, batchId, ...(by ? { by } : {}) }]; - for (const op of state.ops.values()) { - if (op.cause.batchId !== batchId) continue; - const status = this.opStatus(op); - if (status === 'confirmed') { + const ops = [...state.ops.values()].filter((op) => op.cause.batchId === batchId); + records.push(...this.stopRequests(ops, receipt, at, 'batch-cancelled', !!batch.cancelled)); + // A repeated cancel writes nothing; its receipt is still today's history. + if (!batch.cancelled) this.append(records, { durable: true }); + return receipt; + } + + /** + * Count what history says about these requests into a cancel receipt, and + * return the records that stop the ones still able to go out (none when + * the group was already cancelled). History is counted whatever each + * request's cancel state: an attempt on the wire or unanswered may land + * after any cancel, and a confirmation is a fact about Discord. + */ + private stopRequests( + ops: DiscordAwarenessOp[], + receipt: DiscordAwarenessCancelReceipt, + at: number, + reason: 'batch-cancelled' | 'request-cancelled', + alreadyCancelled: boolean, + ): JournalRecord[] { + const records: JournalRecord[] = []; + for (const op of ops) { + const last = op.attempts.at(-1); + if (last?.outcome === 'confirmed') { receipt.confirmed++; continue; } + if (last && last.outcome === undefined) { + if (this.inFlight.has(op.opId)) receipt.inFlight++; + else receipt.unknown++; + } else if (last?.outcome === 'unknown') { + receipt.unknown++; + } + if (alreadyCancelled || op.cancelled) continue; + const status = this.opStatus(op); + if (status !== 'requested' && status !== 'dispatching' && status !== 'unknown') continue; if (status === 'requested') receipt.cancelled++; - else if (status === 'dispatching') receipt.inFlight++; - else if (status === 'unknown') receipt.unknown++; - else continue; // failed or already cancelled - records.push({ t: 'op-cancelled', at, opId: op.opId, reason: 'batch-cancelled' }); + records.push({ t: 'op-cancelled', at, opId: op.opId, reason }); } - this.append(records, { durable: true }); - return receipt; + return records; } /** Stop a retract's removals that are still due; nothing sent is undone. */ @@ -821,22 +846,10 @@ export class DiscordAwarenessOutbox { unresolvedAttempts: unresolvedDispatches(ops, (op, attempt) => this.attemptUnresolved(op, attempt)), legacyOutcomesUnrecorded: 0, }; - if (retract.cancelled) return receipt; const at = Date.now(); const records: JournalRecord[] = [{ t: 'request-cancelled', at, requestId, ...(by ? { by } : {}) }]; - for (const op of ops) { - const status = this.opStatus(op); - if (status === 'confirmed') { - receipt.confirmed++; - continue; - } - if (status === 'requested') receipt.cancelled++; - else if (status === 'dispatching') receipt.inFlight++; - else if (status === 'unknown') receipt.unknown++; - else continue; - records.push({ t: 'op-cancelled', at, opId: op.opId, reason: 'request-cancelled' }); - } - this.append(records, { durable: true }); + records.push(...this.stopRequests(ops, receipt, at, 'request-cancelled', !!retract.cancelled)); + if (!retract.cancelled) this.append(records, { durable: true }); return receipt; } diff --git a/test/discord-awareness-outbox.test.ts b/test/discord-awareness-outbox.test.ts index dba3803e1..a4f93015a 100644 --- a/test/discord-awareness-outbox.test.ts +++ b/test/discord-awareness-outbox.test.ts @@ -218,6 +218,33 @@ test('the cancel receipt keeps an earlier unresolved request after a later answe assert.equal(receipt.unresolvedAttempts, 2, 'both first attempts may still land'); })); +test('a repeated cancel writes nothing, and its receipt is still the truthful history', withJournal((outbox, h) => { + const batch = activeBatch(outbox, [ref('m1'), ref('m2')]); + const onWire = outbox.claimDispatch('discord', (d) => d.key.messageId !== 'm1')!; + const first = outbox.cancel(batch.id); + assert.deepEqual({ cancelled: first.cancelled, inFlight: first.inFlight, confirmed: first.confirmed }, { cancelled: 1, inFlight: 1, confirmed: 0 }); + outbox.recordOutcome(onWire.attempts, 'confirmed'); + const records = h.store.getRecordIdsByType(DISCORD_AWARENESS_RECORD_TYPE).length; + const again = outbox.cancel(batch.id); + assert.deepEqual( + { cancelled: again.cancelled, inFlight: again.inFlight, unknown: again.unknown, confirmed: again.confirmed, unresolvedAttempts: again.unresolvedAttempts }, + { cancelled: 0, inFlight: 0, unknown: 0, confirmed: 1, unresolvedAttempts: 0 }, + ); + assert.equal(h.store.getRecordIdsByType(DISCORD_AWARENESS_RECORD_TYPE).length, records, 'nothing new written'); + + // The same for a retract request. + const other = activeBatch(outbox, [ref('m3'), ref('m4')]); + answer(outbox, 'm3', 'confirmed'); + answer(outbox, 'm4', 'confirmed'); + const { requestId } = outbox.retract(other.id); + const removal = outbox.claimDispatch('discord', (d) => d.key.messageId !== 'm3')!; + const stopped = outbox.cancel(requestId); + assert.deepEqual({ cancelled: stopped.cancelled, inFlight: stopped.inFlight }, { cancelled: 1, inFlight: 1 }); + outbox.recordOutcome(removal.attempts, 'confirmed'); + const repeated = outbox.cancel(requestId); + assert.deepEqual({ cancelled: repeated.cancelled, inFlight: repeated.inFlight, confirmed: repeated.confirmed }, { cancelled: 0, inFlight: 0, confirmed: 1 }); +})); + test('cancelling a batch its surgery has not activated stops every mark; its body can still resume', withJournal((outbox) => { const rollback = outbox.prepare({ agentName: 'cairn', sourceBranch: 'main', targetBranch: 'rollback/cairn/1', refs: [ref('m1'), ref('m2')], scope: 'all' })!; assert.equal(outbox.cancel(rollback.id, 'operator').cancelled, 2); From 511bd52503b952d9c2ea265ed1100f3ee1c9c00b Mon Sep 17 00:00:00 2001 From: Ada-1017 Date: Wed, 7 Oct 2026 10:54:08 -0400 Subject: [PATCH 21/73] docs(awareness): legacy evidence names the outcome its error field establishes and labels the old status as reconciliation state --- README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 464882dec..2c6c1cd09 100644 --- a/README.md +++ b/README.md @@ -405,8 +405,10 @@ const result = await framework.rollbackToMessage('cairn', { A pre-journal awareness ledger (`discord-awareness-outbox.json` under the store, or `discordAwarenessOutboxPath`) is imported once on first use and the file renamed `.migrated-v2`. What it recorded about each message is kept as -evidence (attempt count, last action, status and error, and how many attempt -outcomes it leaves unrecorded), never as a claim about what is on Discord, +evidence (attempt count; the last action and the outcome its error field +establishes; the old writer's delivery status, labelled as the +reconciliation state it was; and how many attempt outcomes it leaves +unrecorded), never as a claim about what is on Discord, and an imported `active` suppression is not taken as proof that its body completed: startup verifies and resumes its intervals on its branch. Its undelivered work is held for an explicit release. From 7623cf2b75805c5ab69f2ad916be66d076b9d3bc Mon Sep 17 00:00:00 2001 From: Ada-1017 Date: Wed, 7 Oct 2026 11:59:59 -0400 Subject: [PATCH 22/73] fix(awareness): refuse a known target-branch collision before preparing intent; offline branch creation settles like any body failure June #46104 (probe /tmp/june-1016-511-offline-name-probe.ts): the offline recovery called contextManager.branchAt() outside its body try/catch, after prepare. With branchName naming an existing branch it threw "Branch already exists" and left a prepared target-branch batch; starting that pre-existing branch later crash-completed it and queued an add for a message still in context. - offline: refuse an existing target name before anything is prepared (also in a dry run, which would fail the same way), and create the branch inside the body try, so a creation failure restores/confirms the source and retires the batch (or keeps it, saying so) like a failed switch or redaction. - live rollbackToMessage/suppressMessages: the same pre-check (OperatorActionError 'invalid'), before reserving or preparing; their branch creation was already inside the failure handling. Tests: offline recovery onto an existing branch with marks chosen is refused and journals nothing; an injected branchAt failure retires the batch on a confirmed source; live rollback and suppression onto an existing branch are refused with no batch. Awareness set + record-journal 87/87. --- src/framework.ts | 8 +++++ src/recovery/offline-branch.ts | 13 ++++++-- test/discord-awareness-outbox.test.ts | 2 +- test/offline-recovery-branch.test.ts | 44 +++++++++++++++++++++++++++ test/surgery-marks-receipt.test.ts | 15 +++++++++ 5 files changed, 78 insertions(+), 4 deletions(-) diff --git a/src/framework.ts b/src/framework.ts index 66ce75541..19dfd9840 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -6182,6 +6182,11 @@ export class AgentFramework { if (targetBranch === sourceBranch) { throw new OperatorActionError('invalid', 'Rollback branch name must differ from the active branch'); } + // A known collision is refused before any intent is prepared: a batch + // whose target names an existing branch would be armed by starting it. + if (this.store.listBranches().some((branch) => branch.name === targetBranch)) { + throw new OperatorActionError('invalid', `Rollback branch ${targetBranch} already exists`); + } // Gate + reserve the whole store (see reserveStoreForSurgery); held // until the switch has landed and its marker scheduling has been @@ -6328,6 +6333,9 @@ export class AgentFramework { if (targetBranch === sourceBranch) { throw new OperatorActionError('invalid', 'Suppression branch name must differ from the active branch'); } + if (this.store.listBranches().some((branch) => branch.name === targetBranch)) { + throw new OperatorActionError('invalid', `Suppression branch ${targetBranch} already exists`); + } // Gate + reserve the whole store (see reserveStoreForSurgery); held // until the fork is fully redacted and its marker scheduling has been // recorded or reported, or the source is restored. Remote delivery is diff --git a/src/recovery/offline-branch.ts b/src/recovery/offline-branch.ts index c00a5a50d..b4a1ae5f4 100644 --- a/src/recovery/offline-branch.ts +++ b/src/recovery/offline-branch.ts @@ -181,6 +181,11 @@ export async function createOfflineRecoveryBranch( if (targetBranch === sourceBranch) { throw new Error('Recovery branch name must differ from the active source branch'); } + // A known collision is refused before anything is prepared: a batch + // whose target names an existing branch would be armed by starting it. + if (store.listBranches().some((branch) => branch.name === targetBranch)) { + throw new Error(`Recovery branch ${targetBranch} already exists; choose another --branch-name`); + } const result: OfflineRecoveryBranchResult = { dryRun: options.dryRun === true, @@ -221,10 +226,12 @@ export async function createOfflineRecoveryBranch( })), }); - // branchAt uses the target message's origin sequence. No message content is - // compiled or submitted to Membrane during this operation. - const createdBranch = contextManager.branchAt(target.id, targetBranch); try { + // branchAt uses the target message's origin sequence. No message + // content is compiled or submitted to Membrane during this operation. + // Creating the branch is part of the body: a failure here settles the + // batch like a failed switch or redaction. + const createdBranch = contextManager.branchAt(target.id, targetBranch); await contextManager.switchBranch(createdBranch); // Work from newest to oldest so earlier redactions never perturb the // live positions used by later range endpoints. diff --git a/test/discord-awareness-outbox.test.ts b/test/discord-awareness-outbox.test.ts index a4f93015a..f1a2b5677 100644 --- a/test/discord-awareness-outbox.test.ts +++ b/test/discord-awareness-outbox.test.ts @@ -799,7 +799,7 @@ function hostCommandFramework(h: ReturnType, messages: Arra }; const framework = Object.create(AgentFramework.prototype) as any; framework.agents = new Map([['cairn', { state: { status: 'idle' }, getContextManager: () => contextManager }]]); - framework.store = { currentBranch: () => ({ name: currentBranch }), sync: () => h.store.sync() }; + framework.store = { currentBranch: () => ({ name: currentBranch }), listBranches: () => [], sync: () => h.store.sync() }; framework.discordAwarenessOutbox = h.outbox; framework.discordAwarenessEmoji = '🫥'; framework.discordAwarenessDrains = new Map(); diff --git a/test/offline-recovery-branch.test.ts b/test/offline-recovery-branch.test.ts index 4fcaa8e7b..ee9b6e9fb 100644 --- a/test/offline-recovery-branch.test.ts +++ b/test/offline-recovery-branch.test.ts @@ -431,3 +431,47 @@ test('a failed offline suppression retires its batch once the source is confirme rmSync(dir, { recursive: true, force: true }); } }); + +test('an offline recovery onto an existing branch is refused before any intent, and a branch-creation failure settles like any body failure', async () => { + const dir = mkdtempSync(join(tmpdir(), 'offline-recovery-name-')); + const storePath = join(dir, 'agent.chronicle'); + try { + const store = JsStore.openOrCreate({ path: storePath }); + const cm = await ContextManager.open({ store, namespace: 'agents/cairn' }); + cm.addMessage('user', [{ type: 'text', text: 'safe' }], { serverId: 'discord', channelId: 'discord:g1:c1', messageId: 'm-safe' }); + const last = cm.addMessage('user', [{ type: 'text', text: 'later' }], { + serverId: 'discord', channelId: 'discord:g1:c1', messageId: 'm-later', tags: ['chat:addressed'], + }); + cm.branchAt(last, 'taken'); // an existing branch that still has both messages + cm.close(); + store.close(); + const inspect = () => { + const reopened = JsStore.openOrCreate({ path: storePath }); + try { + return { branch: reopened.currentBranch().name, statuses: new DiscordAwarenessOutbox(reopened).batches().map((b) => b.status) }; + } finally { + reopened.close(); + } + }; + + await assert.rejects( + createOfflineRecoveryBranch({ storePath, agentName: 'cairn', messageId: 'm-safe', marks: { scope: 'all' }, branchName: 'taken' }), + /Recovery branch taken already exists/, + ); + assert.deepEqual(inspect(), { branch: 'main', statuses: [] }, 'nothing prepared, so starting taken arms nothing'); + + const branchAt = ContextManager.prototype.branchAt; + ContextManager.prototype.branchAt = function () { throw new Error('injected branch failure'); }; + try { + await assert.rejects( + createOfflineRecoveryBranch({ storePath, agentName: 'cairn', messageId: 'm-safe', marks: { scope: 'all' }, branchName: 'fresh' }), + (error: Error) => /injected branch failure/.test(error.message) && !/kept/.test(error.message), + ); + } finally { + ContextManager.prototype.branchAt = branchAt; + } + assert.deepEqual(inspect(), { branch: 'main', statuses: ['discarded'] }); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); diff --git a/test/surgery-marks-receipt.test.ts b/test/surgery-marks-receipt.test.ts index 14448a8cd..2f35dfa14 100644 --- a/test/surgery-marks-receipt.test.ts +++ b/test/surgery-marks-receipt.test.ts @@ -503,6 +503,21 @@ describe('surgery and awareness marks', () => { assert.equal(outbox().operations().length, 0); }); + it('a rollback or suppression onto an existing branch is refused before any intent is prepared', async () => { + await start(false); + const { tail, removed } = seed(2); + cm().branchAt(cm().getAllMessages().at(-1)!.id, 'taken'); + await assert.rejects( + framework.rollbackToMessage('resident', { messageId: tail, branchName: 'taken', marks: { scope: 'all' } }), + /Rollback branch taken already exists/, + ); + await assert.rejects( + framework.suppressMessages('resident', { messageIds: [removed[0]], branchName: 'taken', marks: { scope: 'all' } }), + /Suppression branch taken already exists/, + ); + assert.equal(outbox().batches().length, 0); + }); + it('a surgery that removes nothing addressable schedules no marks even when marks are chosen', async () => { await start(false); const keep = String(cm().addMessage('Operator', [{ type: 'text', text: 'kept' }], {})); From dc2928ce421eb748fe645490b05e6bc06ff16893 Mon Sep 17 00:00:00 2001 From: Ada-1017 Date: Wed, 7 Oct 2026 12:01:50 -0400 Subject: [PATCH 23/73] fix(recover): name the CLI's actual --branch flag in the collision error --- src/recovery/offline-branch.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/recovery/offline-branch.ts b/src/recovery/offline-branch.ts index b4a1ae5f4..ef0541e42 100644 --- a/src/recovery/offline-branch.ts +++ b/src/recovery/offline-branch.ts @@ -184,7 +184,7 @@ export async function createOfflineRecoveryBranch( // A known collision is refused before anything is prepared: a batch // whose target names an existing branch would be armed by starting it. if (store.listBranches().some((branch) => branch.name === targetBranch)) { - throw new Error(`Recovery branch ${targetBranch} already exists; choose another --branch-name`); + throw new Error(`Recovery branch ${targetBranch} already exists; choose another --branch`); } const result: OfflineRecoveryBranchResult = { From 5b34745c8b811e9480a9210faeeb27761e146b69 Mon Sep 17 00:00:00 2001 From: Ansel-1606 Date: Wed, 7 Oct 2026 16:28:11 -0400 Subject: [PATCH 24/73] fix(awareness): a hide or turn undo records its batch after the change is synced, and reports what the journal holds Greptile review of #250 (P1 "Marks return after failed scheduling", P1 "Hide marks can outlive removal"); both reproduced at dc2928c. scheduleAppliedMarks prepared its batch after the change was already applied (an in-place hide's redaction, a turn undo's switch), with only the after-append durability sync. A kill between the batch append and that sync could keep the batch (appends reach the OS at once) and lose the buffered removal; a hide keeps its branch, so startup activated the batch and marked messages the agent still sees. And when prepare() threw after its record reached the store (a failed post-append sync), the receipt said `not-scheduled` ("nothing was recorded") while the batch sat prepared for the current branch, and startup delivered it. The outbox now owns that protocol: settleApplied() builds the batch for exactly the change's branch, appends it with afterCommittedState (sync before append, the barrier activate/discard already use), and settles it like any prepared batch (activate, else retire, else unresolved). If the write fails, it reads the journal back: a batch that is not there is `not-scheduled`; one that is there is settled and the receipt follows that; if the read-back fails too, `unresolved` with the batch id. A journal that can't be read at all writes nothing (`not-scheduled`). The framework maps every settlement onto the receipt and ops alert through one path (settledMarkerReceipt) shared with rollback/suppress; its wording no longer says "batch retired" for a batch never recorded. The receipt docs (operator-log.ts, README) and describeMarkers say what not-scheduled and unresolved now cover. DiscordAwarenessPrepareInput and DiscordAwarenessSettlement are exported. Tests (test/discord-awareness-outbox.test.ts), through host/command hide over a store whose journal seams can be observed and faulted: - the removal is synced before the batch is appended; - a failed barrier sync before the append: not-scheduled, nothing recorded, startup activates nothing; - a failed sync right after the append: the batch is settled (queued) and the receipt names it; - every sync from the append on fails: unresolved, the receipt names the retained prepared batch, and startup may activate it, as it says; - the read-back after the failed write fails: unresolved, with the id. All five fail at dc2928c on their status/ordering assertions. --- README.md | 8 +- changelog.d/surgery-marks-one-shot.changed.md | 10 +- src/framework.ts | 77 +++++--- src/index.ts | 2 + src/operator-log.ts | 9 +- src/recovery/discord-awareness-outbox.ts | 145 +++++++++++---- test/discord-awareness-outbox.test.ts | 171 ++++++++++++++++++ 7 files changed, 346 insertions(+), 76 deletions(-) diff --git a/README.md b/README.md index 2c6c1cd09..68b6548bb 100644 --- a/README.md +++ b/README.md @@ -349,8 +349,9 @@ const result = await framework.rollbackToMessage('cairn', { accepted them. `result.markers` says whether they were scheduled (`queued` with a count and batch id), not chosen or not in scope (`none`), not to be delivered because recording the batch failed after the body change and it - was retired (`not-scheduled`), or `unresolved` (neither activation nor - retirement could be recorded; it names a batch that may still be + was retired or never recorded (`not-scheduled`), or `unresolved` (neither + activation nor retirement could be recorded, or the journal could not be + read back to say whether the batch was; it names a batch that may still be delivered). Every receipt also counts the removed messages left unmarked. Marker bookkeeping never fails or undoes an applied rollback or suppression. @@ -373,7 +374,8 @@ const result = await framework.rollbackToMessage('cairn', { surgery's adds carry the moment of its marks choice, even when activation comes later, so a retract made in between prevails; a release is a new act. A record that certifies a body change (an activation, a completed - suppression, a retirement) is written only after that change is synced. + suppression, a retirement, or the batch a hide or turn undo records after + its change) is written only after that change is synced. A batch whose surgery was interrupted before its branch switch was recorded is held at startup until an operator releases it; an interrupted suppression's redactions are resumed whenever its branch is active at diff --git a/changelog.d/surgery-marks-one-shot.changed.md b/changelog.d/surgery-marks-one-shot.changed.md index 942aedaea..44b4a2f37 100644 --- a/changelog.d/surgery-marks-one-shot.changed.md +++ b/changelog.d/surgery-marks-one-shot.changed.md @@ -11,9 +11,13 @@ redactions are resumed whenever its branch is active at startup, independent of what happened to its marks. - A journal record that certifies a body change (a marks activation, a - completed suppression, a retired batch) is now written only after that - change is synced to the store. Previously a crash could keep marks active - for a rollback the store had lost. + completed suppression, a retired batch, or the batch a `hide` or turn + `undo` records after its change) is now written only after that change is + synced to the store. Previously a crash could keep marks active for a + rollback the store had lost. If writing a `hide` or turn `undo` batch + fails, its receipt follows what the journal then holds: a batch that + reached the store is activated, retired or reported `unresolved`, never + reported `not-scheduled` while it could still be delivered. - Awareness delivery no longer holds MCPL traffic. Previously every MCPL data plane waited at startup, at reconnect, at a tools list change and after each surgery until every queued mark had been attempted, which at Discord's pace diff --git a/src/framework.ts b/src/framework.ts index 19dfd9840..04a399fe4 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -416,6 +416,7 @@ import { type DiscordAwarenessReleaseReceipt, type DiscordAwarenessRetractReceipt, type DiscordAwarenessScope, + type DiscordAwarenessSettlement, } from './recovery/discord-awareness-outbox.js'; import { OperatorActionError, @@ -737,7 +738,7 @@ function describeMarksChoice(marks: DiscordAwarenessMarks | undefined): Record ({ ...interval })) } - : {}), - }; + prepare(input: DiscordAwarenessPrepareInput): DiscordAwarenessBatch | null { + const batch = batchRecord(input); + if (!batch) return null; // Durable before the surgery's switch, so startup can crash-complete it. this.append([{ t: 'batch', at: batch.createdAt, batch }], { durable: true }); return { ...structuredClone(batch), status: 'prepared' }; } + /** + * Marks for a body change already applied on `branch` (an in-place hide, a + * turn undo): a batch for exactly that branch, recorded and activated in + * one call, with what the journal will do (as settleActivation reports + * it). Its record asserts the change before it, so it is written only + * after that change is synced: a crash can never keep the batch and lose + * the change. A batch left prepared by a crash between its two records is + * for exactly this branch, which startup completes. If writing it fails, + * the journal is read back, so the result follows what it holds: a batch + * that is not there is `not-scheduled` (nothing was recorded); one that is + * there is settled like any prepared batch (activated, else retired, else + * `unresolved`); and if the journal can't be read back, `unresolved`, since + * the batch may be there. Returns null when there is nothing to mark. + * Never throws. + */ + settleApplied( + input: Omit + & { branch: string }, + ): { batchId: string; marks: number; settled: DiscordAwarenessSettlement } | null { + const { branch, ...rest } = input; + const batch = batchRecord({ ...rest, sourceBranch: branch, targetBranch: branch, activationPolicy: 'explicit' }); + if (!batch) return null; + const settle = (settled: DiscordAwarenessSettlement) => ({ batchId: batch.id, marks: batch.refs.length, settled }); + try { + this.load(); + } catch (error) { + // Nothing is written to a journal that can't be read. + return settle({ status: 'not-scheduled', error: `the journal could not be read: ${errorText(error)}` }); + } + try { + this.append([{ t: 'batch', at: batch.createdAt, batch }], { afterCommittedState: true, durable: true }); + } catch (error) { + const detail = `recording the batch failed: ${errorText(error)}`; + let landed: boolean; + try { + landed = this.load().batches.has(batch.id); + } catch (readError) { + return settle({ status: 'unresolved', error: `${detail}; reading the journal back also failed: ${errorText(readError)}` }); + } + if (!landed) return settle({ status: 'not-scheduled', error: detail }); + // It reached the store: carry out the choice it records, like any + // prepared batch, and report what that settles. + const settled = this.settleActivation(batch.id); + return settle(settled.status === 'queued' ? settled : { ...settled, error: `${detail}; ${settled.error}` }); + } + return settle(this.settleActivation(batch.id)); + } + /** * Activate a prepared batch after its body change landed: one `add` request * per ref, durable before this returns. Activation completes the surgery's @@ -614,10 +663,7 @@ export class DiscordAwarenessOutbox { * the batch may activate it. * Never throws. */ - settleActivation(batchId: string): - | { status: 'queued'; queued: number } - | { status: 'not-scheduled'; error: string } - | { status: 'unresolved'; error: string } { + settleActivation(batchId: string): DiscordAwarenessSettlement { let detail: string; try { return { status: 'queued', queued: this.activate(batchId) }; @@ -1637,6 +1683,33 @@ function legacyEvidence( }; } +/** A new batch's record, or null when it carries neither marks nor a + * suppression journal. */ +function batchRecord(input: DiscordAwarenessPrepareInput): DiscordAwarenessBatchRecord | null { + const refs = dedupeRefs(input.refs); + if (refs.length === 0 && !input.suppressionIntervals?.length) return null; + return { + id: randomUUID(), + agentName: input.agentName, + sourceBranch: input.sourceBranch, + targetBranch: input.targetBranch, + emoji: input.emoji ?? DEFAULT_DISCORD_AWARENESS_EMOJI, + createdAt: Date.now(), + scope: input.scope, + refs, + ...(input.unmarked ? { unmarked: input.unmarked } : {}), + ...(input.notRemoved ? { notRemoved: input.notRemoved } : {}), + activationPolicy: input.activationPolicy ?? 'target-branch', + ...(input.suppressionIntervals?.length + ? { suppressionIntervals: input.suppressionIntervals.map((interval) => ({ ...interval })) } + : {}), + }; +} + +function errorText(error: unknown): string { + return boundDiscordAwarenessText(error instanceof Error ? error.message : String(error)); +} + // --------------------------------------------------------------------------- // Keys // --------------------------------------------------------------------------- diff --git a/test/discord-awareness-outbox.test.ts b/test/discord-awareness-outbox.test.ts index f1a2b5677..34a46265b 100644 --- a/test/discord-awareness-outbox.test.ts +++ b/test/discord-awareness-outbox.test.ts @@ -933,3 +933,174 @@ test('host/command hide takes the store reservation and marks only on request, t h.cleanup(); } }); + +const JOURNAL_APPEND = `append ${DISCORD_AWARENESS_RECORD_TYPE}`; +/** The call that just happened was a journal entry's append. */ +const rightAfterAppend = (calls: string[]) => calls.at(-1) === JOURNAL_APPEND; +/** A journal entry has been appended (since the seams were made). */ +const afterAnAppend = (calls: string[]) => calls.includes(JOURNAL_APPEND); + +/** + * The harness store with its journal seams observed and faultable. `fail` + * makes calls at a seam throw while `matches` holds for the calls before + * them (up to `times` of them): `before` the real call, so nothing reaches + * the store, or `after` it, so it did. + */ +function journalSeams(store: JsStore) { + type Seam = 'sync' | 'append' | 'list'; + const calls: string[] = []; + const faults: Array<{ seam: Seam; when: 'before' | 'after'; matches: (calls: string[]) => boolean; times: number }> = []; + const through = (seam: Seam, label: string, call: () => T): T => { + const fault = faults.find((candidate) => candidate.seam === seam && candidate.times > 0 && candidate.matches(calls)); + calls.push(label); + if (fault) fault.times--; + if (fault?.when === 'before') throw new Error(`injected ${seam} failure`); + const result = call(); + if (fault?.when === 'after') throw new Error(`injected ${seam} failure`); + return result; + }; + const proxy = new Proxy(store, { + get(target, prop) { + if (prop === 'sync') return () => through('sync', 'sync', () => target.sync()); + if (prop === 'appendJson') { + return (type: string, payload: unknown) => through('append', `append ${type}`, () => target.appendJson(type, payload)); + } + if (prop === 'getRecordIdsByType') { + return (type: string) => through('list', `list ${type}`, () => target.getRecordIdsByType(type)); + } + const value = Reflect.get(target, prop); + return typeof value === 'function' ? value.bind(target) : value; + }, + }); + return { + store: proxy as JsStore, + calls, + fail: (seam: Seam, when: 'before' | 'after', matches: (calls: string[]) => boolean, times = 1) => { + faults.push({ seam, when, matches, times }); + }, + }; +} + +/** + * A hide with marks over a journal whose store faults as `arrange` says. + * The removal shows in the calls as `remove`. + */ +async function hideOverFaultyJournal(arrange: (seams: ReturnType) => void) { + const h = storeHarness(); + const seams = journalSeams(h.store); + const outbox = new DiscordAwarenessOutbox(seams.store); + outbox.batches(); // loaded before the hide, as a running framework's is + const { framework, removed } = hostCommandFramework(h, [ + { id: 'i0', metadata: { ...ref('m0') } }, + { id: 'i1', metadata: { ...ref('m1'), tags: ['chat:addressed'] } }, + ]); + framework.discordAwarenessOutbox = outbox; + framework.agents.get('cairn').getContextManager().removeMessage = (id: string) => { + seams.calls.push('remove'); + removed.add(id); + }; + const alerts: string[] = []; + framework.opsAlert = (kind: string) => { alerts.push(kind); }; + arrange(seams); + const result: any = await muted(() => framework.handleHostCommand('discord', { + command: 'hide', agentName: 'cairn', fromMessageId: 'm1', marks: 'all', + })); + return { h, seams, result, alerts }; +} + +test('a hide records its batch only after the removal is synced: the batch can never outlive the change', async () => { + const { h, seams, result } = await hideOverFaultyJournal(() => {}); + try { + assert.equal(result.markers.status, 'queued'); + const removal = seams.calls.indexOf('remove'); + const batchAppend = seams.calls.indexOf(JOURNAL_APPEND, removal); + assert.ok(removal >= 0 && batchAppend > removal, `calls: ${seams.calls.join(', ')}`); + assert.ok( + seams.calls.slice(removal, batchAppend).includes('sync'), + `the removal is synced before the batch is appended: ${seams.calls.join(', ')}`, + ); + } finally { + h.cleanup(); + } +}); + +test('an applied change\'s batch that failed before reaching the store is not scheduled, and nothing delivers it', async () => { + // The barrier sync between the removal and the batch's append fails: + // nothing is written. + const { h, result, alerts } = await hideOverFaultyJournal((seams) => { + seams.fail('sync', 'before', (calls) => calls.includes('remove') && !afterAnAppend(calls)); + }); + try { + assert.equal(result.ok, true, 'the hide stands'); + assert.equal(result.markers.status, 'not-scheduled'); + const fresh = new DiscordAwarenessOutbox(h.store); + assert.equal(fresh.batches().length, 0, 'nothing was recorded'); + assert.deepEqual(fresh.recoverAtStartup('main'), { activated: [], held: [], unknownAttempts: 0 }); + assert.equal(fresh.pendingDispatches('discord').length, 0); + assert.match(result.markers.error, /recording the batch failed: injected sync failure/); + assert.deepEqual(alerts, ['discord-awareness-not-scheduled']); + } finally { + h.cleanup(); + } +}); + +test('an applied change\'s batch that reached the store before its barrier failed is settled, and the receipt follows it', async () => { + // The append lands and its durability sync fails: the batch is in the + // journal, so the operator's choice is carried out (here, activated) + // rather than reported as never scheduled. + const { h, result, alerts } = await hideOverFaultyJournal((seams) => { + seams.fail('sync', 'before', rightAfterAppend); + }); + try { + assert.equal(result.markers.status, 'queued', 'not a not-scheduled receipt over a batch that would be delivered'); + assert.equal(result.markers.queued, 1); + const fresh = new DiscordAwarenessOutbox(h.store); + const [batch] = fresh.batches(); + assert.equal(batch.id, result.markers.batchId); + assert.equal(batch.status, 'active'); + assert.deepEqual(fresh.pendingDispatches('discord').map((d) => d.key.messageId), ['m1']); + assert.deepEqual(fresh.recoverAtStartup('main'), { activated: [], held: [], unknownAttempts: 0 }); + assert.deepEqual(alerts, []); + } finally { + h.cleanup(); + } +}); + +test('an applied change\'s batch that landed but can be neither activated nor retired is unresolved, with its id', async () => { + // The append lands, and every sync from its own on fails: the batch is + // in the journal, and activation and retirement each write nothing. + const { h, result, alerts } = await hideOverFaultyJournal((seams) => { + seams.fail('sync', 'before', afterAnAppend, Number.POSITIVE_INFINITY); + }); + try { + assert.equal(result.markers.status, 'unresolved'); + const fresh = new DiscordAwarenessOutbox(h.store); + const [batch] = fresh.batches(); + assert.equal(batch.id, result.markers.batchId, 'the receipt names the retained batch'); + assert.equal(batch.status, 'prepared'); + // As the receipt says, a startup that reads it may deliver it. + assert.deepEqual(fresh.recoverAtStartup('main').activated, [batch.id]); + assert.match(result.markers.error, /recording the batch failed.*retiring it also failed/); + assert.deepEqual(alerts, ['discord-awareness-unresolved']); + } finally { + h.cleanup(); + } +}); + +test('an applied change\'s batch whose journal cannot be read back after a failed write is unresolved, with its id', async () => { + // The append lands, its durability sync fails, and the read-back fails. + const { h, result, alerts } = await hideOverFaultyJournal((seams) => { + seams.fail('sync', 'before', rightAfterAppend); + seams.fail('list', 'before', afterAnAppend); + }); + try { + assert.equal(result.markers.status, 'unresolved'); + const [batch] = new DiscordAwarenessOutbox(h.store).batches(); + assert.equal(batch.id, result.markers.batchId, 'the receipt names the batch that may be delivered'); + assert.equal(batch.status, 'prepared'); + assert.match(result.markers.error, /reading the journal back also failed: injected list failure/); + assert.deepEqual(alerts, ['discord-awareness-unresolved']); + } finally { + h.cleanup(); + } +}); From 9d17ad67b82b974026f734b2ca687d157d4d06ba Mon Sep 17 00:00:00 2001 From: Ansel-1606 Date: Wed, 7 Oct 2026 16:37:01 -0400 Subject: [PATCH 25/73] fix(mcpl): a reconnect holds control traffic as well as data until the new grant is established MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Greptile review of #250 (P1 "First reconnect loses channel registration"), plus the same-root case Mira-1605 identified in review. A reconnect resets the server's grant (a fresh initialize is a fresh §5.3 policy epoch), but McplServerConnection.attemptReconnect paused only the data plane, and #250's reconnect listener then called readyControlPlane() before re-running registerMcplServerFeatures. So control traffic from the fresh transport reached admission against the empty grant: a channels/register sent right after initialize (as chat bridges do) was refused with -32002 ("initial policy not yet established") and never replayed, leaving the route's incoming messages unknown channels until the server registered again. Two paths led there: - a stub whose first connect failed and which reconnects while startup is still staging other servers: its control plane was still closed until the listener's new readyControlPlane() opened it (the regression); - any connection that was already open: its control plane was never closed at the boundary (pre-existing; the old barrier code paused data only, too). attemptReconnect now closes both planes with the grant reset, as initial connect holds them, and the framework's reconnect listener no longer opens the control plane early: it re-establishes the grant (policy responses are never event-buffered, so holding control cannot deadlock the exchange) and then opens the planes through readyMcplPlanes, as it already did, also when the exchange fails. Quiesce still pauses only the data plane (pauseDataPlane's doc now says that is its use). Tests: - test/mcpl-reconnect-registration.test.ts (new, with fixture reregistering-mcpl-server.mjs): the real framework, a server that registers its channel on notifications/initialized and exits once on demand; after an ordinary reconnect the fresh process's registration must be answered without error. At dc2928c it is refused -32002; with only the listener change, or only the connection change, it is still refused; both together pass. - test/mcpl-reconnect-events.test.ts: after a reconnect, a control event is held until ready(), then released (fails at dc2928c). Changelog: mcpl-reconnect-registration.fixed.md. --- .../mcpl-reconnect-registration.fixed.md | 9 ++ src/framework.ts | 13 ++- src/mcpl/server-connection.ts | 18 ++-- test/fixtures/reregistering-mcpl-server.mjs | 68 +++++++++++++ test/mcpl-reconnect-events.test.ts | 21 ++++ test/mcpl-reconnect-registration.test.ts | 96 +++++++++++++++++++ 6 files changed, 211 insertions(+), 14 deletions(-) create mode 100644 changelog.d/mcpl-reconnect-registration.fixed.md create mode 100644 test/fixtures/reregistering-mcpl-server.mjs create mode 100644 test/mcpl-reconnect-registration.test.ts diff --git a/changelog.d/mcpl-reconnect-registration.fixed.md b/changelog.d/mcpl-reconnect-registration.fixed.md new file mode 100644 index 000000000..b52983a18 --- /dev/null +++ b/changelog.d/mcpl-reconnect-registration.fixed.md @@ -0,0 +1,9 @@ +- An MCPL server that reconnects now has its channel registration accepted. + A reconnect resets the server's grant, and its traffic used to reach + admission before the host's new §5.3 policy Request was answered: a + `channels/register` sent right after initialize, as chat bridges do, was + refused (-32002) and never replayed, so the route's incoming messages + stayed unknown channels until the server registered again. A reconnect + now holds control traffic as well as data until the new grant is + established, as initial connect already did, both for a connection that + was open and for one whose first connect had failed. diff --git a/src/framework.ts b/src/framework.ts index 04a399fe4..60d80bbf6 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -14559,13 +14559,12 @@ export class AgentFramework { // preserved across the transient close are resumed by idempotent // registration. Then refresh tools (server may have different tools). connection.on('reconnect', (info?: { attempts?: number }) => { - // The connection paused its data plane before wiring the fresh - // transport. Let control traffic through for registration, then - // re-establish the grant: the §5.3 policy Request must be answered - // before this connection's data plane opens, otherwise the first - // post-reconnect events land while the grant is still empty and are - // rejected fail-closed instead of delivered. - connection.readyControlPlane(); + // The connection closed both planes before wiring the fresh transport, + // as at initial connect. Re-establish the grant first: the §5.3 policy + // Request (whose response is never buffered) must be answered before + // any plane opens, otherwise the first post-reconnect traffic, control + // included (the server's channels/register), lands while the grant is + // still empty and is rejected fail-closed instead of delivered. void (async () => { try { const config = this.mcplServerConfigs.get(connection.id); diff --git a/src/mcpl/server-connection.ts b/src/mcpl/server-connection.ts index 273f7b61a..8dab9ac99 100644 --- a/src/mcpl/server-connection.ts +++ b/src/mcpl/server-connection.ts @@ -354,8 +354,8 @@ export class McplServerConnection extends EventEmitter { /** * Close the data plane synchronously while leaving an already-released - * control plane live. Used before reconnect adoption, so no data event can - * arrive before the server's grant is re-established, and by quiesce. + * control plane live: quiesce holds wakes this way while operator traffic + * (host/command) still flows. A reconnect closes both planes instead. */ pauseDataPlane(): void { this.dataPlaneReady = false; @@ -953,11 +953,15 @@ export class McplServerConnection extends EventEmitter { this.hostCapabilities, ); - // Close the data plane before the new transport can emit. The reconnect - // lifecycle event bypasses buffering; its framework listener releases - // the control traffic needed to re-establish the server's grant while - // data stays held until the grant is in place. - this.pauseDataPlane(); + // Close both planes before the new transport can emit, as at initial + // connect: its traffic is admitted against the grant, which this + // boundary resets, so control traffic too (a server's channels/register + // right after initialize) must wait for the new grant rather than be + // refused against the empty one. The reconnect lifecycle event bypasses + // buffering; its framework listener re-establishes the grant (§5.3 + // policy responses are never buffered) and then opens the planes. + this.controlPlaneReady = false; + this.dataPlaneReady = false; this.resetPolicyForTransportBoundary(); this.transport = transport; this.capabilities = capabilities; diff --git a/test/fixtures/reregistering-mcpl-server.mjs b/test/fixtures/reregistering-mcpl-server.mjs new file mode 100644 index 000000000..c18e3639f --- /dev/null +++ b/test/fixtures/reregistering-mcpl-server.mjs @@ -0,0 +1,68 @@ +// MCPL server for test/mcpl-reconnect-registration.test.ts. +// +// Like a chat bridge, it registers its channel as soon as the host sends +// notifications/initialized, before the host's §5.3 policy Request is +// answered, and appends the host's answer to that registration to +// EVENTS_PATH. When CRASH_PATH appears it removes the file and exits, once, +// so the host's reconnect starts a fresh process that registers again. +import { appendFileSync, existsSync, unlinkSync } from 'node:fs'; + +const eventsPath = process.env.EVENTS_PATH; +const crashPath = process.env.CRASH_PATH; + +const send = (message) => process.stdout.write(JSON.stringify(message) + '\n'); +const event = (name, extra = {}) => { + if (eventsPath) appendFileSync(eventsPath, JSON.stringify({ event: name, pid: process.pid, ...extra }) + '\n'); +}; + +let buf = ''; +process.stdin.on('data', (chunk) => { + buf += chunk.toString('utf8'); + let newline; + while ((newline = buf.indexOf('\n')) >= 0) { + const line = buf.slice(0, newline); + buf = buf.slice(newline + 1); + if (!line.trim()) continue; + const message = JSON.parse(line); + if (message.method === 'initialize') { + send({ + jsonrpc: '2.0', + id: message.id, + result: { + capabilities: { + experimental: { + mcpl: { + version: '0.5', + pushEvents: true, + channels: { register: true, incoming: true }, + featureSets: { chat: { description: 'chat', uses: ['pushEvents'] } }, + }, + }, + }, + }, + }); + } else if (message.method === 'notifications/initialized') { + send({ + jsonrpc: '2.0', + id: 100, + method: 'channels/register', + params: { channels: [{ id: 'chat:room-1', type: 'chat', label: 'room-1', direction: 'bidirectional' }] }, + }); + } else if (message.method === 'featureSets/update') { + if (message.id !== undefined && message.id !== null) send({ jsonrpc: '2.0', id: message.id, result: { accepted: true } }); + } else if (message.method === 'tools/list') { + send({ jsonrpc: '2.0', id: message.id, result: { tools: [] } }); + } else if (message.id === 100 && message.method === undefined) { + event('register-answered', { error: message.error ?? null }); + } + } +}); + +const timer = setInterval(() => { + if (crashPath && existsSync(crashPath)) { + unlinkSync(crashPath); + event('crashing'); + process.exit(9); + } +}, 5); +timer.unref(); diff --git a/test/mcpl-reconnect-events.test.ts b/test/mcpl-reconnect-events.test.ts index 2c12d7407..5a58b2abc 100644 --- a/test/mcpl-reconnect-events.test.ts +++ b/test/mcpl-reconnect-events.test.ts @@ -170,4 +170,25 @@ describe('McplServerConnection — reconnect lifecycle events', () => { grant: [], }); }); + + it('holds a reconnected transport\'s control traffic, not only its data, until the host opens the planes', async () => { + writeFileSync(FLAG_FILE, ''); + connection = await McplServerConnection.connect(makeConfig(), HOST_CAPS); + const controlEvents: string[] = []; + connection.on('tools-list-changed', () => controlEvents.push('tools-list-changed')); + let reconnected = false; + connection.on('reconnect', () => { reconnected = true; }); + connection.ready(); + connection.emit('tools-list-changed'); + assert.deepStrictEqual(controlEvents, ['tools-list-changed'], 'an open control plane delivers'); + + connection.sendToolsList().catch(() => {}); + await waitFor(() => reconnected, 5000, 'reconnect after crash'); + // What the fresh transport sends is admitted against a grant the host + // has not re-established yet: it waits, as at initial connect. + connection.emit('tools-list-changed'); + assert.deepStrictEqual(controlEvents, ['tools-list-changed'], 'control traffic is held after the boundary'); + connection.ready(); + assert.deepStrictEqual(controlEvents, ['tools-list-changed', 'tools-list-changed'], 'and released when the planes open'); + }); }); diff --git a/test/mcpl-reconnect-registration.test.ts b/test/mcpl-reconnect-registration.test.ts new file mode 100644 index 000000000..d194978bf --- /dev/null +++ b/test/mcpl-reconnect-registration.test.ts @@ -0,0 +1,96 @@ +/** + * A reconnected MCPL server's control traffic waits for its new grant. + * + * A transport boundary resets the server's grant: a fresh initialize is a + * fresh §5.3 policy epoch. A chat bridge registers its channels right after + * initialize, before the host's policy Request has been answered. If that + * registration reaches admission while the grant is still empty, it is + * refused (-32002) and never replayed, and the route's incoming messages are + * then unknown channels until the server happens to register again. Initial + * connect holds both planes until the grant is established; a reconnect must + * hold them too, both for a connection that was already open (an ordinary + * disconnect) and for a stub that had never opened (a first reconnect while + * startup is still staging other servers). + */ +import { describe, it, beforeEach, afterEach } from 'node:test'; +import assert from 'node:assert/strict'; +import { fileURLToPath } from 'node:url'; +import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { AgentFramework } from '../src/index.js'; +import { MockMembrane } from './helpers/mock-membrane.js'; + +const FIXTURE = fileURLToPath(new URL('./fixtures/reregistering-mcpl-server.mjs', import.meta.url)); + +interface ServerEvent { + event: string; + pid: number; + error?: { code: number; message: string } | null; +} + +function serverEvents(path: string): ServerEvent[] { + if (!existsSync(path)) return []; + return readFileSync(path, 'utf8').split('\n').filter(Boolean).map((line) => JSON.parse(line) as ServerEvent); +} + +async function waitFor(description: string, predicate: () => boolean, timeoutMs = 10_000): Promise { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + if (predicate()) return; + await new Promise((resolve) => setTimeout(resolve, 10)); + } + assert.fail(`timed out waiting for ${description}`); +} + +describe('MCPL reconnect and channel registration', () => { + let dir: string; + let framework: AgentFramework | undefined; + + beforeEach(() => { + dir = mkdtempSync(join(tmpdir(), 'mcpl-reconnect-registration-')); + }); + + afterEach(async () => { + await framework?.stop(); + framework = undefined; + rmSync(dir, { recursive: true, force: true }); + }); + + it('a server that registers right after initialize is registered again after an ordinary reconnect', async () => { + const eventsPath = join(dir, 'events.jsonl'); + const crashPath = join(dir, 'crash'); + framework = await AgentFramework.create({ + storePath: join(dir, 'store'), + membrane: new MockMembrane().asMembrane(), + agents: [{ name: 'resident', model: 'test-model', systemPrompt: 'You are a resident.' }], + modules: [], + maintenanceIntervalMs: 0, + mcplServers: [{ + id: 'chat', + command: process.execPath, + args: [FIXTURE], + env: { EVENTS_PATH: eventsPath, CRASH_PATH: crashPath }, + enabledFeatureSets: ['chat'], + reconnect: true, + reconnectIntervalMs: 50, + }], + }); + const answers = () => serverEvents(eventsPath).filter((e) => e.event === 'register-answered'); + + await waitFor('the first registration answer', () => answers().length === 1); + assert.equal(answers()[0].error, null, 'initial connect admits the registration against the established grant'); + + // The server process dies; the host reconnects to a fresh one, which + // registers again before the host's new policy Request is answered. + writeFileSync(crashPath, ''); + await waitFor('the registration answer after the reconnect', () => answers().length === 2); + const [first, second] = answers(); + assert.notEqual(second.pid, first.pid, 'a fresh server process registered'); + assert.equal( + second.error, + null, + `the reconnected server's registration waits for its new grant instead of being refused: ${JSON.stringify(second.error)}`, + ); + }); +}); From 25891f07cd0160fc8171168a7afbb21dba657b27 Mon Sep 17 00:00:00 2001 From: Ansel-1606 Date: Wed, 7 Oct 2026 16:44:53 -0400 Subject: [PATCH 26/73] fix(recover): the CLI records its awareness controls and recoveries in the operator log Greptile review of #250 (P2 "Offline controls skip the log"), with the extension agreed in room-245 (#48091, #48115): the CLI's own recovery. The live surfaces record every operator mutation in /operator-actions.jsonl (who asked, from where, what it did, or the error), and the changelog promises that each awareness control is recorded. agent-framework-recover --awareness cancel|retract|release called the outbox directly and only printed the receipt, so the operator history lost every offline control. The CLI's recovery, which #250 lets publish marks (--marks), was not recorded either, so nothing said who chose those marks. The CLI now appends, best-effort (OperatorLog.append never throws, and the OS account lookup falls back to {via: 'cli'}): - for cancel/retract/release: kind awareness-, agent '*', params {target} or {batchId} as the live wrapper logs them, and the receipt or the error; - for a recovery: kind 'recovery', the agent, params (anchor, any suppressions, the marks choice as 'none' or {scope}), and the result (branches, messages removed and suppressed, discordRefs, markers), or the error when it is refused or fails. The requester is {via: 'cli', name: }, as the web UI records its basic-auth user. A dry run and a list record nothing. The log path is the host's default under the store. Tests: test/recover-cli-operator-log.test.ts spawns the compiled CLI: list records nothing; cancel and retract record their receipts; a refused release records its error; a dry run records nothing; a recovery records its marks choice and the receipt it printed; a refused recovery (existing branch) records its error. Docs: the CLI help, README's offline recovery section, and the controls changelog fragment. --- README.md | 5 +- changelog.d/surgery-marks-controls.added.md | 5 + src/recovery/recover-cli.ts | 125 +++++++++++++++---- test/recover-cli-operator-log.test.ts | 131 ++++++++++++++++++++ 4 files changed, 243 insertions(+), 23 deletions(-) create mode 100644 test/recover-cli-operator-log.test.ts diff --git a/README.md b/README.md index 68b6548bb..311852747 100644 --- a/README.md +++ b/README.md @@ -173,7 +173,10 @@ awareness journal kept in the store itself, and the marks are delivered when the host and that agent's `discord-mcpl` bot next connect (see [Live surgery and Discord awareness marks](#live-surgery-and-discord-awareness-marks)). If the branch is made but the marks can't be recorded, the output's `markers` -says so and the branch stands. +says so and the branch stands. Like a live surgery, the recovery (with its +marks choice and receipt) is recorded in the store's operator log, +`operator-actions.jsonl`, with the OS account that ran it; so is each +`--awareness` cancel, retract and release. A dry run records nothing. Portal and other non-Discord records are ignored. When the safe point is an assistant/tool-side record rather than a Discord diff --git a/changelog.d/surgery-marks-controls.added.md b/changelog.d/surgery-marks-controls.added.md index 175fb7539..45c6cc924 100644 --- a/changelog.d/surgery-marks-controls.added.md +++ b/changelog.d/surgery-marks-controls.added.md @@ -22,3 +22,8 @@ available as the `host/command` `marks` verb (`action: list | cancel | retract | release`, `target`) and, with the host stopped, as `agent-framework-recover --awareness list|cancel|retract|release`. +- `agent-framework-recover` records its acts in the store's operator log + (`/operator-actions.jsonl`), done or refused, with the OS account + that ran it: each `--awareness` cancel, retract and release, and each + recovery with its marks choice and receipt (kind `recovery`). A dry run or + a list records nothing. diff --git a/src/recovery/recover-cli.ts b/src/recovery/recover-cli.ts index 9191cea2a..9735e3267 100644 --- a/src/recovery/recover-cli.ts +++ b/src/recovery/recover-cli.ts @@ -1,5 +1,6 @@ #!/usr/bin/env node +import { userInfo } from 'node:os'; import { JsStore } from '@animalabs/chronicle'; import { createOfflineRecoveryBranch } from './offline-branch.js'; import { @@ -7,6 +8,12 @@ import { defaultDiscordAwarenessOutboxPath, type DiscordAwarenessMarks, } from './discord-awareness-outbox.js'; +import { + OperatorLog, + defaultOperatorLogPath, + type OperatorLogInput, + type OperatorRequester, +} from '../operator-log.js'; interface CliOptions { storePath?: string; @@ -68,6 +75,9 @@ Awareness journal (host stopped): --awareness release Queue a held batch's recorded operations Queued work is delivered when the host next connects to the Discord server. +Each recovery and each cancel, retract or release (done or refused; not a +dry run or a list) is recorded in /operator-actions.jsonl, the host's +operator log, with the OS account that ran it. The agent host must be stopped before running this command.`; } @@ -138,6 +148,28 @@ function parseArgs(args: string[]): CliOptions { return options; } +/** Who ran the command, as far as the CLI can say: the OS account. */ +function cliRequester(): OperatorRequester { + try { + return { via: 'cli', name: userInfo().username }; + } catch { + return { via: 'cli' }; + } +} + +/** + * Record an act in the store's operator log (`/operator-actions.jsonl`, + * where the host records its own), whether it was done or refused, like the + * live surfaces' acts. Best-effort: a logging failure never fails the act. + */ +function recordOperatorAction(storePath: string, entry: OperatorLogInput): void { + new OperatorLog(defaultOperatorLogPath(storePath)).append(entry); +} + +function errorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} + function awareness(options: CliOptions): void { // The journal lives in the store; with the host stopped, this process is // its only writer. @@ -147,14 +179,28 @@ function awareness(options: CliOptions): void { legacyPath: options.outboxPath ?? defaultDiscordAwarenessOutboxPath(options.storePath!), }); const { verb, target } = options.awareness!; + if (verb === 'list') { + console.log(JSON.stringify({ store: options.storePath, batches: outbox.view() }, null, 2)); + return; + } const by = 'agent-framework-recover'; - let result: unknown; - switch (verb) { - case 'list': result = { store: options.storePath, batches: outbox.view() }; break; - case 'cancel': result = outbox.cancel(target!, by); break; - case 'retract': result = outbox.retract(target!, by); break; - case 'release': result = outbox.release(target!, by); break; + // Logged as the live controls log theirs. + const logBase: OperatorLogInput = { + kind: `awareness-${verb}`, + agent: '*', + requester: cliRequester(), + params: verb === 'release' ? { batchId: target } : { target }, + }; + let result: object; + try { + result = verb === 'cancel' ? outbox.cancel(target!, by) + : verb === 'retract' ? outbox.retract(target!, by) + : outbox.release(target!, by); + } catch (error) { + recordOperatorAction(options.storePath!, { ...logBase, error: errorMessage(error) }); + throw error; } + recordOperatorAction(options.storePath!, { ...logBase, result: { ...result } }); console.log(JSON.stringify(result, null, 2)); } finally { store.close(); @@ -181,22 +227,57 @@ async function main(): Promise { // A CLI choice and its application are the same moment (the host is // stopped), so the scope is evaluated now rather than frozen to refs. const marks: DiscordAwarenessMarks = options.marks === 'none' ? 'none' : { scope: options.marks }; - const result = await createOfflineRecoveryBranch({ - storePath: options.storePath, - agentName: options.agentName, - messageId: options.messageId, - contextId: options.contextId, - messages: options.messages, - suppressMessageIds: options.suppressMessageIds, - suppressRanges: options.suppressRanges, - branchName: options.branchName, - namespace: options.namespace, - discordServerId: options.discordServerId, - outboxPath: options.outboxPath, - emoji: options.emoji, - marks, - dryRun: options.dryRun, - }); + // A real recovery is recorded in the operator log, done or failed, with + // its marks choice and receipt, as a live rollback or suppression is. A + // dry run changes nothing and records nothing. + const logBase: OperatorLogInput = { + kind: 'recovery', + agent: options.agentName, + requester: cliRequester(), + params: { + ...(options.messageId !== undefined ? { messageId: options.messageId } : {}), + ...(options.contextId !== undefined ? { contextId: options.contextId } : {}), + ...(options.messages !== undefined ? { messages: options.messages } : {}), + ...(options.suppressMessageIds.length > 0 ? { suppressMessageIds: options.suppressMessageIds } : {}), + ...(options.suppressRanges.length > 0 ? { suppressRanges: options.suppressRanges } : {}), + marks: marks === 'none' ? 'none' : { scope: marks.scope }, + }, + }; + let result: Awaited>; + try { + result = await createOfflineRecoveryBranch({ + storePath: options.storePath, + agentName: options.agentName, + messageId: options.messageId, + contextId: options.contextId, + messages: options.messages, + suppressMessageIds: options.suppressMessageIds, + suppressRanges: options.suppressRanges, + branchName: options.branchName, + namespace: options.namespace, + discordServerId: options.discordServerId, + outboxPath: options.outboxPath, + emoji: options.emoji, + marks, + dryRun: options.dryRun, + }); + } catch (error) { + if (!options.dryRun) recordOperatorAction(options.storePath, { ...logBase, error: errorMessage(error) }); + throw error; + } + if (!result.dryRun) { + recordOperatorAction(options.storePath, { + ...logBase, + result: { + sourceBranch: result.sourceBranch, + targetBranch: result.targetBranch, + messagesRemoved: result.messagesRemoved, + messagesSuppressed: result.messagesSuppressed, + discordRefs: result.discordAddressable, + ...(result.markers ? { markers: result.markers } : {}), + }, + }); + } console.log(JSON.stringify({ ...result, diff --git a/test/recover-cli-operator-log.test.ts b/test/recover-cli-operator-log.test.ts new file mode 100644 index 000000000..1c0202e37 --- /dev/null +++ b/test/recover-cli-operator-log.test.ts @@ -0,0 +1,131 @@ +/** + * `agent-framework-recover` records its acts in the store's operator log. + * + * The live surfaces record every operator mutation (who asked, from where, + * what it did) in `/operator-actions.jsonl`. The CLI acts on the same + * store while the host is stopped: an offline recovery, which can choose to + * publish awareness marks, and the awareness controls (cancel, retract, + * release). Each is recorded the same way, done or refused, so the history + * says who chose to publish or stop marks whichever surface they used. A dry + * run and a list change nothing and record nothing. + */ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { spawnSync } from 'node:child_process'; +import { existsSync, mkdtempSync, readFileSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { JsStore } from '@animalabs/chronicle'; +import { ContextManager } from '@animalabs/context-manager'; +import { DiscordAwarenessOutbox } from '../src/recovery/discord-awareness-outbox.js'; +import type { OperatorLogEntry } from '../src/operator-log.js'; + +const CLI = fileURLToPath(new URL('../src/recovery/recover-cli.js', import.meta.url)); + +function recover(storePath: string, ...args: string[]) { + const run = spawnSync(process.execPath, [CLI, '--store', storePath, ...args], { encoding: 'utf8' }); + return { status: run.status, stdout: run.stdout, stderr: run.stderr }; +} + +function operatorLog(storePath: string): OperatorLogEntry[] { + const path = join(storePath, 'operator-actions.jsonl'); + if (!existsSync(path)) return []; + return readFileSync(path, 'utf8').split('\n').filter(Boolean).map((line) => JSON.parse(line) as OperatorLogEntry); +} + +const ref = (messageId: string) => ({ serverId: 'discord', channelId: 'discord:g1:c1', messageId }); + +test('the offline awareness controls record each act in the operator log, done or refused', () => { + const dir = mkdtempSync(join(tmpdir(), 'recover-cli-log-')); + const storePath = join(dir, 'store'); + try { + const store = JsStore.openOrCreate({ path: storePath }); + const outbox = new DiscordAwarenessOutbox(store); + const batch = outbox.prepare({ + agentName: 'cairn', sourceBranch: 'main', targetBranch: 'rollback/cairn/1', refs: [ref('m1'), ref('m2')], scope: 'all', + })!; + outbox.activate(batch.id); + store.close(); + + const listed = recover(storePath, '--awareness', 'list'); + assert.equal(listed.status, 0, listed.stderr); + assert.deepEqual(operatorLog(storePath), [], 'a list changes nothing and records nothing'); + + const cancelled = recover(storePath, '--awareness', 'cancel', batch.id); + assert.equal(cancelled.status, 0, cancelled.stderr); + const receipt = JSON.parse(cancelled.stdout) as Record; + assert.equal(receipt.cancelled, 2); + + const retracted = recover(storePath, '--awareness', 'retract', 'all'); + assert.equal(retracted.status, 0, retracted.stderr); + + const refused = recover(storePath, '--awareness', 'release', batch.id); + assert.equal(refused.status, 1, 'releasing a batch that is not held is refused'); + + const entries = operatorLog(storePath); + assert.deepEqual(entries.map((entry) => entry.kind), ['awareness-cancel', 'awareness-retract', 'awareness-release']); + for (const entry of entries) { + assert.equal(entry.agent, '*'); + assert.equal(entry.requester?.via, 'cli'); + assert.ok(entry.at); + } + assert.deepEqual(entries[0].params, { target: batch.id }); + assert.deepEqual(entries[0].result, receipt); + assert.deepEqual(entries[1].params, { target: 'all' }); + assert.equal(entries[1].result?.removalsQueued, 2); + assert.deepEqual(entries[2].params, { batchId: batch.id }); + assert.equal(entries[2].result, undefined); + assert.match(entries[2].error ?? '', /is active, not held/); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); + +test('an offline recovery records its marks choice and receipt in the operator log; a dry run records nothing', async () => { + const dir = mkdtempSync(join(tmpdir(), 'recover-cli-log-')); + const storePath = join(dir, 'store'); + try { + const store = JsStore.openOrCreate({ path: storePath }); + const cm = await ContextManager.open({ store, namespace: 'agents/cairn' }); + cm.addMessage('user', [{ type: 'text', text: 'safe' }], { ...ref('m-safe') }); + cm.addMessage('user', [{ type: 'text', text: 'one' }], { ...ref('m-1'), tags: ['chat:addressed'] }); + cm.addMessage('user', [{ type: 'text', text: 'two' }], { ...ref('m-2'), tags: ['chat:ambient'] }); + cm.close(); + store.close(); + + const dry = recover(storePath, '--agent', 'cairn', '--message-id', 'm-safe', '--marks', 'addressed', '--dry-run'); + assert.equal(dry.status, 0, dry.stderr); + assert.deepEqual(operatorLog(storePath), [], 'a dry run changes nothing and records nothing'); + + const done = recover(storePath, '--agent', 'cairn', '--message-id', 'm-safe', '--marks', 'addressed', '--branch', 'recovery/cairn/1'); + assert.equal(done.status, 0, done.stderr); + const output = JSON.parse(done.stdout) as { markers: Record }; + + const again = recover(storePath, '--agent', 'cairn', '--message-id', 'm-safe', '--branch', 'main'); + assert.equal(again.status, 1, 'a recovery onto an existing branch is refused'); + + const [recorded, refused, ...rest] = operatorLog(storePath); + assert.equal(rest.length, 0); + assert.equal(recorded.kind, 'recovery'); + assert.equal(recorded.agent, 'cairn'); + assert.equal(recorded.requester?.via, 'cli'); + assert.deepEqual(recorded.params, { messageId: 'm-safe', marks: { scope: 'addressed' } }); + assert.deepEqual(recorded.result, { + sourceBranch: 'main', + targetBranch: 'recovery/cairn/1', + messagesRemoved: 2, + messagesSuppressed: 0, + discordRefs: 2, + markers: output.markers, + }); + assert.equal((recorded.result?.markers as { status: string; queued: number }).status, 'queued'); + assert.equal((recorded.result?.markers as { status: string; queued: number }).queued, 1); + + assert.equal(refused.kind, 'recovery'); + assert.deepEqual(refused.params, { messageId: 'm-safe', marks: 'none' }); + assert.match(refused.error ?? '', /already exists/); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); From 75d7eba10d1de0e9f3d7df5419b24946a128e996 Mon Sep 17 00:00:00 2001 From: Ansel-1606 Date: Wed, 7 Oct 2026 16:46:27 -0400 Subject: [PATCH 27/73] fix(awareness): a turn undo records its marks choice and receipt in the operator log Greptile review of #250 (P2 "Turn marks aren't logged"). Turn-based host/command undo accepts a marks choice and schedules marks for the messages its turns removed, but its only operator-log entries came from undoLastTurn(), one per turn, with the turn and branches. So nothing recorded who chose to publish (or not to publish) marks, while hide and rollback (message-granular undo) record their choice and receipt. After scheduling, the command now records one entry: kind 'undo-turns', the requester, params {turns: requested, marks: none | {scope}}, and result {undone, markers}. It is written whenever at least one turn was undone, `marks: none` included (agreed with Mira-1605: the local-only choice is worth recording, as hide and rollback do); a command that undid nothing changed nothing and records nothing. The per-turn undo-turn entries are unchanged. Test (test/discord-awareness-outbox.test.ts): a marked turn undo, a local one, and one with nothing left to undo log exactly two entries, each with the receipt the command returned. --- changelog.d/surgery-marks-controls.added.md | 4 ++ src/framework.ts | 10 ++++ test/discord-awareness-outbox.test.ts | 53 +++++++++++++++++++++ 3 files changed, 67 insertions(+) diff --git a/changelog.d/surgery-marks-controls.added.md b/changelog.d/surgery-marks-controls.added.md index 45c6cc924..51484e58b 100644 --- a/changelog.d/surgery-marks-controls.added.md +++ b/changelog.d/surgery-marks-controls.added.md @@ -22,6 +22,10 @@ available as the `host/command` `marks` verb (`action: list | cancel | retract | release`, `target`) and, with the host stopped, as `agent-framework-recover --awareness list|cancel|retract|release`. +- A turn-based `host/command` `undo` that undoes a turn records one + `undo-turns` entry in the operator log: the requested turns, the marks + choice (`none` included) and the receipt, beside the per-turn `undo-turn` + entries. - `agent-framework-recover` records its acts in the store's operator log (`/operator-actions.jsonl`), done or refused, with the OS account that ran it: each `--awareness` cancel, retract and release, and each diff --git a/src/framework.ts b/src/framework.ts index 60d80bbf6..da8c24419 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -5476,6 +5476,16 @@ export class AgentFramework { const after = new Set(this.addressingSnapshot(agentName).keys()); const removed = [...before].filter(([id]) => !after.has(id)).map(([, carrier]) => carrier); markers = this.scheduleAppliedMarks('undo', agentName, serverId, removed, turnMarks); + // Each turn is logged by undoLastTurn; the command's marks choice + // (`none` included) and what it scheduled are logged here, as hide and + // rollback log theirs. + this.recordOperatorAction({ + kind: 'undo-turns', + agent: agentName, + requester: hostCommandRequester(serverId, params), + params: { turns: requested, marks: describeMarksChoice(turnMarks) }, + result: { undone, markers }, + }); } finally { release(); } diff --git a/test/discord-awareness-outbox.test.ts b/test/discord-awareness-outbox.test.ts index 34a46265b..6329b2c77 100644 --- a/test/discord-awareness-outbox.test.ts +++ b/test/discord-awareness-outbox.test.ts @@ -895,6 +895,59 @@ test('turn-based host/command undo honours the marks choice for the messages its } }); +test('a turn undo records its marks choice and what it scheduled in the operator log, also for marks none', async () => { + const h = storeHarness(); + try { + const messages = [ + { id: 'i0', metadata: { ...ref('m0') } }, + { id: 'i1', metadata: { ...ref('m1'), tags: ['chat:addressed'] } }, + { id: 'i2', metadata: { ...ref('m2') } }, + ]; + const { framework, removed } = hostCommandFramework(h, messages); + // Each stubbed turn undo removes the next of these from the context. + let undoable = ['i1', 'i2']; + framework.undoLastTurn = () => { + const next = undoable[0]; + if (!next) return { undone: false }; + undoable = undoable.slice(1); + removed.add(next); + return { undone: true }; + }; + const logged: Array> = []; + framework.recordOperatorAction = (entry: Record) => { logged.push(entry); return entry; }; + let marked: any; + let local: any; + await muted(async () => { + marked = await framework.handleHostCommand('discord', { + command: 'undo', agentName: 'cairn', turns: 1, marks: 'addressed', requesterName: 'op', + }); + local = await framework.handleHostCommand('discord', { command: 'undo', agentName: 'cairn', turns: 3 }); + const nothing = await framework.handleHostCommand('discord', { command: 'undo', agentName: 'cairn', turns: 1, marks: 'all' }); + assert.equal(nothing.undone, 0); + }); + assert.equal(marked.markers.status, 'queued'); + assert.equal(local.markers.status, 'none'); + assert.deepEqual(logged, [ + { + kind: 'undo-turns', + agent: 'cairn', + requester: { via: 'host-command:discord', name: 'op' }, + params: { turns: 1, marks: { scope: 'addressed' } }, + result: { undone: 1, markers: marked.markers }, + }, + { + kind: 'undo-turns', + agent: 'cairn', + requester: { via: 'host-command:discord' }, + params: { turns: 3, marks: 'none' }, + result: { undone: 1, markers: local.markers }, + }, + ], 'one entry per command that undid a turn, with the receipt it returned; none when nothing was undone'); + } finally { + h.cleanup(); + } +}); + test('host/command hide takes the store reservation and marks only on request, through the journal', async () => { const h = storeHarness(); try { From 538b136cfd040a7b5fe9e6e8b0b19a12ddb2b243 Mon Sep 17 00:00:00 2001 From: Ansel-1606 Date: Wed, 7 Oct 2026 16:52:17 -0400 Subject: [PATCH 28/73] fix(awareness): stop checkpointing the awareness journal; replay every entry Greptile review of #250 (P2 "Checkpoints repeatedly copy all history"). Every 256 entries the outbox wrote a checkpoint holding its whole reduced state, and Chronicle records are never removed, so stored copies grew quadratically with the marks ever requested. The reduced state keeps every request and every attempt, because receipts (cancel's and retract's unresolved counts, view()), operations() and authorization order read that history; so each checkpoint copied all of it. Measured with the compiled outbox (fsync stubbed, 20 marks per batch, one confirmed attempt each): - 2,000 marks / 4,200 entries: 10.8 MB with checkpoints, 1.65 MB without; opening took 0.14 s and 0.11 s; - 8,000 marks / 16,800 entries: 149.6 MB vs 6.6 MB; opening took 0.70 s with checkpoints and 1.43 s without; - 20,000 marks: 918.5 MB with checkpoints (the run without them was not completed). Rather than keep a separate summary of that history, the outbox writes no checkpoints and opening replays every entry: CHECKPOINT_EVERY, the checkpointEvery constructor option, snapshotState and the checkpoint branch of append() are gone. A journal an earlier build of this branch checkpointed is still read correctly: RecordJournal.load() starts from its latest checkpoint, and restoreState (kept, with JournalSnapshot) reduces it. RecordJournal itself is unchanged; it is the base this PR shares with #247. Agreed with Mira-1605 in room-245. Tests (test/discord-awareness-outbox.test.ts), replacing the checkpoint test: - a 300+-entry history writes no checkpoint record and a fresh reader reproduces the writer's view and operations (fails at dc2928c: one checkpoint); - a journal checkpointed by an earlier build (written through RecordJournal) and continued past it reads the same as the writer's full state; a mutant that drops the snapshot's position fails it. --- src/recovery/discord-awareness-outbox.ts | 47 +++++++------------ test/discord-awareness-outbox.test.ts | 57 +++++++++++++++++++----- 2 files changed, 61 insertions(+), 43 deletions(-) diff --git a/src/recovery/discord-awareness-outbox.ts b/src/recovery/discord-awareness-outbox.ts index d49f8fce2..8850c3df4 100644 --- a/src/recovery/discord-awareness-outbox.ts +++ b/src/recovery/discord-awareness-outbox.ts @@ -493,7 +493,17 @@ interface JournalEntryGroup { records: JournalRecord[]; } -/** The reduced state as a checkpoint stores it. */ +/** + * The reduced state as a checkpoint stores it. This journal writes no + * checkpoints. Its reduced state keeps every request and every attempt, + * because receipts, `operations()` and authorization order read that + * history, so a snapshot of it copies all of it, and copying it every N + * entries grows the store quadratically with the marks ever requested. + * Rather than keep a separate summary of that history, opening replays every + * entry. A journal an earlier build of this journal checkpointed is still + * read from its latest checkpoint, which is where RecordJournal.load() + * starts. + */ interface JournalSnapshot { batches: DiscordAwarenessBatch[]; ops: DiscordAwarenessOp[]; @@ -504,8 +514,6 @@ interface JournalSnapshot { } export const DISCORD_AWARENESS_RECORD_TYPE = 'af:discord-awareness'; -/** Replay at open is bounded by a checkpoint every this many entries. */ -const CHECKPOINT_EVERY = 256; interface JournalState { batches: Map; @@ -539,7 +547,6 @@ interface RetractRequest { export class DiscordAwarenessOutbox { private readonly journal: RecordJournal; private readonly legacyPath?: string; - private readonly checkpointEvery: number; /** Reduced state; rebuilt from the journal when null. */ private state: JournalState | null = null; /** Operations dispatched by this process and still awaiting an outcome. */ @@ -548,12 +555,10 @@ export class DiscordAwarenessOutbox { /** * @param store the Chronicle store the journal lives in. * @param opts.legacyPath a pre-journal JSON ledger to import once, if present. - * @param opts.checkpointEvery entries between checkpoints (default 256). */ - constructor(store: JsStore, opts: { legacyPath?: string; checkpointEvery?: number } = {}) { + constructor(store: JsStore, opts: { legacyPath?: string } = {}) { this.journal = new RecordJournal(store, { type: DISCORD_AWARENESS_RECORD_TYPE }); if (opts.legacyPath) this.legacyPath = opts.legacyPath; - this.checkpointEvery = Math.max(1, Math.floor(opts.checkpointEvery ?? CHECKPOINT_EVERY)); } // -- surgery side ----------------------------------------------------------- @@ -1270,8 +1275,9 @@ export class DiscordAwarenessOutbox { /** * The reduced state, rebuilt from the journal when there is none yet or an * earlier write left the journal unreconciled (RecordJournal then requires - * a reload before writing again). The first load also imports a - * pre-journal ledger, once. + * a reload before writing again): every entry is replayed, after an + * earlier build's checkpoint if there is one (see JournalSnapshot). The + * first load also imports a pre-journal ledger, once. */ private load(): JournalState { if (this.state && !this.journal.needsReconcile) return this.state; @@ -1338,17 +1344,6 @@ export class DiscordAwarenessOutbox { throw error; } applyRecords(state, records); - if (this.journal.entriesSinceCheckpoint >= this.checkpointEvery) { - try { - this.journal.checkpoint(snapshotState(state)); - } catch (error) { - // Replay just stays longer; the journal reloads before its next write. - this.state = null; - console.error( - `[discord-awareness] checkpoint failed: ${error instanceof Error ? error.message : String(error)}`, - ); - } - } } } @@ -1366,17 +1361,7 @@ function liveRequests(records: JournalRecord[]): number { return records.filter((record) => record.t === 'requested' && !cancelled.has(record.opId)).length; } -function snapshotState(state: JournalState): JournalSnapshot { - return { - batches: [...state.batches.values()].map((batch) => structuredClone(batch)), - ops: [...state.ops.values()].map((op) => structuredClone(op)), - legacy: [...state.legacy.values()].flat().map((evidence) => structuredClone(evidence)), - retracts: [...state.retracts.values()].map((retract) => structuredClone(retract)), - importedSources: [...state.importedSources], - position: state.position, - }; -} - +/** The state an earlier build's checkpoint recorded (see JournalSnapshot). */ function restoreState(snapshot: JournalSnapshot): JournalState { const state: JournalState = { batches: new Map(snapshot.batches.map((batch) => [batch.id, structuredClone(batch)])), diff --git a/test/discord-awareness-outbox.test.ts b/test/discord-awareness-outbox.test.ts index 6329b2c77..7e47bb3f0 100644 --- a/test/discord-awareness-outbox.test.ts +++ b/test/discord-awareness-outbox.test.ts @@ -29,6 +29,7 @@ import { import { AgentFramework } from '../src/framework.js'; import { McplRequestError } from '../src/mcpl/server-connection.js'; import { OperatorLog } from '../src/operator-log.js'; +import { RecordJournal } from '../src/record-journal.js'; const ref = (messageId: string, channel = 'c1'): DiscordAwarenessRef => ({ serverId: 'discord', channelId: `discord:g1:${channel}`, messageId }); @@ -511,22 +512,54 @@ test('a corrupt journal entry stops the reader rather than being skipped', withJ assert.throws(() => h.reopen().batches(), /Corrupt Discord awareness journal entry/); })); -test('checkpoints bound replay and reproduce the reduced state', withJournal((_outbox, h) => { - const outbox = new DiscordAwarenessOutbox(h.store, { checkpointEvery: 8 }); - const batch = activeBatch(outbox, Array.from({ length: 5 }, (_, i) => ref(`m${i}`))); - for (let round = 0; round < 3; round++) { - for (let i = 0; i < 5; i++) { - const claimed = outbox.claimDispatch('discord', (d) => d.key.messageId !== `m${i}`); - if (claimed) outbox.recordOutcome(claimed.attempts, round === 2 ? 'confirmed' : 'failed', { error: 'Discord 503' }); - } +test('the journal writes no checkpoints: a long history is replayed whole', withJournal((_outbox, h) => { + // Every reduction needs the whole history, so a checkpoint would copy all + // of it, and copying it every N entries grows the store quadratically. + // (Durability is not under test here: skip the per-entry fsync.) + const unsynced = new Proxy(h.store, { + get(target, prop) { + if (prop === 'sync') return () => {}; + const value = Reflect.get(target, prop); + return typeof value === 'function' ? value.bind(target) : value; + }, + }); + const outbox = new DiscordAwarenessOutbox(unsynced); + const batch = activeBatch(outbox, Array.from({ length: 150 }, (_, i) => ref(`m${i}`))); + for (let claimed = outbox.claimDispatch('discord'); claimed; claimed = outbox.claimDispatch('discord')) { + outbox.recordOutcome(claimed.attempts, 'confirmed'); } outbox.retract(batch.id); - assert.ok(h.store.getRecordIdsByType(`${DISCORD_AWARENESS_RECORD_TYPE}/checkpoint`).length >= 2, 'checkpoints were written'); - const reread = new DiscordAwarenessOutbox(h.store, { checkpointEvery: 8 }); + const entries = h.store.getRecordIdsByType(DISCORD_AWARENESS_RECORD_TYPE).length; + assert.ok(entries > 256, `${entries} entries: more than an earlier build wrote between checkpoints`); + assert.equal(h.store.getRecordIdsByType(`${DISCORD_AWARENESS_RECORD_TYPE}/checkpoint`).length, 0); + const reread = h.reopen(); + assert.deepEqual(reread.view(), outbox.view()); + assert.deepEqual(reread.operations(), outbox.operations()); +})); + +test('a journal an earlier build checkpointed is read from that checkpoint and continues the same', withJournal((outbox, h) => { + const batch = activeBatch(outbox, [ref('m1'), ref('m2'), ref('m3')]); + answer(outbox, 'm1', 'confirmed'); + answer(outbox, 'm2', 'unknown'); + // An earlier build checkpointed its reduced state here, with the journal + // position its next record would follow. + const earlier = new RecordJournal<{ records: unknown[] }, unknown>(h.store, { type: DISCORD_AWARENESS_RECORD_TYPE }); + const { entries } = earlier.load(); + earlier.checkpoint({ + batches: outbox.batches(), + ops: outbox.operations(), + legacy: [], + retracts: [], + importedSources: [], + position: entries.reduce((sum, { entry }) => sum + entry.records.length, 0), + }); + // History continues past it: a retract, whose removals' authorization is + // its position after the checkpointed ones. + outbox.retract(batch.id); + answer(outbox, 'm3', 'confirmed'); + const reread = h.reopen(); assert.deepEqual(reread.view(), outbox.view()); assert.deepEqual(reread.operations(), outbox.operations()); - assert.equal(batches(reread).find((v) => v.id === batch.id)!.adds.confirmed, 5); - assert.equal(retracts(reread).length, 1); })); test('v2 import keeps what the old ledger recorded as evidence, holds pending work, and happens once', withJournal((_outbox, h) => { From a5439e20805d3fdafe8978c63482ddbc1f8fa869 Mon Sep 17 00:00:00 2001 From: Ansel-1606 Date: Wed, 7 Oct 2026 17:13:58 -0400 Subject: [PATCH 29/73] fix(mcpl): no other plane opener releases a reconnect's traffic before its policy exchange settles Review of 9d17ad6 by Mira-1605 (room-245, #48850), reproduced: Greptile's first-reconnect scenario still lost its registration. A stub whose first connect failed reconnects while startup is still staging another server; its reconnect listener correctly waits for its policy exchange, but initializeMcpl's final readyMcplPlanes() opens every connection, including this one, mid-exchange, and flushes its buffered channels/register against the empty grant (-32002, "initial policy not yet established"). resume()'s readyMcplPlanes() is the same kind of caller. The framework now marks, per connection, the transport epoch whose policy exchange a reconnect has in flight (mcplPolicyExchanges, a WeakMap keyed by the connection object, so a replacement connection under the same id is unaffected). readyMcplPlanes skips a connection whose current epoch's exchange is pending; the reconnect listener clears its mark and opens the planes once the exchange settles, granted, refused or unanswered (so an MCP-only connection and host/command still come up, as at initial connect), unless a newer transport's exchange has replaced its mark, in which case that one opens them. Tests (test/mcpl-reconnect-registration.test.ts): - Mira's reproduction as a deterministic integration test: `retry` fails its first connect and holds its retry's policy answer; `stager` answers initialize only once that policy is pending, so startup's open of every plane runs mid-exchange. Its registration must stay unanswered until the policy is released, then be admitted after it. Without this commit it is answered -32002 at startup. - Unit cases on a framework stub with deferred policy answers: a global readyMcplPlanes() during the exchange opens nothing, and the exchange's settlement opens once; a superseded exchange never opens the planes of the transport that replaced it. Both fail without this commit. The fixture gains file-based coordination (fail-first, initialize-after, policy pending/release). Object.create harnesses that emit 'reconnect' initialize the new field (mcpl-reconnect-reregistration). --- .../mcpl-reconnect-registration.fixed.md | 3 +- src/framework.ts | 36 ++++-- test/fixtures/reregistering-mcpl-server.mjs | 58 ++++++++-- test/mcpl-reconnect-registration.test.ts | 108 ++++++++++++++++++ test/mcpl-reconnect-reregistration.test.ts | 1 + 5 files changed, 187 insertions(+), 19 deletions(-) diff --git a/changelog.d/mcpl-reconnect-registration.fixed.md b/changelog.d/mcpl-reconnect-registration.fixed.md index b52983a18..daaff2abc 100644 --- a/changelog.d/mcpl-reconnect-registration.fixed.md +++ b/changelog.d/mcpl-reconnect-registration.fixed.md @@ -6,4 +6,5 @@ stayed unknown channels until the server registered again. A reconnect now holds control traffic as well as data until the new grant is established, as initial connect already did, both for a connection that - was open and for one whose first connect had failed. + was open and for one whose first connect had failed, and even when + startup or a quiesce resume opens every connection meanwhile. diff --git a/src/framework.ts b/src/framework.ts index da8c24419..ab4c36cc6 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -13874,14 +13874,26 @@ export class AgentFramework { } /** - * Open MCPL connections' planes (every connection when none is given). A - * quiesced host (issue #122) opens only control planes, so data planes stay - * held for the whole maintenance window; resume() calls this again once - * the flag is cleared. Control planes always come up, since host/command - * (and with it resume) rides the control plane. + * Per connection, the transport epoch whose §5.3 policy exchange a + * reconnect has in flight. Until it settles, that transport's traffic must + * wait for its grant, so its planes stay closed whoever else opens planes + * meanwhile (startup's staged open, resume()); the reconnect opens them. + */ + private readonly mcplPolicyExchanges = new WeakMap(); + + /** + * Open MCPL connections' planes (every connection when none is given), + * except a connection whose reconnect's policy exchange is still in flight + * (see mcplPolicyExchanges). A quiesced host (issue #122) opens only + * control planes, so data planes stay held for the whole maintenance + * window; resume() calls this again once the flag is cleared. Control + * planes always come up, since host/command (and with it resume) rides the + * control plane. */ private readyMcplPlanes(connections?: McplServerConnection[]): void { for (const connection of connections ?? this.mcplServerRegistry?.getAllServers() ?? []) { + const exchange = this.mcplPolicyExchanges.get(connection); + if (exchange !== undefined && exchange === connection.transportEpoch) continue; if (this.quiesced) connection.readyControlPlane(); else connection.ready(); } @@ -14574,7 +14586,11 @@ export class AgentFramework { // Request (whose response is never buffered) must be answered before // any plane opens, otherwise the first post-reconnect traffic, control // included (the server's channels/register), lands while the grant is - // still empty and is rejected fail-closed instead of delivered. + // still empty and is rejected fail-closed instead of delivered. The + // exchange is marked in flight for this transport, so no other opener + // (startup's staged open, resume) releases the traffic early. + const epoch = connection.transportEpoch; + this.mcplPolicyExchanges.set(connection, epoch); void (async () => { try { const config = this.mcplServerConfigs.get(connection.id); @@ -14587,7 +14603,13 @@ export class AgentFramework { error instanceof Error ? error.message : error, ); } - this.readyMcplPlanes([connection]); + // Settled, granted or not (an unanswered or refused policy leaves the + // grant empty and the connection MCP-only, as at initial connect): + // open the planes, unless a newer transport's exchange has taken over. + if (this.mcplPolicyExchanges.get(connection) === epoch) { + this.mcplPolicyExchanges.delete(connection); + this.readyMcplPlanes([connection]); + } this.handleToolsListChanged(connection.id); this.emitTrace({ type: 'mcpl:server-reconnected', diff --git a/test/fixtures/reregistering-mcpl-server.mjs b/test/fixtures/reregistering-mcpl-server.mjs index c18e3639f..b73be1c60 100644 --- a/test/fixtures/reregistering-mcpl-server.mjs +++ b/test/fixtures/reregistering-mcpl-server.mjs @@ -2,17 +2,39 @@ // // Like a chat bridge, it registers its channel as soon as the host sends // notifications/initialized, before the host's §5.3 policy Request is -// answered, and appends the host's answer to that registration to -// EVENTS_PATH. When CRASH_PATH appears it removes the file and exits, once, -// so the host's reconnect starts a fresh process that registers again. -import { appendFileSync, existsSync, unlinkSync } from 'node:fs'; +// answered, and appends the host's answer to that registration (and its own +// steps) to EVENTS_PATH, tagged with ROLE. Optional coordination, each by +// file so tests order steps without timing: +// - CRASH_PATH: when it appears, remove it and exit (once), so the host +// reconnects to a fresh process; +// - FAIL_FIRST_PATH: if it is missing at initialize, create it and exit, so +// the host's first connect fails and its retry succeeds; +// - INITIALIZE_AFTER_PATH: answer initialize only once it exists; +// - POLICY_PENDING_PATH / POLICY_RELEASE_PATH: on the policy Request, create +// the first, and answer only once the second exists. +import { appendFileSync, existsSync, unlinkSync, writeFileSync } from 'node:fs'; -const eventsPath = process.env.EVENTS_PATH; -const crashPath = process.env.CRASH_PATH; +const { + EVENTS_PATH: eventsPath, + ROLE: role = 'server', + CRASH_PATH: crashPath, + FAIL_FIRST_PATH: failFirstPath, + INITIALIZE_AFTER_PATH: initializeAfterPath, + POLICY_PENDING_PATH: policyPendingPath, + POLICY_RELEASE_PATH: policyReleasePath, +} = process.env; const send = (message) => process.stdout.write(JSON.stringify(message) + '\n'); const event = (name, extra = {}) => { - if (eventsPath) appendFileSync(eventsPath, JSON.stringify({ event: name, pid: process.pid, ...extra }) + '\n'); + if (eventsPath) appendFileSync(eventsPath, JSON.stringify({ event: name, role, pid: process.pid, ...extra }) + '\n'); +}; +const once = (path, fn) => { + if (!path || existsSync(path)) return fn(); + const timer = setInterval(() => { + if (!existsSync(path)) return; + clearInterval(timer); + fn(); + }, 5); }; let buf = ''; @@ -25,7 +47,12 @@ process.stdin.on('data', (chunk) => { if (!line.trim()) continue; const message = JSON.parse(line); if (message.method === 'initialize') { - send({ + if (failFirstPath && !existsSync(failFirstPath)) { + writeFileSync(failFirstPath, '1'); + event('initial-fail'); + process.exit(9); + } + once(initializeAfterPath, () => send({ jsonrpc: '2.0', id: message.id, result: { @@ -40,16 +67,25 @@ process.stdin.on('data', (chunk) => { }, }, }, - }); + })); } else if (message.method === 'notifications/initialized') { send({ jsonrpc: '2.0', id: 100, method: 'channels/register', - params: { channels: [{ id: 'chat:room-1', type: 'chat', label: 'room-1', direction: 'bidirectional' }] }, + params: { channels: [{ id: `chat:${role}`, type: 'chat', label: role, direction: 'bidirectional' }] }, }); + event('register-sent'); } else if (message.method === 'featureSets/update') { - if (message.id !== undefined && message.id !== null) send({ jsonrpc: '2.0', id: message.id, result: { accepted: true } }); + if (message.id === undefined || message.id === null) continue; + if (policyPendingPath) { + writeFileSync(policyPendingPath, '1'); + event('policy-held'); + } + once(policyReleasePath, () => { + event('policy-answered'); + send({ jsonrpc: '2.0', id: message.id, result: { accepted: true } }); + }); } else if (message.method === 'tools/list') { send({ jsonrpc: '2.0', id: message.id, result: { tools: [] } }); } else if (message.id === 100 && message.method === undefined) { diff --git a/test/mcpl-reconnect-registration.test.ts b/test/mcpl-reconnect-registration.test.ts index d194978bf..5bc7d40cb 100644 --- a/test/mcpl-reconnect-registration.test.ts +++ b/test/mcpl-reconnect-registration.test.ts @@ -18,13 +18,16 @@ import { fileURLToPath } from 'node:url'; import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; +import { EventEmitter } from 'node:events'; import { AgentFramework } from '../src/index.js'; +import { FeatureSetManager } from '../src/mcpl/feature-set-manager.js'; import { MockMembrane } from './helpers/mock-membrane.js'; const FIXTURE = fileURLToPath(new URL('./fixtures/reregistering-mcpl-server.mjs', import.meta.url)); interface ServerEvent { event: string; + role: string; pid: number; error?: { code: number; message: string } | null; } @@ -93,4 +96,109 @@ describe('MCPL reconnect and channel registration', () => { `the reconnected server's registration waits for its new grant instead of being refused: ${JSON.stringify(second.error)}`, ); }); + + it('a first reconnect during staged startup keeps its registration held through startup\'s open of every plane', async () => { + // Mira-1605's reproduction (room-245): `retry` fails its first connect; + // its retry reconnects while `stager` is still staging, and holds its + // policy answer; `stager` answers initialize only once that policy is + // pending, so startup's final open of every connection's planes runs + // while the retry's exchange is still in flight. + const eventsPath = join(dir, 'events.jsonl'); + const pending = join(dir, 'policy-pending'); + const release = join(dir, 'policy-release'); + const server = (id: string, env: Record) => ({ + id, + command: process.execPath, + args: [FIXTURE], + env: { EVENTS_PATH: eventsPath, ROLE: id, ...env }, + enabledFeatureSets: ['chat'], + ...(id === 'retry' ? { reconnect: true, reconnectIntervalMs: 20 } : {}), + }); + framework = await AgentFramework.create({ + storePath: join(dir, 'store'), + membrane: new MockMembrane().asMembrane(), + agents: [{ name: 'resident', model: 'test-model', systemPrompt: 'You are a resident.' }], + modules: [], + maintenanceIntervalMs: 0, + mcplServers: [ + server('retry', { FAIL_FIRST_PATH: join(dir, 'failed-once'), POLICY_PENDING_PATH: pending, POLICY_RELEASE_PATH: release }), + server('stager', { INITIALIZE_AFTER_PATH: pending }), + ], + }); + const retry = () => serverEvents(eventsPath).filter((e) => e.role === 'retry'); + // Startup has opened every plane it may; the retry's policy is still held. + await new Promise((resolve) => setTimeout(resolve, 100)); + assert.ok(retry().some((e) => e.event === 'policy-held'), `events: ${JSON.stringify(retry())}`); + assert.equal( + retry().find((e) => e.event === 'register-answered'), + undefined, + `its registration is not answered while its policy exchange is in flight: ${JSON.stringify(retry())}`, + ); + + writeFileSync(release, '1'); + await waitFor('the retry\'s registration answer', () => retry().some((e) => e.event === 'register-answered')); + const steps = retry().map((e) => e.event); + const answer = retry().find((e) => e.event === 'register-answered')!; + assert.equal(answer.error, null, `admitted against the new grant: ${JSON.stringify(answer.error)}`); + assert.ok(steps.indexOf('policy-answered') < steps.indexOf('register-answered'), `steps: ${steps.join(', ')}`); + }); +}); + +describe('MCPL reconnect: a pending policy exchange holds its planes', () => { + /** A framework stub with one connection whose policy answers are deferred. */ + function harness() { + const answers: Array<(receipt: { accepted: true }) => void> = []; + let opened = 0; + const connection = Object.assign(new EventEmitter(), { + id: 'srv', + transportEpoch: 0, + capabilities: { version: '0.5', pushEvents: true, featureSets: { chat: { description: 'chat', uses: ['pushEvents'] } } }, + mcpToolsAdvertised: false, + establishGrant: () => {}, + sendFeatureSetsUpdateRequest: () => new Promise<{ accepted: true }>((resolve) => { answers.push(resolve); }), + ready: () => { opened++; }, + readyControlPlane: () => { opened++; }, + }); + const fw = Object.create(AgentFramework.prototype) as any; + fw.traceListeners = []; + fw.mcplPolicyExchanges = new WeakMap(); + fw.mcplServerConfigs = new Map([['srv', { id: 'srv', command: 'unused', enabledFeatureSets: ['chat'] }]]); + fw.featureSetManager = new FeatureSetManager(); + fw.checkpointManager = null; + fw.mcplServerRegistry = { getAllServers: () => [connection], getServer: () => connection }; + fw.handleToolsListChanged = () => {}; + fw.wireMcplEvents(connection); + return { + fw, + connection, + answers, + opened: () => opened, + reconnect: () => { connection.transportEpoch++; connection.emit('reconnect', { attempts: 1 }); }, + }; + } + + it('no other opener (startup\'s staged open, resume) releases a reconnect\'s traffic before its exchange settles', async () => { + const h = harness(); + h.reconnect(); + h.fw.readyMcplPlanes(); // startup's open of every connection, or resume() + assert.equal(h.opened(), 0, 'held while the policy exchange is in flight'); + h.answers[0]({ accepted: true }); + await new Promise((resolve) => setImmediate(resolve)); + assert.equal(h.opened(), 1, 'opened by the reconnect once its exchange settled'); + h.fw.readyMcplPlanes(); + assert.equal(h.opened(), 2, 'and no longer held'); + }); + + it('a superseded exchange never opens the planes of the transport that replaced it', async () => { + const h = harness(); + h.reconnect(); + h.reconnect(); // the first fresh transport died; a newer one is exchanging + h.answers[0]({ accepted: true }); + await new Promise((resolve) => setImmediate(resolve)); + h.fw.readyMcplPlanes(); + assert.equal(h.opened(), 0, 'the newer exchange is still in flight'); + h.answers[1]({ accepted: true }); + await new Promise((resolve) => setImmediate(resolve)); + assert.equal(h.opened(), 1); + }); }); diff --git a/test/mcpl-reconnect-reregistration.test.ts b/test/mcpl-reconnect-reregistration.test.ts index 35a1fadd6..eb3d20630 100644 --- a/test/mcpl-reconnect-reregistration.test.ts +++ b/test/mcpl-reconnect-reregistration.test.ts @@ -108,6 +108,7 @@ async function makeHarness() { fw.channelRegistry = null; fw.inferenceRouter = null; fw.eventGate = null; + fw.mcplPolicyExchanges = new WeakMap(); fw.featureSetManager = new FeatureSetManager(); fw.scopeManager = new ScopeManager(); From 485a3c75a0029fe1aab490af4fb2c13927fc17ed Mon Sep 17 00:00:00 2001 From: Ansel-1606 Date: Wed, 7 Oct 2026 17:13:58 -0400 Subject: [PATCH 30/73] fix(recover): refuse --dry-run with an awareness control instead of carrying the control out Review of 25891f0 by Mira-1605 (room-245, #48850), reproduced: the CLI parsed --dry-run but the --awareness path ignored it, so `--awareness cancel ID --dry-run` cancelled the batch's adds and logged it, while the new help text said a dry run changes and records nothing; a "preview" of retract or release would have published removals or held marks. A control has no preview (inspect with --awareness list), so the CLI now refuses --dry-run with --awareness before it opens or creates any store, as Mira favoured, rather than simulating controls. The help text limits --dry-run to a recovery and says each control acts at once. Test (test/recover-cli-operator-log.test.ts): cancel, retract and release with --dry-run each exit 1 with the refusal; the journal (views and operations) is unchanged and nothing is logged; against a store path that doesn't exist, nothing is created. Fails without this commit (cancel exits 0). --- changelog.d/surgery-marks-controls.added.md | 5 ++- src/recovery/recover-cli.ts | 16 ++++++-- test/recover-cli-operator-log.test.ts | 44 +++++++++++++++++++++ 3 files changed, 59 insertions(+), 6 deletions(-) diff --git a/changelog.d/surgery-marks-controls.added.md b/changelog.d/surgery-marks-controls.added.md index 51484e58b..b562b6481 100644 --- a/changelog.d/surgery-marks-controls.added.md +++ b/changelog.d/surgery-marks-controls.added.md @@ -29,5 +29,6 @@ - `agent-framework-recover` records its acts in the store's operator log (`/operator-actions.jsonl`), done or refused, with the OS account that ran it: each `--awareness` cancel, retract and release, and each - recovery with its marks choice and receipt (kind `recovery`). A dry run or - a list records nothing. + recovery with its marks choice and receipt (kind `recovery`). A list, and + a recovery's dry run, record nothing. `--dry-run` with `--awareness` is + refused: a control has no preview. diff --git a/src/recovery/recover-cli.ts b/src/recovery/recover-cli.ts index 9735e3267..4940165f8 100644 --- a/src/recovery/recover-cli.ts +++ b/src/recovery/recover-cli.ts @@ -61,7 +61,8 @@ Options: --outbox A pre-journal awareness ledger to import (default: /recovery/discord-awareness-outbox.json) --emoji Marker reaction (default: 💤) - --dry-run Inspect counts/addresses without writing + --dry-run Inspect a recovery's counts/addresses without + writing (a recovery only; refused with --awareness) --help Show this help Awareness journal (host stopped): @@ -73,11 +74,13 @@ Awareness journal (host stopped): ref's configured MCPL route) on a batch's refs, or on every ref an add was ever sent for --awareness release Queue a held batch's recorded operations + Each acts at once: inspect first with --awareness list. Queued work is delivered when the host next connects to the Discord server. -Each recovery and each cancel, retract or release (done or refused; not a -dry run or a list) is recorded in /operator-actions.jsonl, the host's -operator log, with the OS account that ran it. +Each recovery, and each cancel, retract or release, is recorded in +/operator-actions.jsonl, the host's operator log, with the OS account +that ran it, whether it was carried out or refused. A dry run and a list +record nothing. The agent host must be stopped before running this command.`; } @@ -213,6 +216,11 @@ async function main(): Promise { if (!options.storePath) { throw new Error(`--awareness needs --store\n\n${usage()}`); } + // There is no preview of a control: refuse rather than act on a command + // that asked not to change anything. + if (options.dryRun) { + throw new Error('--dry-run previews a recovery; it does not apply to --awareness (inspect with --awareness list)'); + } awareness(options); return; } diff --git a/test/recover-cli-operator-log.test.ts b/test/recover-cli-operator-log.test.ts index 1c0202e37..91c470490 100644 --- a/test/recover-cli-operator-log.test.ts +++ b/test/recover-cli-operator-log.test.ts @@ -82,6 +82,50 @@ test('the offline awareness controls record each act in the operator log, done o } }); +test('--dry-run with an awareness control is refused before the store opens: nothing is done or recorded', () => { + const dir = mkdtempSync(join(tmpdir(), 'recover-cli-log-')); + const storePath = join(dir, 'store'); + const journal = () => { + const store = JsStore.openOrCreate({ path: storePath }); + try { + const outbox = new DiscordAwarenessOutbox(store); + return { view: outbox.view(), operations: outbox.operations() }; + } finally { + store.close(); + } + }; + try { + const store = JsStore.openOrCreate({ path: storePath }); + const outbox = new DiscordAwarenessOutbox(store); + const active = outbox.prepare({ + agentName: 'cairn', sourceBranch: 'main', targetBranch: 'rollback/cairn/1', refs: [ref('m1')], scope: 'all', + })!; + outbox.activate(active.id); + const held = outbox.prepare({ + agentName: 'cairn', sourceBranch: 'main', targetBranch: 'rollback/cairn/interrupted', refs: [ref('m2')], scope: 'all', + })!; + outbox.recoverAtStartup('main'); // its switch was never recorded: held + store.close(); + const before = journal(); + const heldView = before.view.find((v) => v.id === held.id); + assert.equal(heldView?.kind === 'batch' ? heldView.status : undefined, 'held'); + + for (const control of [['cancel', active.id], ['retract', 'all'], ['release', held.id]]) { + const run = recover(storePath, '--awareness', ...control, '--dry-run'); + assert.equal(run.status, 1, `${control[0]} --dry-run is refused`); + assert.match(run.stderr, /--dry-run previews a recovery; it does not apply to --awareness/); + } + assert.deepEqual(journal(), before, 'nothing was cancelled, retracted or released'); + assert.deepEqual(operatorLog(storePath), [], 'and nothing was recorded'); + + const missing = join(dir, 'no-such-store'); + assert.equal(recover(missing, '--awareness', 'cancel', active.id, '--dry-run').status, 1); + assert.equal(existsSync(missing), false, 'refused before any store is opened or created'); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); + test('an offline recovery records its marks choice and receipt in the operator log; a dry run records nothing', async () => { const dir = mkdtempSync(join(tmpdir(), 'recover-cli-log-')); const storePath = join(dir, 'store'); From 4221eb867cae5cc086d286adeaf868dee11596b2 Mon Sep 17 00:00:00 2001 From: Agnes-1009 Date: Wed, 7 Oct 2026 06:30:48 -0400 Subject: [PATCH 31/73] feat(framework): runAtSafeBoundary, a store reservation held across a host callback MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A host that changes an agent's body (settings, branch, tools) needs a boundary at which no turn is running and none can start while its asynchronous change is applied. Today the only such boundary is the surgery reservation inside rollbackToMessage / suppressMessages, which refuses when anything is busy instead of waiting for the store to drain. An idle-status observation or an inference:completed trace doesn't establish it: a wake can start a turn between the observation and the change. runAtSafeBoundary(opts, fn) waits for that boundary and holds it: - It waits until every agent sharing the store is idle with no turn alive (the scheduler's own busy test, status + turn-alive), no ephemeral run is registered, and no other lease or live surgery holds the store. It then takes exactly the reservation a live surgery takes: a turn token for every agent, plus the store-wide hold for agents registered later. It runs fn(lease) and releases in finally. - While a request waits, processInferenceRequests holds new turns of agents that aren't running as ephemeral streams, as quiesce does. Their wakes stay queued, so the store drains to the boundary instead of being re-woken past it. Two kinds of turn pass: continuations of a held turn (isTurnContinuation), because that turn has to end for the boundary to come; and a registered ephemeral stream's own wakes, because the lease waits for its run to end and holding them would deadlock. - Requests are granted oldest first. processNextEvent tries the grant before and after each scheduler pass, and a request tries it on arrival. - opts.signal withdraws a waiting request (AbortError). After the grant, fn runs to its end. - The lease (id, verb, since, agents) is what lease-aware operator methods take. puppetToolCall(…, { lease }) runs on the token that lease holds for the agent, instead of refusing the hold and minting its own. It refuses a lease that isn't the one currently held, and an agent the lease doesn't cover. The lease's release flushes what deferred meanwhile. reserveStoreForSurgery is split into storeBusyReasons() and takeStoreReservation(), which returns the reserved tokens. Its refusals are unchanged. One fix in the shared release flush: it stored drained writes without their deferredWriteId and never acknowledged them. drainDeferredFor hands entries to the durable un-acked queue, so a persisted deferral flushed there would replay as a duplicate after a crash. It now writes each entry with its id and calls ackDeferredWrites(), as the puppet and resume flushes do. SafeBoundaryLease and SafeBoundaryOptions are exported. Tests: test/safe-boundary-lease.test.ts, 9 cases. - An idle store grants at once. The lease holds every agent's token; a wake is requeued, not started; a puppet without the lease is refused; the requeued wake runs after release. - A live turn makes it wait. Meanwhile an idle agent's new wake is held, and the lease runs before it. - A budget-restart continuation passes a waiting lease while the new wake beside it stays held. - A registered ephemeral stream's turn runs to completion (the real runEphemeralToCompletion, with a mock membrane) before the lease is granted. - Abort withdraws a waiting lease and stops holding wakes; abort after the grant doesn't interrupt the callback. - Two waiters are granted in order, and a throwing callback still releases. - A puppet under the lease stores its pair adjacently and keeps the lease's token; a stale lease is refused. - A write deferred behind the lease lands at release carrying its deferredWriteId, with nothing left un-acked. This is R1 of the agent-framework companion for the connectome-host body-change contract. Co-Authored-By: Agnes (Claude Opus 5.5) --- src/framework.ts | 273 ++++++++++++++++++++++++++++--- src/index.ts | 2 +- test/safe-boundary-lease.test.ts | 251 ++++++++++++++++++++++++++++ 3 files changed, 498 insertions(+), 28 deletions(-) create mode 100644 test/safe-boundary-lease.test.ts diff --git a/src/framework.ts b/src/framework.ts index ab4c36cc6..87e20dba0 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -621,6 +621,32 @@ export interface HostModeStatus { unabandonable?: string[]; } +/** + * A store-wide reservation held across one host callback + * (`runAtSafeBoundary`). While it is held no turn starts for any agent, + * exactly as during a live surgery, and lease-aware operator methods accept + * it so they run inside that reservation instead of refusing it. + */ +export interface SafeBoundaryLease { + readonly id: string; + readonly verb: string; + /** Epoch ms when the lease was granted. */ + readonly since: number; + /** The agents whose turn tokens the lease holds: every agent registered + * when it was granted. One registered later is covered by the hold. */ + readonly agents: readonly string[]; +} + +export interface SafeBoundaryOptions { + /** What the lease is for, named in refusals and busy traces. */ + verb: string; + requester?: OperatorRequester; + /** Aborting while the lease waits withdraws the request (the promise + * rejects with an AbortError). Once granted, the callback runs to its end + * and the lease is released after it; the signal no longer applies. */ + signal?: AbortSignal; +} + /** * Thrown by `resume()` when the current runtime settings do not compile for * one or more agents — returning to service would OverBudget-wedge them on @@ -1404,6 +1430,18 @@ export class AgentFramework { * scheduler, ephemeral admission and puppet entry points. */ private surgeryHold: { verb: string; agentName: string; since: number } | null = null; + /** + * Hosts waiting for a safe boundary (`runAtSafeBoundary`), oldest first. + * While any wait, new resident turns are held (see processInferenceRequests) + * so the store drains to the boundary instead of being re-woken past it. + */ + private boundaryWaiters: Array<{ + verb: string; + requester?: OperatorRequester; + grant: (held: { lease: SafeBoundaryLease; release: () => void }) => void; + }> = []; + /** The lease currently held, and the turn token it reserved per agent. */ + private heldLease: { lease: SafeBoundaryLease; tokens: Map } | null = null; /** Serialize per-server drains so reconnect and an online undo cannot race. */ private discordAwarenessDrains: Map> = new Map(); /** Framework-global inference gate; older generations cannot release it. */ @@ -6027,6 +6065,22 @@ export class AgentFramework { `Cannot ${verb}: a live ${this.surgeryHold.verb} for ${this.surgeryHold.agentName} is already in progress`, ); } + const busy = this.storeBusyReasons(); + if (busy.length > 0) { + throw new OperatorActionError( + 'agent-busy', + `Cannot ${verb} while ${busy.join(', ')} — every agent sharing the store must be idle (quiesce the host first)`, + ); + } + return this.takeStoreReservation(verb, agentName).release; + } + + /** + * Why the store isn't at a safe boundary, one reason per busy agent; empty + * when every agent sharing it is idle with no turn alive (the scheduler's + * own busy test, status + turn-alive). + */ + private storeBusyReasons(): string[] { const busy: string[] = []; for (const [name, a] of this.agents) { if (a.state.status !== 'idle') busy.push(`${name} is ${a.state.status}`); @@ -6035,12 +6089,15 @@ export class AgentFramework { for (const name of this.activeTurnTokens.keys()) { if (!this.agents.has(name)) busy.push(`${name} is turn-alive`); } - if (busy.length > 0) { - throw new OperatorActionError( - 'agent-busy', - `Cannot ${verb} while ${busy.join(', ')} — every agent sharing the store must be idle (quiesce the host first)`, - ); - } + return busy; + } + + /** + * Take the store reservation reserveStoreForSurgery describes, for a + * caller that has already found the store free. Returns the tokens it + * holds, so a lease can run its holder's work under them. + */ + private takeStoreReservation(verb: string, agentName: string): { tokens: Map; release: () => void } { const reserved = new Map(); for (const name of this.agents.keys()) { const token = this.nextTurnToken++; @@ -6052,7 +6109,7 @@ export class AgentFramework { const hold = { verb, agentName, since: Date.now() }; this.surgeryHold = hold; let released = false; - return () => { + const release = () => { if (released) return; released = true; if (this.surgeryHold === hold) this.surgeryHold = null; @@ -6062,20 +6119,133 @@ export class AgentFramework { // Writers that deferred behind the reservation land now, on the branch // the operator chose (or the restored source) — the same end-of-turn // flush driveStream/puppet perform, under the same guard. + // Each write carries its deferredWriteId and the batch is acknowledged, + // as the puppet and resume flushes do: drainDeferredFor hands entries + // to the durable un-acked queue, so a persisted deferral stored here + // without its id would replay as a duplicate after a crash. if (this.deferredMessages.length > 0 && this.pendingAssistantBlocks.size === 0) { for (const name of reserved.keys()) { if (this.activeTurnTokens.has(name)) continue; for (const msg of this.drainDeferredFor(name)) { try { - this.addMessage(msg.participant, msg.content, msg.metadata, - msg.forAgent ? { forAgent: msg.forAgent } : undefined); + this.addMessage(msg.participant, msg.content, msg.metadata, { + deferredWriteId: msg.id, + ...(msg.forAgent ? { forAgent: msg.forAgent } : {}), + }); } catch (error) { console.error(`[operator] post-surgery deferred flush failed for ${name}: ${error instanceof Error ? error.message : String(error)}`); } } } + this.ackDeferredWrites(); } }; + return { tokens: reserved, release }; + } + + /** + * Run `fn` at a safe boundary, holding the store reservation across it. + * + * The request waits until every agent sharing the store is idle with no + * turn alive, no ephemeral run is registered, and no other lease or live + * surgery holds the store. Then it takes the same reservation a live + * surgery takes (a turn token for every agent, plus the store-wide hold + * for agents registered later), runs `fn`, and releases in `finally`; + * release flushes writers that deferred behind it. + * + * While a request waits, new turns of agents that aren't running as + * ephemeral streams are held, as quiesce holds them, so the store drains + * to the boundary instead of being re-woken past it. Continuations of a + * held turn still run, and so do a registered ephemeral stream's own + * wakes, since the lease waits for those runs to end. Requests are granted + * oldest first. Aborting `opts.signal` while waiting withdraws the request; + * after the grant, `fn` runs to its end. + * + * Operator methods that would refuse the hold take the lease instead: + * `puppetToolCall(…, { lease })` runs under the lease's token for that + * agent. + */ + async runAtSafeBoundary(opts: SafeBoundaryOptions, fn: (lease: SafeBoundaryLease) => Promise): Promise { + const withdrawn = () => { + const error = new Error(`safe boundary for ${opts.verb} withdrawn before it was granted`); + error.name = 'AbortError'; + return error; + }; + if (opts.signal?.aborted) throw withdrawn(); + const requested = Date.now(); + const held = await new Promise<{ lease: SafeBoundaryLease; release: () => void }>((resolve, reject) => { + let onAbort: (() => void) | undefined; + const waiter = { + verb: opts.verb, + ...(opts.requester ? { requester: opts.requester } : {}), + grant: (granted: { lease: SafeBoundaryLease; release: () => void }) => { + if (onAbort) opts.signal?.removeEventListener('abort', onAbort); + resolve(granted); + }, + }; + if (opts.signal) { + onAbort = () => { + const at = this.boundaryWaiters.indexOf(waiter); + if (at < 0) return; // already granted + this.boundaryWaiters.splice(at, 1); + reject(withdrawn()); + }; + opts.signal.addEventListener('abort', onAbort, { once: true }); + } + this.boundaryWaiters.push(waiter); + this.tryGrantSafeBoundary(); + }); + console.log( + `[safe-boundary] ${held.lease.verb}: granted after ${held.lease.since - requested}ms ` + + `(${held.lease.agents.length} agent(s) reserved` + + `${opts.requester ? `, for ${opts.requester.name ?? opts.requester.id ?? 'operator'} via ${opts.requester.via}` : ''})`, + ); + try { + return await fn(held.lease); + } finally { + held.release(); + console.log(`[safe-boundary] ${held.lease.verb}: released after ${Date.now() - held.lease.since}ms`); + } + } + + /** + * Grant the oldest waiting lease if the store is at a safe boundary now. + * Called when a request arrives and on every scheduler pass, so a waiting + * lease is granted as soon as the last turn or ephemeral run ends. + */ + private tryGrantSafeBoundary(): void { + if (this.boundaryWaiters.length === 0 || this.heldLease || this.surgeryHold) return; + if (this.ephemeralRuns.size > 0 || this.storeBusyReasons().length > 0) return; + const waiter = this.boundaryWaiters.shift()!; + const { tokens, release } = this.takeStoreReservation(waiter.verb, 'safe-boundary'); + const lease: SafeBoundaryLease = Object.freeze({ + id: randomUUID(), + verb: waiter.verb, + since: Date.now(), + agents: Object.freeze([...tokens.keys()]), + }); + const entry = { lease, tokens }; + this.heldLease = entry; + waiter.grant({ + lease, + release: () => { + if (this.heldLease === entry) this.heldLease = null; + release(); + // The next waiter may be grantable at once. + this.tryGrantSafeBoundary(); + }, + }); + } + + /** The tokens `lease` holds, or a refusal naming why it can't be used. */ + private heldLeaseTokens(lease: SafeBoundaryLease, verb: string): Map { + if (!this.heldLease || this.heldLease.lease !== lease) { + throw new OperatorActionError( + 'invalid', + `Cannot ${verb} under lease ${lease.id} (${lease.verb}): it is not the lease currently held`, + ); + } + return this.heldLease.tokens; } /** @@ -7067,9 +7237,16 @@ export class AgentFramework { // Close idle conversation forks (no-op unless conversations configured) this.sweepExpiredConversations(); + // A waiting safe-boundary lease is granted the moment the store is free: + // before the scheduler can start another turn, and again after it, for a + // pass that ended the last turn. + this.tryGrantSafeBoundary(); + // Check for inference requests await this.processInferenceRequests(); + this.tryGrantSafeBoundary(); + // Yield to the event loop between iterations. A pending inference request // is not necessarily runnable: while its agent is streaming or waiting for // tools, processInferenceRequests() deliberately requeues it. Treating that @@ -8832,6 +9009,24 @@ export class AgentFramework { requests.push(...passThrough); } + // A host waiting for a safe boundary (runAtSafeBoundary) holds new + // turns as quiesce does: the wakes stay queued, so the store drains to + // the boundary instead of being re-woken past it. Continuations of a + // held turn pass, since that turn has to end for the boundary to come. + // A registered ephemeral stream's wakes pass too: the lease waits for + // its run to end, and holding them would deadlock it. Once the lease is + // granted, its reservation (surgeryHold, below) holds everything. + if (this.boundaryWaiters.length > 0 && !this.ephemeralRuns.has(agentName)) { + const held = requests.filter((r) => !isTurnContinuation(r.reason)); + if (held.length > 0) { + this.pendingRequests.push(...held); + const passThrough = requests.filter((r) => isTurnContinuation(r.reason)); + if (passThrough.length === 0) continue; + requests.length = 0; + requests.push(...passThrough); + } + } + const turnAlive = !budgetRestart && this.activeTurnTokens.has(agentName); const providerGate = this.providerGates?.get(agentName); // A primary can own provider admission while yielding to an already in-flight @@ -11414,28 +11609,46 @@ export class AgentFramework { * deliberately NOT in the stored messages — metadata there would break * byte-parity with real turns. Whether to disclose to the resident is * the operator's call; the princess precedent was disclosed first. + * - Under a held safe-boundary lease (`opts.lease`, from + * runAtSafeBoundary) it runs on the turn token that lease holds for the + * agent instead of refusing the hold and minting its own; the lease's + * release flushes what deferred meanwhile. */ async puppetToolCall( agentName: string, toolName: string, input: Record, + opts?: { lease?: SafeBoundaryLease }, ): Promise<{ toolUseId: string; result: ToolResult }> { const agent = this.agents.get(agentName); if (!agent) throw new Error(`Unknown agent: ${agentName}`); - // Idle AND no turn alive: status reads 'idle' from dequeue until the - // stream registers, and again while a turn's teardown is pending - // (the scheduler's own busy test, 'idle+turn-alive'). - if (this.surgeryHold) { - throw new Error( - `puppet refused: the store is under live ${this.surgeryHold.verb} for ${this.surgeryHold.agentName} — retry when it completes`, - ); - } - if (agent.state.status !== 'idle' || this.activeTurnTokens.has(agentName)) { - const shown = agent.state.status === 'idle' ? 'idle+turn-alive' : agent.state.status; - throw new Error( - `puppet refused: agent ${agentName} is ${shown} (requires idle — ` + - `injecting a turn under an active stream corrupts wire ordering)`, - ); + // Under a lease: the agent's turn token is the lease's. An agent + // registered after the grant has none, so it can't be puppeted here. + let leaseToken: number | undefined; + if (opts?.lease) { + leaseToken = this.heldLeaseTokens(opts.lease, `puppet ${toolName} as ${agentName}`).get(agentName); + if (leaseToken === undefined || this.activeTurnTokens.get(agentName) !== leaseToken) { + throw new Error(`puppet refused: lease ${opts.lease.id} (${opts.lease.verb}) holds no turn for ${agentName}`); + } + if (agent.state.status !== 'idle') { + throw new Error(`puppet refused: agent ${agentName} is ${agent.state.status} under lease ${opts.lease.id}`); + } + } else { + // Idle AND no turn alive: status reads 'idle' from dequeue until the + // stream registers, and again while a turn's teardown is pending + // (the scheduler's own busy test, 'idle+turn-alive'). + if (this.surgeryHold) { + throw new Error( + `puppet refused: the store is under live ${this.surgeryHold.verb} for ${this.surgeryHold.agentName} — retry when it completes`, + ); + } + if (agent.state.status !== 'idle' || this.activeTurnTokens.has(agentName)) { + const shown = agent.state.status === 'idle' ? 'idle+turn-alive' : agent.state.status; + throw new Error( + `puppet refused: agent ${agentName} is ${shown} (requires idle — ` + + `injecting a turn under an active stream corrupts wire ordering)`, + ); + } } const onSurface = this.getToolsForAgent(agentName) .some((t) => t.name === toolName && agent.canUseTool(t.name)); @@ -11463,9 +11676,11 @@ export class AgentFramework { // requeues wakes) and the addMessage guard reads (cross-turn writers // defer), so for the duration this behaves like a turn with no stream. // Token-matched release in finally, same leak-proofing as - // startAgentStream: a token nobody clears wedges the agent. - const turnToken = this.nextTurnToken++; - this.activeTurnTokens.set(agentName, turnToken); + // startAgentStream: a token nobody clears wedges the agent. Under a + // lease the lease's token already reserves the agent, and the lease + // releases it. + const turnToken = leaseToken ?? this.nextTurnToken++; + if (leaseToken === undefined) this.activeTurnTokens.set(agentName, turnToken); let ownedToEnd = false; try { const started = Date.now(); @@ -11525,7 +11740,11 @@ export class AgentFramework { ); return { toolUseId, result }; } finally { - if (this.activeTurnTokens.get(agentName) === turnToken) this.activeTurnTokens.delete(agentName); + if (leaseToken !== undefined) { + // The lease owns the token, and its release flushes what deferred. + } else if (this.activeTurnTokens.get(agentName) === turnToken) { + this.activeTurnTokens.delete(agentName); + } // Messages deferred while we held the token land now, after the pair // — the same end-of-turn flush driveStream performs, under the same // tool-cycle guard. (No wake is requested: the agent sees the pair, diff --git a/src/index.ts b/src/index.ts index 2e1c021c4..4fb06cbe7 100644 --- a/src/index.ts +++ b/src/index.ts @@ -3,7 +3,7 @@ export * from './types/index.js'; // Core classes export { AgentFramework, BudgetPreflightError, ResumeBlockedError } from './framework.js'; -export type { RuntimeSettingsPreview, HostModeStatus } from './framework.js'; +export type { RuntimeSettingsPreview, HostModeStatus, SafeBoundaryLease, SafeBoundaryOptions } from './framework.js'; export { Agent } from './agent.js'; export type { StartStreamResult } from './agent.js'; export { ProcessQueueImpl } from './queue.js'; diff --git a/test/safe-boundary-lease.test.ts b/test/safe-boundary-lease.test.ts new file mode 100644 index 000000000..e77641203 --- /dev/null +++ b/test/safe-boundary-lease.test.ts @@ -0,0 +1,251 @@ +/** + * runAtSafeBoundary: a host callback held at a safe boundary. + * + * The lease waits until every agent sharing the store is idle with no turn + * alive and no ephemeral run registered, then holds the same reservation a + * live surgery takes across an async callback. While it waits, new resident + * turns are held (continuations and registered ephemeral streams pass); + * while it is held, nothing starts; at release, writes that deferred behind + * it land, acknowledged. + */ +import { describe, it, beforeEach, afterEach } from 'node:test'; +import assert from 'node:assert/strict'; +import { tmpdir } from 'node:os'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { join } from 'node:path'; +import { AgentFramework } from '../src/index.js'; +import type { InferenceRequest, SafeBoundaryLease } from '../src/index.js'; +import type { ContentBlock } from '@animalabs/membrane'; +import { MockMembrane, createMockResponse } from './helpers/mock-membrane.js'; + +type Internals = { + pendingRequests: InferenceRequest[]; + processInferenceRequests(): Promise; + tryGrantSafeBoundary(): void; + activeTurnTokens: Map; + executeToolCall: (call: Record) => Promise; + boundaryWaiters: unknown[]; + heldLease: { lease: SafeBoundaryLease; tokens: Map } | null; + surgeryHold: unknown; + ephemeralRuns: Map; + deferredMessages: unknown[]; + unackedDeferredWrites: unknown[]; + addMessage(participant: string, content: ContentBlock[], metadata?: Record, opts?: { forAgent?: string }): string; +}; + +const wake = (agentName: string, reason = 'mcpl:channel-incoming'): InferenceRequest => + ({ agentName, reason, source: 'test', timestamp: Date.now() }) as InferenceRequest; +const tick = () => new Promise((r) => setImmediate(r)); + +describe('runAtSafeBoundary', () => { + let tempDir: string; + let membrane: MockMembrane; + let framework: AgentFramework; + let i: Internals; + let quiet: typeof console.log; + + beforeEach(async () => { + tempDir = mkdtempSync(join(tmpdir(), 'safe-boundary-')); + membrane = new MockMembrane(); + framework = await AgentFramework.create({ + storePath: join(tempDir, 'test.chronicle'), + membrane: membrane.asMembrane(), + agents: [ + { name: 'scout', model: 'test-model', systemPrompt: 'You are scout.' }, + { name: 'other', model: 'test-model', systemPrompt: 'You are other.' }, + ], + modules: [], + }); + i = framework as unknown as Internals; + quiet = console.log; + console.log = () => {}; + }); + afterEach(async () => { + console.log = quiet; + await framework.stop(); + rmSync(tempDir, { recursive: true, force: true }); + }); + + it('is granted at once on an idle store and holds every agent; a wake waits for the release', async () => { + membrane.pushResponse(createMockResponse([{ type: 'text', text: 'after the change' }])); + const result = await framework.runAtSafeBoundary({ verb: 'test change' }, async (lease) => { + assert.deepEqual([...lease.agents].sort(), ['other', 'scout']); + assert.equal(i.activeTurnTokens.get('scout'), i.heldLease!.tokens.get('scout'), 'scout reserved by the lease'); + assert.equal(i.activeTurnTokens.get('other'), i.heldLease!.tokens.get('other'), 'other reserved by the lease'); + + i.pendingRequests.push(wake('scout')); + await i.processInferenceRequests(); + assert.equal(membrane.calls.length, 0, 'no turn starts under the lease'); + assert.equal(i.pendingRequests.length, 1, 'the wake is requeued, not dropped'); + + await assert.rejects( + framework.puppetToolCall('scout', 'agent_settings', { action: 'get' }), + /under live test change/, + 'a puppet without the lease is refused by the hold', + ); + return 'applied'; + }); + assert.equal(result, 'applied'); + assert.equal(i.heldLease, null); + assert.equal(i.surgeryHold, null); + assert.equal(i.activeTurnTokens.size, 0, 'every reserved token released'); + + await i.processInferenceRequests(); + await framework.runUntilIdle(); + assert.equal(membrane.calls.length, 1, 'the requeued wake runs after the release'); + }); + + it('waits for a live turn, holds new resident wakes meanwhile, and runs before them', async () => { + membrane.pushResponse(createMockResponse([{ type: 'text', text: 'woken after' }])); + i.activeTurnTokens.set('other', 9_999); // other is mid-turn + let callsWhenGranted = -1; + const leased = framework.runAtSafeBoundary({ verb: 'waits' }, async () => { + callsWhenGranted = membrane.calls.length; + }); + await tick(); + assert.equal(i.boundaryWaiters.length, 1, 'waiting'); + assert.equal(i.heldLease, null, 'not granted while a turn is alive'); + + // scout is idle with no turn alive, yet its new wake is held. + i.pendingRequests.push(wake('scout')); + await i.processInferenceRequests(); + assert.equal(membrane.calls.length, 0, 'the waiting lease holds the new wake'); + assert.equal(i.pendingRequests.length, 1, 'held, not dropped'); + + i.activeTurnTokens.delete('other'); // other's turn ends + i.tryGrantSafeBoundary(); + await leased; + assert.equal(callsWhenGranted, 0, 'the lease ran before the held wake'); + + await i.processInferenceRequests(); + await framework.runUntilIdle(); + assert.equal(membrane.calls.length, 1, 'the held wake runs after the release'); + }); + + it('lets a continuation of a held turn through while it waits', async () => { + membrane.pushResponse(createMockResponse([{ type: 'text', text: 'continued' }])); + i.activeTurnTokens.set('other', 9_999); + const leased = framework.runAtSafeBoundary({ verb: 'waits' }, async () => {}); + await tick(); + i.pendingRequests.push(wake('scout'), wake('scout', 'context_budget_restart')); + await i.processInferenceRequests(); + assert.equal(membrane.calls.length, 1, 'the continuation started its turn'); + assert.deepEqual(i.pendingRequests.map((r) => r.reason), ['mcpl:channel-incoming'], 'the new wake stays held'); + + for (let n = 0; n < 200 && (i.activeTurnTokens.has('scout') || framework.getAgent('scout')!.state.status !== 'idle'); n++) { + await new Promise((r) => setTimeout(r, 5)); + } + i.activeTurnTokens.delete('other'); + i.tryGrantSafeBoundary(); + await leased; + }); + + it('lets a registered ephemeral stream finish its run, then is granted', async () => { + membrane.pushResponse(createMockResponse([{ type: 'text', text: 'ephemeral done' }])); + const { agent, contextManager } = await framework.createEphemeralAgent({ + name: 'worker', model: 'test-model', systemPrompt: 'Do the task.', allowedTools: 'all', + }); + contextManager.addMessage('user', [{ type: 'text', text: 'Run once.' }]); + const run = framework.runEphemeralToCompletion(agent, contextManager); + assert.equal(i.ephemeralRuns.has('worker'), true, 'registered at once'); + + let seen: { ephemeral: number; agents: readonly string[] } | null = null; + const leased = framework.runAtSafeBoundary({ verb: 'after the worker' }, async (lease) => { + seen = { ephemeral: i.ephemeralRuns.size, agents: lease.agents }; + }); + framework.start(); + const settled = await run; + assert.equal(settled.speech.includes('ephemeral done'), true, "the worker's turn ran while the lease waited"); + await leased; + assert.deepEqual(seen, { ephemeral: 0, agents: ['scout', 'other'] }); + }); + + it('withdraws a waiting lease on abort and stops holding wakes', async () => { + membrane.pushResponse(createMockResponse([{ type: 'text', text: 'not held now' }])); + i.activeTurnTokens.set('other', 9_999); + const controller = new AbortController(); + let ran = false; + const leased = framework.runAtSafeBoundary({ verb: 'withdrawn', signal: controller.signal }, async () => { ran = true; }); + await tick(); + controller.abort(); + await assert.rejects(leased, { name: 'AbortError' }); + assert.equal(ran, false); + assert.equal(i.boundaryWaiters.length, 0); + + i.pendingRequests.push(wake('scout')); + await i.processInferenceRequests(); + assert.equal(membrane.calls.length, 1, 'no lease waits, so the wake runs'); + i.activeTurnTokens.delete('other'); + await framework.runUntilIdle(); + }); + + it('runs a granted callback to its end whatever the signal does after the grant', async () => { + const controller = new AbortController(); + let finished = false; + await framework.runAtSafeBoundary({ verb: 'granted', signal: controller.signal }, async () => { + controller.abort(); + await tick(); + finished = true; + }); + assert.equal(finished, true); + assert.equal(i.heldLease, null); + }); + + it('grants waiting leases oldest first, and releases after a failing callback', async () => { + i.activeTurnTokens.set('other', 9_999); + const order: string[] = []; + const first = framework.runAtSafeBoundary({ verb: 'first' }, async () => { + order.push('first'); + throw new Error('first failed'); + }); + const second = framework.runAtSafeBoundary({ verb: 'second' }, async () => { + order.push('second'); + assert.equal(i.heldLease?.lease.verb, 'second'); + }); + await tick(); + assert.equal(i.boundaryWaiters.length, 2); + i.activeTurnTokens.delete('other'); + i.tryGrantSafeBoundary(); + await assert.rejects(first, /first failed/); + await second; + assert.deepEqual(order, ['first', 'second']); + assert.equal(i.heldLease, null); + assert.equal(i.activeTurnTokens.size, 0); + }); + + it("runs a puppet under the lease on the lease's token, and refuses a lease no longer held", async () => { + i.executeToolCall = async () => ({ success: true, data: 'settings snapshot', isError: false }); + let kept: SafeBoundaryLease | null = null; + await framework.runAtSafeBoundary({ verb: 'puppet under lease' }, async (lease) => { + kept = lease; + const token = i.activeTurnTokens.get('scout'); + await framework.puppetToolCall('scout', 'agent_settings', { action: 'get' }, { lease }); + assert.equal(i.activeTurnTokens.get('scout'), token, "the lease's token still reserves scout"); + }); + const all = framework.getAgent('scout')!.getContextManager().getAllMessages() as Array<{ content: Array<{ type: string }> }>; + const types = all.map((m) => m.content[0]?.type); + const use = types.indexOf('tool_use'); + assert.ok(use >= 0, 'pair stored'); + assert.equal(types[use + 1], 'tool_result', 'pair adjacent'); + assert.equal(i.activeTurnTokens.size, 0); + await assert.rejects( + framework.puppetToolCall('scout', 'agent_settings', { action: 'get' }, { lease: kept! }), + /not the lease currently held/, + ); + }); + + it('lands writes deferred behind it at release, with their ids, and acknowledges them', async () => { + await framework.runAtSafeBoundary({ verb: 'defer' }, async () => { + i.addMessage('user', [{ type: 'text', text: 'arrived during the change' }], undefined, { forAgent: 'scout' }); + assert.equal(i.deferredMessages.length, 1, 'deferred behind the reservation'); + }); + assert.equal(i.deferredMessages.length, 0); + assert.equal(i.unackedDeferredWrites.length, 0, 'acknowledged'); + const all = framework.getAgent('scout')!.getContextManager().getAllMessages() as Array<{ + content: Array<{ type: string; text?: string }>; metadata?: { deferredWriteId?: string }; + }>; + const landed = all.find((m) => m.content[0]?.text === 'arrived during the change'); + assert.ok(landed, 'landed at release'); + assert.equal(typeof landed!.metadata?.deferredWriteId, 'string', 'carries its deferred-write id'); + }); +}); From 84664d6623c5b49e21e5d6c0a563bee0bd2198c0 Mon Sep 17 00:00:00 2001 From: Agnes-1009 Date: Wed, 7 Oct 2026 06:56:22 -0400 Subject: [PATCH 32/73] feat(framework): a tool call carries who initiated it through every redispatch MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A tool can't tell an operator acting through an agent's surface from the agent's own act. puppetToolCall executes as the agent, and the call it dispatches is indistinguishable from the model's. A host that treats an operator's change to an agent's body differently from the agent's own self-change (the connectome-host body-change contract) needs that distinction at the tool, including when the tool is reached through a wrapper. ToolCall gains two fields: - `origin?: 'puppet' | 'host'`. 'puppet' means an operator acting through the agent's surface; 'host' means a module calling a tool. Absent means the agent's model. - `admission?: OperatorAdmission` ({ id }): an operator change the host has already admitted. It travels with origin, so a gated tool can apply it rather than stage it again. The gate itself arrives with the operator-change admission work. Where they are set: - puppetToolCall stamps origin 'puppet', plus opts.admission when given. - ModuleContext.callTool stamps 'host' unless the module set an origin itself. - The model path sets nothing. Where they are carried, because each of these points rebuilt the call and dropped them: - ModuleRegistry.handleToolCall's un-prefixed module call; - handleUtilsToolCall's reconstruction for `utils run`; - code_execution's inner calls. runCodeExecution records the provenance of the interactive call for its agent while the exec runs. The runner allows one interactive script per agent, and only an exec that can run sets it. It is cleared when the exec ends. A background script binds the provenance of the call that started it for its whole life. handleScriptToolCall / dispatchScriptToolCall pass it into dispatchToolCall, whose spreads already preserve it. Also the room-225 "lane 4" fix (handed over by Ada-1017). A puppeted agent_settings fell through executeToolCallFrom to the module registry's name parsing and failed as "Invalid tool name format", because the promise-based path lacked the synthesized verb the model path routes. dispatchAgentSettingsToolCall is split into runAgentSettingsToolCall, which returns the result with its traces, and a wrapper that pushes the tool-result event. executeToolCallFrom routes agent_settings to the same handler, so puppet, ModuleContext.callTool and utils callers reach it. Other synthesized verbs (gate_status, read_image, fetch_reference, …) are still model-path only on that route, as before. Tests: test/tool-call-origin.test.ts, 5 cases. - A puppeted agent_settings get and update reach the handler, a spy sees origin 'puppet', the update takes effect, and both pairs are stored. - `utils run` delivers origin 'puppet' and the admission id to the module. - ModuleContext.callTool is marked 'host'. - The model's own call carries nothing. - A puppeted code_execution (real python3) delivers origin and admission to the script's inner call, and a later host-started script carries only its own. Mutation checks, each caught: - removing the agent_settings route fails the lane-4 case; - dropping the utils carry fails the utils case; - dropping the registry carry fails three cases; - dropping the script provenance fails the code_execution case; - not marking callTool fails the host and code_execution cases. The suites around both changes pass: origin, lease, code_execution, utils surface, module context, both puppet suites, surgery receipts, operator surgery and runtime settings, 107/107. This is R3 of the agent-framework companion for the connectome-host body-change contract. Co-Authored-By: Agnes (Claude Opus 5.5) --- src/framework.ts | 89 ++++++++++++++++++---- src/module-registry.ts | 5 +- src/types/events.ts | 21 ++++++ test/tool-call-origin.test.ts | 134 ++++++++++++++++++++++++++++++++++ 4 files changed, 232 insertions(+), 17 deletions(-) create mode 100644 test/tool-call-origin.test.ts diff --git a/src/framework.ts b/src/framework.ts index 87e20dba0..7d0414851 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -22,6 +22,7 @@ import type { FrameworkConfig, InferencePolicy, ErrorPolicy, + OperatorAdmission, ProviderHoldHook, ErrorAction, FrameworkState, @@ -1372,6 +1373,13 @@ export class AgentFramework { private codeExecutionConfig: import('./types/index.js').CodeExecutionConfig | null = null; private codeExecutionRunners: Map = new Map(); private scriptToolWaiters: Map void> = new Map(); + /** + * Who initiated the interactive code_execution call running for each + * agent (its origin and admission), so the script's inner tool calls carry + * them too. The runner allows one interactive script per agent at a time. + * A background script binds its own at start. + */ + private scriptCallProvenance: Map> = new Map(); /** Agents whose running script hit an endTurn-carrying inner result — * deferred and applied to the final code_execution result instead of * cancelling the stream mid-script (which would wedge the turn). */ @@ -1505,7 +1513,7 @@ export class AgentFramework { // framework hands to its own module registry. The public // executeToolCall() is shared with model/ephemeral callers and always // stamps agent origin (see there). - callTool: (call) => this.executeToolCallFrom(call, { kind: 'module' }), + callTool: (call) => this.executeToolCallFrom({ ...call, origin: call.origin ?? 'host' }, { kind: 'module' }), notifyOps: (kind, agentName, message, data) => this.notifyOps(kind, agentName, message, data), }); } @@ -11613,12 +11621,17 @@ export class AgentFramework { * runAtSafeBoundary) it runs on the turn token that lease holds for the * agent instead of refusing the hold and minting its own; the lease's * release flushes what deferred meanwhile. + * - The call carries `origin: 'puppet'` through every dispatch path, so a + * tool can tell an operator's act from the agent's own. `opts.admission` + * marks it as carrying out an operator change the host already + * admitted (FrameworkConfig.operatorChangeGate), which a gated tool + * then applies instead of staging again. */ async puppetToolCall( agentName: string, toolName: string, input: Record, - opts?: { lease?: SafeBoundaryLease }, + opts?: { lease?: SafeBoundaryLease; admission?: OperatorAdmission }, ): Promise<{ toolUseId: string; result: ToolResult }> { const agent = this.agents.get(agentName); if (!agent) throw new Error(`Unknown agent: ${agentName}`); @@ -11689,6 +11702,8 @@ export class AgentFramework { name: toolName, input, callerAgentName: agentName, + origin: 'puppet', + ...(opts?.admission ? { admission: opts.admission } : {}), }); const durationMs = Date.now() - started; @@ -11808,6 +11823,13 @@ export class AgentFramework { return this.handleUtilsToolCall(call.callerAgentName ?? '__ephemeral__', call); } + // agent_settings, through the handler the model path uses. Without this + // route a puppeted agent_settings fell through to module-name parsing + // and failed as "Invalid tool name format". + if (call.name === 'agent_settings') { + return this.runAgentSettingsToolCall(call.callerAgentName ?? '__ephemeral__', call); + } + // Module tools return this.moduleRegistry.handleToolCall(call); } @@ -11979,8 +12001,12 @@ export class AgentFramework { surface.map((t) => t.name).filter((name) => name !== CODE_EXECUTION_TOOL_NAME), ); + const provenance: Pick | undefined = call.origin || call.admission + ? { ...(call.origin ? { origin: call.origin } : {}), ...(call.admission ? { admission: call.admission } : {}) } + : undefined; + if (input.background === true) { - const started = this.startBackgroundScript(agentName, input.code, injected, timeLimitMs); + const started = this.startBackgroundScript(agentName, input.code, injected, timeLimitMs, provenance); if (capNote && started.success && started.data && typeof started.data === 'object') { (started.data as Record).time_limit_note = capNote; } @@ -11989,7 +12015,19 @@ export class AgentFramework { const runner = this.getOrCreateScriptRunner(agentName); this.scriptDeferredEndTurn.delete(agentName); - const exec = await runner.exec(input.code, injected, undefined, timeLimitMs !== undefined ? { deadlineMs: timeLimitMs } : undefined); + // A busy runner refuses this exec, so only an exec that can run sets + // the provenance its inner calls read; it's cleared when that exec ends. + const owned = !runner.busy; + if (owned) { + if (provenance) this.scriptCallProvenance.set(agentName, provenance); + else this.scriptCallProvenance.delete(agentName); + } + let exec: Awaited>; + try { + exec = await runner.exec(input.code, injected, undefined, timeLimitMs !== undefined ? { deadlineMs: timeLimitMs } : undefined); + } finally { + if (owned && this.scriptCallProvenance.get(agentName) === provenance) this.scriptCallProvenance.delete(agentName); + } const endTurn = this.scriptDeferredEndTurn.delete(agentName); return { @@ -12022,6 +12060,8 @@ export class AgentFramework { code: string, injected: import('./code-execution/py-runner.js').InjectedTool[], timeLimitMs?: number, + /** Who started it: its inner calls carry this for the script's life. */ + provenance?: Pick, ): ToolResult { // v1: primary-agent-only. A conversation fork's or ephemeral's daemon // would outlive its owner and its wake would land in the primary @@ -12071,7 +12111,7 @@ export class AgentFramework { scriptTimeoutMs: cfg?.scriptTimeoutMs, idleReclaimMs: 0, // dedicated runner; lifetime is the exec deadline label: `${agentName}:${scriptId}`, - onToolCall: (toolName, args) => this.handleScriptToolCall(agentName, toolName, args), + onToolCall: (toolName, args) => this.handleScriptToolCall(agentName, toolName, args, provenance), }); const record: BackgroundScriptRecord = { @@ -12395,7 +12435,8 @@ export class AgentFramework { scriptTimeoutMs: cfg?.scriptTimeoutMs, idleReclaimMs: cfg?.idleReclaimMs, label: agentName, - onToolCall: (toolName, args) => this.handleScriptToolCall(agentName, toolName, args), + onToolCall: (toolName, args) => + this.handleScriptToolCall(agentName, toolName, args, this.scriptCallProvenance.get(agentName)), }); this.codeExecutionRunners.set(agentName, runner); } @@ -12415,11 +12456,12 @@ export class AgentFramework { agentName: string, toolName: string, args: Record, + provenance?: Pick, ): Promise { if (toolName === CODE_EXECUTION_TOOL_NAME) { return 'Error: code_execution cannot be called from within a script'; } - const result = await this.dispatchScriptToolCall(agentName, toolName, args); + const result = await this.dispatchScriptToolCall(agentName, toolName, args, provenance); if (result.endTurn) { // Deferred: applied to the final code_execution result (see // scriptDeferredEndTurn) — ending the turn mid-script would cancel the @@ -12461,6 +12503,7 @@ export class AgentFramework { agentName: string, toolName: string, args: Record, + provenance?: Pick, ): Promise { return new Promise((resolve) => { const callId = `pytc-${randomUUID()}`; @@ -12485,7 +12528,7 @@ export class AgentFramework { }); try { - this.dispatchToolCall(agentName, { id: callId, name: toolName, input: args }); + this.dispatchToolCall(agentName, { id: callId, name: toolName, input: args, ...provenance }); } catch (error) { if (this.scriptToolWaiters.delete(callId)) { clearTimeout(safety); @@ -16200,12 +16243,15 @@ export class AgentFramework { }; } // Same dispatch as a first-class tool call — the module cannot tell - // which surface the call came through. + // which surface the call came through. Who initiated it, and any + // admission it carries, travel with it. return this.moduleRegistry.handleToolCall({ id: call.id, name: def.name, input: args, callerAgentName: agentName, + ...(call.origin ? { origin: call.origin } : {}), + ...(call.admission ? { admission: call.admission } : {}), }); } default: @@ -16239,6 +16285,23 @@ export class AgentFramework { } private dispatchAgentSettingsToolCall(agentName: string, call: ToolCall): void { + const result = this.runAgentSettingsToolCall(agentName, call); + this.pushEvent({ + type: 'tool-result', + callId: call.id, + agentName, + moduleName: 'agent', + result, + }); + } + + /** + * The agent_settings handler, for every dispatch path: the model's + * (dispatchAgentSettingsToolCall wraps it in a tool-result event) and the + * promise-based one puppetToolCall, ModuleContext.callTool and + * code_execution's callers use (executeToolCallFrom). + */ + private runAgentSettingsToolCall(agentName: string, call: ToolCall): ToolResult { this.emitTrace({ type: 'tool:started', module: 'agent', @@ -16401,13 +16464,7 @@ export class AgentFramework { durationMs: 0, ...(result.isError ? { error: result.error } : {}), }); - this.pushEvent({ - type: 'tool-result', - callId: call.id, - agentName, - moduleName: 'agent', - result, - }); + return result; } /** Aggregate the event-tag vocabulary: reserved chat:* core + each connected diff --git a/src/module-registry.ts b/src/module-registry.ts index 7d2bcc87e..fc6b8cfa8 100644 --- a/src/module-registry.ts +++ b/src/module-registry.ts @@ -286,12 +286,15 @@ export class ModuleRegistry { }; } - // Create a call with the un-prefixed name (preserving caller identity) + // Create a call with the un-prefixed name (preserving caller identity, + // who initiated it, and any operator admission it carries) const moduleCall: ToolCall = { id: call.id, name: toolName, input: call.input, callerAgentName: call.callerAgentName, + ...(call.origin ? { origin: call.origin } : {}), + ...(call.admission ? { admission: call.admission } : {}), }; try { diff --git a/src/types/events.ts b/src/types/events.ts index b5e8afb8e..b56f37721 100644 --- a/src/types/events.ts +++ b/src/types/events.ts @@ -209,6 +209,27 @@ export interface ToolCall { input: unknown; /** The agent that made this tool call. Set by the framework dispatch layer. */ callerAgentName?: string; + /** + * Who initiated the call, when it isn't the agent's own model: 'puppet' + * for an operator acting through the agent's surface (puppetToolCall), + * 'host' for a module calling a tool (ModuleContext.callTool). Absent + * means the agent's model. Set where the call enters the framework and + * carried through every redispatch (the utils meta-tool, code_execution's + * inner calls), so a tool reached through a wrapper still sees who asked. + */ + origin?: 'puppet' | 'host'; + /** + * An operator change this call carries out, already admitted by the + * host's operator-change gate. Carried with `origin`, so a gated tool + * applies it instead of staging it again. + */ + admission?: OperatorAdmission; +} + +/** The host's admission of one operator change (see FrameworkConfig.operatorChangeGate). */ +export interface OperatorAdmission { + /** The host's id for the admitted change. */ + id: string; } // ============================================================================ diff --git a/test/tool-call-origin.test.ts b/test/tool-call-origin.test.ts new file mode 100644 index 000000000..27d77b26a --- /dev/null +++ b/test/tool-call-origin.test.ts @@ -0,0 +1,134 @@ +/** + * Who initiated a tool call travels with it. + * + * puppetToolCall stamps origin 'puppet' (and any admission the host passed); + * ModuleContext.callTool stamps 'host'; the agent's own model calls carry + * none. The stamp survives the framework's redispatches: the utils + * meta-tool's reconstruction and code_execution's inner calls. A puppeted + * agent_settings reaches the agent_settings handler (it used to fall + * through to module-name parsing as "Invalid tool name format"). + */ +import { describe, it, beforeEach, afterEach } from 'node:test'; +import assert from 'node:assert/strict'; +import { tmpdir } from 'node:os'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { join } from 'node:path'; +import { AgentFramework } from '../src/index.js'; +import type { InferenceRequest, Module, ModuleContext, ToolCall, ToolDefinition, ToolResult } from '../src/index.js'; +import { MockMembrane, createMockResponse } from './helpers/mock-membrane.js'; + +class ProbeModule implements Module { + readonly name = 'probe'; + readonly calls: ToolCall[] = []; + ctx!: ModuleContext; + async start(ctx: ModuleContext): Promise { this.ctx = ctx; } + async stop(): Promise {} + getTools(): ToolDefinition[] { + return [{ name: 'record', description: 'Record the call.', inputSchema: { type: 'object', properties: {} } }]; + } + getUtilities(): ToolDefinition[] { + return [{ name: 'util_record', description: 'Record the call, as a utility.', inputSchema: { type: 'object', properties: {} } }]; + } + async handleToolCall(call: ToolCall): Promise { + this.calls.push(call); + return { success: true, data: { recorded: call.name } }; + } + async onProcess(): Promise> { return {}; } +} + +type Internals = { + runAgentSettingsToolCall: (agentName: string, call: ToolCall) => ToolResult; + pendingRequests: InferenceRequest[]; +}; + +const provenance = (c: ToolCall) => ({ origin: c.origin, admission: c.admission?.id }); + +describe('tool-call origin', () => { + let tempDir: string; + let membrane: MockMembrane; + let probe: ProbeModule; + let framework: AgentFramework; + let quiet: typeof console.log; + + beforeEach(async () => { + tempDir = mkdtempSync(join(tmpdir(), 'tool-call-origin-')); + membrane = new MockMembrane(); + probe = new ProbeModule(); + framework = await AgentFramework.create({ + storePath: join(tempDir, 'test.chronicle'), + membrane: membrane.asMembrane(), + agents: [{ name: 'scout', model: 'test-model', systemPrompt: 'You are scout.' }], + modules: [probe], + codeExecution: { enabled: true }, + }); + quiet = console.log; + console.log = () => {}; + }); + afterEach(async () => { + console.log = quiet; + await framework.stop(); + rmSync(tempDir, { recursive: true, force: true }); + }); + + it('a puppeted agent_settings reaches the agent_settings handler, as an operator', async () => { + const i = framework as unknown as Internals; + const seen: Array<{ origin?: string }> = []; + const real = i.runAgentSettingsToolCall.bind(framework); + i.runAgentSettingsToolCall = (agentName, call) => { + seen.push({ origin: call.origin }); + return real(agentName, call); + }; + + const got = await framework.puppetToolCall('scout', 'agent_settings', { action: 'get' }); + assert.equal(got.result.success, true, String(got.result.error)); + assert.equal(typeof (got.result.data as { contextBudgetTokens?: unknown }).contextBudgetTokens, 'number'); + + const updated = await framework.puppetToolCall('scout', 'agent_settings', { action: 'update', context_budget_tokens: 120_000 }); + assert.equal(updated.result.success, true, String(updated.result.error)); + assert.equal(framework.getAgentRuntimeSettings('scout').contextBudgetTokens, 120_000); + assert.deepEqual(seen, [{ origin: 'puppet' }, { origin: 'puppet' }]); + + const types = (framework.getAgent('scout')!.getContextManager().getAllMessages() as Array<{ content: Array<{ type: string }> }>) + .map((m) => m.content[0]?.type); + assert.deepEqual(types, ['tool_use', 'tool_result', 'tool_use', 'tool_result'], 'both pairs stored'); + }); + + it('origin and admission survive the utils meta-tool', async () => { + const { result } = await framework.puppetToolCall( + 'scout', 'utils', { action: 'run', name: 'probe--util_record', args: {} }, { admission: { id: 'chg-7' } }, + ); + assert.equal(result.success, true, String(result.error)); + assert.deepEqual(probe.calls.map(provenance), [{ origin: 'puppet', admission: 'chg-7' }]); + }); + + it("marks a module's callTool as the host's", async () => { + await probe.ctx.callTool!({ id: 'host-1', name: 'probe--record', input: {}, callerAgentName: 'scout' }); + assert.deepEqual(probe.calls.map(provenance), [{ origin: 'host', admission: undefined }]); + }); + + it("leaves the agent's own model calls unmarked", async () => { + membrane.pushResponse(createMockResponse([{ type: 'tool_use', id: 'toolu_model_1', name: 'probe--record', input: {} }], 'tool_use')); + membrane.pushResponse(createMockResponse([{ type: 'text', text: 'done' }])); + (framework as unknown as Internals).pendingRequests.push( + { agentName: 'scout', reason: 'mcpl:channel-incoming', source: 'test', timestamp: Date.now() } as InferenceRequest, + ); + await framework.runUntilIdle(); + assert.deepEqual(probe.calls.map(provenance), [{ origin: undefined, admission: undefined }]); + }); + + it("carries origin and admission into code_execution's inner calls (real python3), and only for that script", async () => { + framework.start(); + const code = ['await probe__record({})', 'print("ok")'].join('\n'); + const { result } = await framework.puppetToolCall('scout', 'code_execution', { code }, { admission: { id: 'chg-9' } }); + assert.equal(result.success, true, String(result.error)); + assert.equal((result.data as { stdout?: string }).stdout?.trim(), 'ok'); + assert.equal(probe.calls.length, 1); + assert.match(String(probe.calls[0]!.id), /^pytc-/, 'an inner call'); + assert.deepEqual(provenance(probe.calls[0]!), { origin: 'puppet', admission: 'chg-9' }); + + // A later script the operator didn't start carries nothing over. + const hostRun = await probe.ctx.callTool!({ id: 'host-ce', name: 'code_execution', input: { code }, callerAgentName: 'scout' }); + assert.equal(hostRun.success, true, String(hostRun.error)); + assert.deepEqual(provenance(probe.calls[1]!), { origin: 'host', admission: undefined }); + }); +}); From 1d918f0b2234c8f463d64a511bbd20e017f0bbc6 Mon Sep 17 00:00:00 2001 From: Agnes-1009 Date: Wed, 7 Oct 2026 07:16:03 -0400 Subject: [PATCH 33/73] feat(modules): targeted, receipted, optionally durable ModuleContext.addMessage A host module that tells a particular agent something before changing its body (the connectome-host body-change contract) needs four things from ModuleContext.addMessage: - delivery into that agent's window, not the primary's; - the framework's turn guard applied to that agent, so the message never lands inside its live turn; - a receipt saying whether the message was stored or deferred; - for a notice the host acts on, a deferral that survives a restart. ModuleContext.addMessage only reached the primary, and a module's only alternative, the context manager's addMessage, bypasses the guard entirely. The framework's private addMessage already had targeting (forAgent) and an out-param placement. This exposes them, with one addition. addMessage(participant, content, metadata?, options?: ModuleMessageOptions), all optional: - forAgent: deliver into that registered agent's window, with the deferral guard evaluated against that agent's own turn state. An unknown name is logged and dropped, as before. - placement (MessagePlacement), filled with: - agent; - messageId when stored; - deferredId when deferred, the id the landed message carries as metadata.deferredWriteId; - durable: true when stored, or deferred with its queue persisted. False when deferred in memory only (not asked to be durable, or the persist failed or exceeded its cap). - durable: if the message has to wait, keep it in the persisted crash-recovery queue (quiesce's) rather than memory only. persistDeferredWrites gains { force } to enter persisted mode outside quiesce; the mode ends, as after quiesce, once the queue drains. It now reports whether the queue is on disk as it stands. Every flush path already stamps deferredWriteId and acks, and a serving boot already lands a restored remainder, so a durable deferral replays exactly once. The boot log no longer attributes every restored write to an unfinished resume. Deferral semantics are unchanged: the same guard decides when a message is held and when it lands. Resident agents share one message store, so a targeted message is visible across residents' windows as any message is; targeting chooses whose turn guards it. Tests: test/module-targeted-delivery.test.ts, 6 cases. - An idle target stores at once, with a stored placement. - The guard is the target's: held for a mid-turn target while the idle primary isn't. - A deferral behind the target's turn lands at the next boundary carrying its deferredId. - A durable deferral is in the recovery file, survives a crash-equivalent stop, lands at boot, and stays exactly once after a second reopen. - A non-durable deferral stays memory-only and is lost with the process, as before. - An unknown target leaves the placement empty. Mutation checks, each caught: - not forcing persistence fails the durable case; - reporting every placement as durable fails two cases; - the registry dropping the options fails all six. Neighbouring suites pass: host quiesce, module context, deferred-wake integrity, per-agent delivery, the lease and origin suites, 63/63. This is R4 of the agent-framework companion for the connectome-host body-change contract. Co-Authored-By: Agnes (Claude Opus 5.5) --- src/framework.ts | 47 +++++++-- src/module-registry.ts | 14 +-- src/types/module.ts | 47 ++++++++- test/module-targeted-delivery.test.ts | 146 ++++++++++++++++++++++++++ 4 files changed, 236 insertions(+), 18 deletions(-) create mode 100644 test/module-targeted-delivery.test.ts diff --git a/src/framework.ts b/src/framework.ts index 7d0414851..a6f8255ab 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -1502,7 +1502,11 @@ export class AgentFramework { // Initialize module registry with callbacks this.moduleRegistry = new ModuleRegistry(store, this.queue, { getAgents: () => Array.from(this.agents.values()), - addMessage: (p, c, m) => this.addMessage(p, c, m), + addMessage: (p, c, m, o) => this.addMessage(p, c, m, o ? { + ...(o.forAgent !== undefined ? { forAgent: o.forAgent } : {}), + ...(o.placement ? { placement: o.placement } : {}), + ...(o.durable ? { durable: true } : {}), + } : undefined), editMessage: (id, c) => this.editMessage(id, c), removeMessage: (id) => this.removeMessage(id), getMessage: (id) => this.getMessage(id), @@ -1843,7 +1847,8 @@ export class AgentFramework { if (!hostMode?.quiesced && restoredWrites > 0) { console.error( `[host-mode] ${restoredWrites} deferred context write(s) found at a serving boot ` + - `(an earlier resume did not finish its flush) — landing them now`, + `(persisted before the restart: a resume that didn't finish its flush, or a durable ` + + `module delivery still waiting) — landing them now`, ); await framework.flushDeferredWrites('boot-recovery'); } @@ -3541,7 +3546,15 @@ export class AgentFramework { * Deferrals outside a quiesce window (turn-alive, mid-tool-cycle) are * still memory-only: they flush within the turn, as before. */ - private persistDeferredWrites(): void { + /** + * Returns true when the durable queue is on disk as it now stands (written, + * or cleared because nothing is left), false when it is kept in memory + * only: not in persisted mode, over its size cap, or the write failed. + * `opts.force` enters persisted mode outside quiesce, for a module message + * that asked to be durable (ModuleMessageOptions.durable); the mode ends, + * as it does after quiesce, once the queue drains. + */ + private persistDeferredWrites(opts?: { force?: boolean }): boolean { try { // The durable queue is everything not yet acked: writes still pending // AND writes handed to a context manager whose sync has not happened. @@ -3549,7 +3562,7 @@ export class AgentFramework { // split: a re-deferred entry must not jump the queue. const durable = [...this.unackedDeferredWrites, ...this.deferredMessages].sort(bySeq); const scanFrom = Object.fromEntries(this.deferredScanFrom); - if ((this.quiesced || this.deferredWritesPersisted) && durable.length > 0) { + if ((this.quiesced || this.deferredWritesPersisted || opts?.force) && durable.length > 0) { const payload = JSON.stringify(durable); if (payload.length > DEFERRED_WRITES_PERSIST_CAP_BYTES) { if (!this.deferredWritesCapWarned) { @@ -3560,7 +3573,7 @@ export class AgentFramework { `a crash before resume loses them`, ); } - return; + return false; } if (this.deferredWritesPath) { this.writeRecoveryFile(this.deferredWritesPath, { version: 2, pending: durable, scanFrom }); @@ -3569,6 +3582,7 @@ export class AgentFramework { this.store.setStateJson(DEFERRED_WRITES_ID, { version: 2, pending: durable, scanFrom }); } this.deferredWritesPersisted = true; + return true; } else if (this.deferredWritesPersisted) { if (this.deferredWritesPath) { this.writeRecoveryFile(this.deferredWritesPath, { version: 1, pending: [] }); @@ -3576,9 +3590,12 @@ export class AgentFramework { this.store.setStateJson(DEFERRED_WRITES_ID, []); } this.deferredWritesPersisted = false; + return true; } + return false; } catch (err) { console.error('[host-mode] failed to persist deferred context writes:', err); + return false; } } @@ -13124,8 +13141,12 @@ export class AgentFramework { * minting a second, independently replayable one. */ deferredWriteId?: string; - /** Out-param: where the message landed (RFC-006 needs to find it again). */ - placement?: { agent?: string; messageId?: MessageId; deferredId?: string }; + /** Out-param: where the message landed (RFC-006 needs to find it + * again; a module's placement receipt). */ + placement?: { agent?: string; messageId?: MessageId; deferredId?: string; durable?: boolean }; + /** If deferred, persist the recovery queue even outside quiesce + * (ModuleMessageOptions.durable). Deferral itself is unchanged. */ + durable?: boolean; /** * RFC-006 assembly: the named agent's turn is alive but its compile has * not run yet (the same window the turn-start deferred flush writes @@ -13197,8 +13218,14 @@ export class AgentFramework { this.unackedDeferredWrites = this.unackedDeferredWrites.filter((m) => m.id !== id); } this.deferredMessages.push({ id, seq: seq ?? ++this.deferredSeq, participant, content, metadata, forAgent: opts?.forAgent }); - if (this.quiesced || this.deferredWritesPersisted) this.persistDeferredWrites(); - if (opts?.placement) { opts.placement.agent = agent.name; opts.placement.deferredId = id; } + const persisted = this.quiesced || this.deferredWritesPersisted || opts?.durable + ? this.persistDeferredWrites({ force: opts?.durable === true }) + : false; + if (opts?.placement) { + opts.placement.agent = agent.name; + opts.placement.deferredId = id; + opts.placement.durable = persisted; + } return '' as MessageId; // Deferred — flushed at the target's next boundary } @@ -13207,7 +13234,7 @@ export class AgentFramework { content, opts?.deferredWriteId ? withDeferredWriteId(metadata, opts.deferredWriteId) : metadata, ); - if (opts?.placement) { opts.placement.agent = agent.name; opts.placement.messageId = stored; } + if (opts?.placement) { opts.placement.agent = agent.name; opts.placement.messageId = stored; opts.placement.durable = true; } return stored; } diff --git a/src/module-registry.ts b/src/module-registry.ts index fc6b8cfa8..374a6e683 100644 --- a/src/module-registry.ts +++ b/src/module-registry.ts @@ -11,6 +11,7 @@ import type { import type { Module, ModuleContext, + ModuleMessageOptions, ProcessState, ProcessQueue, ToolDefinition, @@ -66,7 +67,7 @@ export class ModuleRegistry { private store: JsStore; private queue: ProcessQueue; private getAgents: () => Agent[]; - private addMessageFn: (participant: string, content: ContentBlock[], metadata?: MessageMetadata) => MessageId; + private addMessageFn: (participant: string, content: ContentBlock[], metadata?: MessageMetadata, options?: ModuleMessageOptions) => MessageId; private editMessageFn: (id: MessageId, content: ContentBlock[]) => void; private removeMessageFn: (id: MessageId) => void; private getMessageFn: (id: MessageId) => StoredMessage | null; @@ -90,7 +91,7 @@ export class ModuleRegistry { queue: ProcessQueue, options: { getAgents: () => Agent[]; - addMessage: (participant: string, content: ContentBlock[], metadata?: MessageMetadata) => MessageId; + addMessage: (participant: string, content: ContentBlock[], metadata?: MessageMetadata, options?: ModuleMessageOptions) => MessageId; editMessage: (id: MessageId, content: ContentBlock[]) => void; removeMessage: (id: MessageId) => void; getMessage: (id: MessageId) => StoredMessage | null; @@ -445,7 +446,7 @@ class ModuleContextImpl implements ModuleContext { private registry: ModuleRegistry; readonly isRestart: boolean; private getAgentsFn: () => Agent[]; - private addMessageFn: (participant: string, content: ContentBlock[], metadata?: MessageMetadata) => MessageId; + private addMessageFn: (participant: string, content: ContentBlock[], metadata?: MessageMetadata, options?: ModuleMessageOptions) => MessageId; private editMessageFn: (id: MessageId, content: ContentBlock[]) => void; private removeMessageFn: (id: MessageId) => void; private getMessageFn: (id: MessageId) => StoredMessage | null; @@ -465,7 +466,7 @@ class ModuleContextImpl implements ModuleContext { registry: ModuleRegistry, isRestart: boolean, getAgents: () => Agent[], - addMessage: (participant: string, content: ContentBlock[], metadata?: MessageMetadata) => MessageId, + addMessage: (participant: string, content: ContentBlock[], metadata?: MessageMetadata, options?: ModuleMessageOptions) => MessageId, editMessage: (id: MessageId, content: ContentBlock[]) => void, removeMessage: (id: MessageId) => void, getMessage: (id: MessageId) => StoredMessage | null, @@ -596,9 +597,10 @@ class ModuleContextImpl implements ModuleContext { addMessage( participant: string, content: ContentBlock[], - metadata?: MessageMetadata & { external?: ExternalIdRef } + metadata?: MessageMetadata & { external?: ExternalIdRef }, + options?: ModuleMessageOptions, ): MessageId { - const id = this.addMessageFn(participant, content, metadata); + const id = this.addMessageFn(participant, content, metadata, options); // Track external ID if provided if (metadata?.external) { diff --git a/src/types/module.ts b/src/types/module.ts index 724abaae0..52bb6afa5 100644 --- a/src/types/module.ts +++ b/src/types/module.ts @@ -146,6 +146,43 @@ export interface AgentSettingsExtension { /** * Context provided to modules for interacting with the framework. */ +/** Delivery options for ModuleContext.addMessage. */ +export interface ModuleMessageOptions { + /** + * Deliver into this registered agent's window instead of the primary's, + * with the deferral guard evaluated against that agent's own turn state. + * An unknown name is logged and the message dropped (placement stays empty). + */ + forAgent?: string; + /** Out-param: filled with where the message went. */ + placement?: MessagePlacement; + /** + * If the message has to be deferred, keep it in the persisted + * crash-recovery queue (the one quiesce uses) rather than in memory only, + * so a restart before its flush replays it once (deduplicated by its + * deferred-write id). Deferral itself is unchanged: the same guard decides + * when it is held and when it lands. + */ + durable?: boolean; +} + +/** Where a module's message went (ModuleMessageOptions.placement). */ +export interface MessagePlacement { + /** The agent whose window it went to; unset when it was dropped. */ + agent?: string; + /** Set when it was stored at once. */ + messageId?: MessageId; + /** Set when it was deferred: the deferred write's id, also stamped as + * `metadata.deferredWriteId` on the message when it lands. */ + deferredId?: string; + /** + * True when it was stored, or deferred with its queue persisted. False + * when it is deferred in memory only: not asked to be durable, or the + * persist failed or exceeded its size cap. + */ + durable?: boolean; +} + export interface ModuleContext { /** * Get persistent state for this module. @@ -219,12 +256,18 @@ export interface ModuleContext { getModule(name: string): T | null; /** - * Add a message to the conversation. + * Add a message to the conversation: the primary agent's, or with + * `options.forAgent`, that agent's. The framework's turn guard applies + * either way: while the target is mid-turn, mid tool cycle or quiesced, the + * message is deferred to its next boundary and the returned id is ''. + * `options.placement` reports where it went, and `options.durable` keeps a + * deferral in the crash-recovery queue (see ModuleMessageOptions). */ addMessage( participant: string, content: ContentBlock[], - metadata?: MessageMetadata & { external?: ExternalIdRef } + metadata?: MessageMetadata & { external?: ExternalIdRef }, + options?: ModuleMessageOptions, ): MessageId; /** diff --git a/test/module-targeted-delivery.test.ts b/test/module-targeted-delivery.test.ts new file mode 100644 index 000000000..f61481330 --- /dev/null +++ b/test/module-targeted-delivery.test.ts @@ -0,0 +1,146 @@ +/** + * ModuleContext.addMessage options: a module can deliver into a named + * agent's window under that agent's turn guard, learn where the message + * went, and ask for a deferral to survive a restart. + * + * Deferral is unchanged: the same guard decides when a message is held and + * when it lands. `durable` only keeps a held message in the persisted + * recovery queue (the one quiesce uses) instead of memory, so it replays + * exactly once after a crash. + */ +import { describe, it, beforeEach, afterEach } from 'node:test'; +import assert from 'node:assert/strict'; +import { tmpdir } from 'node:os'; +import { existsSync, mkdtempSync, readFileSync, rmSync } from 'node:fs'; +import { join } from 'node:path'; +import { AgentFramework } from '../src/index.js'; +import type { MessagePlacement, Module, ModuleContext, ToolDefinition, ToolResult } from '../src/index.js'; +import { MockMembrane } from './helpers/mock-membrane.js'; + +class Courier implements Module { + readonly name = 'courier'; + ctx!: ModuleContext; + async start(ctx: ModuleContext): Promise { this.ctx = ctx; } + async stop(): Promise {} + getTools(): ToolDefinition[] { return []; } + async handleToolCall(): Promise { return { success: true }; } + async onProcess(): Promise> { return {}; } +} + +type Internals = { activeTurnTokens: Map }; + +const texts = (framework: AgentFramework, agent: string) => + (framework.getAgent(agent)!.getContextManager().getAllMessages() as Array<{ + content: Array<{ text?: string }>; metadata?: { deferredWriteId?: string }; + }>).map((m) => ({ text: m.content[0]?.text, deferredWriteId: m.metadata?.deferredWriteId })); + +describe('ModuleContext.addMessage delivery options', () => { + let tempDir: string; + let storePath: string; + let courier: Courier; + let framework: AgentFramework; + let quiet: typeof console.log; + let quietErr: typeof console.error; + + const open = async () => { + courier = new Courier(); + framework = await AgentFramework.create({ + storePath, + membrane: new MockMembrane().asMembrane(), + agents: [ + { name: 'scout', model: 'test-model', systemPrompt: 'You are scout.' }, + { name: 'other', model: 'test-model', systemPrompt: 'You are other.' }, + ], + modules: [courier], + }); + }; + + beforeEach(async () => { + tempDir = mkdtempSync(join(tmpdir(), 'module-delivery-')); + storePath = join(tempDir, 'test.chronicle'); + quiet = console.log; + quietErr = console.error; + console.log = () => {}; + console.error = () => {}; + await open(); + }); + afterEach(async () => { + await framework.stop(); + console.log = quiet; + console.error = quietErr; + rmSync(tempDir, { recursive: true, force: true }); + }); + + it('lands in the named idle agent at once, and the placement says where', async () => { + const placement: MessagePlacement = {}; + const id = courier.ctx.addMessage('user', [{ type: 'text', text: 'for other' }], undefined, { forAgent: 'other', placement }); + assert.ok(id, 'stored, with an id'); + assert.deepEqual(placement, { agent: 'other', messageId: id, durable: true }); + assert.deepEqual(texts(framework, 'other').map((m) => m.text), ['for other']); + }); + + it("holds a targeted message for the target's turn even when the primary is idle", async () => { + // The guard is the target's: other mid-turn defers it although scout, + // the primary, has no turn alive. + (framework as unknown as Internals).activeTurnTokens.set('other', 77); + const placement: MessagePlacement = {}; + assert.equal(courier.ctx.addMessage('user', [{ type: 'text', text: 'held for other' }], undefined, { forAgent: 'other', placement }), ''); + assert.equal(placement.agent, 'other'); + // And an untargeted message for the idle primary is not held by other's turn. + const direct: MessagePlacement = {}; + assert.ok(courier.ctx.addMessage('user', [{ type: 'text', text: 'for the primary' }], undefined, { placement: direct })); + assert.equal(direct.agent, 'scout'); + assert.equal(typeof direct.messageId, 'string'); + }); + + it("waits behind the target's live turn, then lands carrying its deferred-write id", async () => { + (framework as unknown as Internals).activeTurnTokens.set('other', 77); + const placement: MessagePlacement = {}; + const id = courier.ctx.addMessage('user', [{ type: 'text', text: 'after your turn' }], undefined, { forAgent: 'other', placement }); + assert.equal(id, '', 'deferred'); + assert.equal(placement.agent, 'other'); + assert.equal(typeof placement.deferredId, 'string'); + assert.equal(placement.durable, false, 'not asked to be durable: memory only'); + assert.deepEqual(texts(framework, 'other'), []); + + (framework as unknown as Internals).activeTurnTokens.delete('other'); // the turn ends + await framework.runAtSafeBoundary({ verb: 'boundary' }, async () => {}); // a boundary flush + assert.deepEqual(texts(framework, 'other'), [{ text: 'after your turn', deferredWriteId: placement.deferredId }]); + }); + + it('keeps a durable deferral through a crash, and lands it exactly once', async () => { + (framework as unknown as Internals).activeTurnTokens.set('other', 77); + const placement: MessagePlacement = {}; + courier.ctx.addMessage('user', [{ type: 'text', text: 'must not be lost' }], undefined, { forAgent: 'other', placement, durable: true }); + assert.equal(placement.durable, true, 'persisted'); + const queuePath = join(storePath, 'recovery', 'deferred-writes.json'); + const queued = JSON.parse(readFileSync(queuePath, 'utf8')) as { pending: Array<{ id: string; forAgent?: string }> }; + assert.deepEqual(queued.pending.map((p) => ({ id: p.id, forAgent: p.forAgent })), [{ id: placement.deferredId, forAgent: 'other' }]); + + await framework.stop(); // crash-equivalent: the turn never ended and nothing flushed + await open(); + assert.deepEqual(texts(framework, 'other'), [{ text: 'must not be lost', deferredWriteId: placement.deferredId }], 'landed at boot'); + await framework.stop(); + await open(); + assert.deepEqual(texts(framework, 'other').map((m) => m.text), ['must not be lost'], 'still exactly once after a second reopen'); + }); + + it('leaves a deferral that did not ask to be durable in memory, as before', async () => { + (framework as unknown as Internals).activeTurnTokens.set('other', 77); + const placement: MessagePlacement = {}; + courier.ctx.addMessage('user', [{ type: 'text', text: 'best effort' }], undefined, { forAgent: 'other', placement }); + assert.equal(placement.durable, false); + const queuePath = join(storePath, 'recovery', 'deferred-writes.json'); + assert.equal(existsSync(queuePath) && (JSON.parse(readFileSync(queuePath, 'utf8')) as { pending: unknown[] }).pending.length > 0, false); + await framework.stop(); + await open(); + assert.deepEqual(texts(framework, 'other'), [], 'lost with the process, as memory-only deferrals always were'); + }); + + it('drops a message for an unknown agent, and the empty placement says so', async () => { + const placement: MessagePlacement = {}; + const id = courier.ctx.addMessage('user', [{ type: 'text', text: 'to nobody' }], undefined, { forAgent: 'ghost', placement, durable: true }); + assert.equal(id, ''); + assert.deepEqual(placement, {}); + }); +}); From 9d25528c4cf304690262f7ad9e79b985aa01a22c Mon Sep 17 00:00:00 2001 From: Agnes-1009 Date: Wed, 7 Oct 2026 07:31:32 -0400 Subject: [PATCH 34/73] feat(modules): Module.getSubconsciousTools offers tools on the subconscious surface The subconscious resident (tune-out) has its own small surface: its fixed tools plus think / skip_reply / end_turn. It has no residents' board and no `utils`. A host module whose tool every addressee must be able to call can't offer it there. The connectome-host body-change contract has one: an agent's ready/hold answer to a change the host asks it about. The subconscious shares the store, so a branch move asks it too, and today it couldn't answer. Module.getSubconsciousTools?() returns ToolDefinitions in the same shape and with the same un-prefixed names as getTools(). Each appears on the subconscious surface as `module--name`. A call takes the ordinary module dispatch and arrives at handleToolCall with callerAgentName naming the subconscious. Residents keep reaching the same tools through getTools() or getUtilities(). Nothing else opens: the subconscious still gets no `utils`, and the residents' board doesn't gain these tools. The subconscious is created before modules are added, and its Agent holds a fixed allowedTools list. getToolsForAgent(subconscious) now appends moduleRegistry.getAllSubconsciousTools(). The allowed list is a framework-owned array that syncSubconsciousTools() rewrites in place to the fixed names plus the current modules' tools. That runs at creation and after every addModule and removeModule (boot's modules arrive through addModule too). Tests: test/subconscious-module-tools.test.ts, 3 cases. - A module's subconscious tool is on the subconscious surface (listToolClasses) without `utils`, and isn't a first-class resident tool, which the resident reaches through `utils` instead. - A module added after boot offers its tool at once, and removing it withdraws the tool while the others stay. - The subconscious's own call, on the queued tool-call path, reaches the module with callerAgentName set to the subconscious and no origin. A puppet call passes the on-surface check and arrives marked origin 'puppet'. Mutation checks: - leaving module tools off the surface fails all three; - not syncing the allowed list on addModule fails all three, since boot's modules arrive that way. Neighbouring suites pass: tune-out e2e, lifecycle and gate composition, utils surface, tool-class listing and tool presentation, 32/32. This is R5 of the agent-framework companion for the connectome-host body-change contract. Co-Authored-By: Agnes (Claude Opus 5.5) --- src/framework.ts | 31 +++++++- src/module-registry.ts | 11 +++ src/types/module.ts | 13 ++++ test/subconscious-module-tools.test.ts | 103 +++++++++++++++++++++++++ 4 files changed, 156 insertions(+), 2 deletions(-) create mode 100644 test/subconscious-module-tools.test.ts diff --git a/src/framework.ts b/src/framework.ts index a6f8255ab..6e54d455f 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -1219,6 +1219,8 @@ export class AgentFramework { // tool_use → tool_result adjacency required by the Anthropic API). /** The subconscious resident's registry name (issue #77), or null. */ private subconsciousAgentName: string | null = null; + /** The subconscious Agent's allowedTools, kept current by syncSubconsciousTools. */ + private readonly subconsciousAllowedTools: string[] = []; /** Its windowed strategy — the coordinator moves the anchor on tune-out entry. */ private subconsciousStrategy: WindowedPassthroughStrategy | null = null; /** Its config (speak_in_channel gate, voice block provenance). */ @@ -2463,6 +2465,7 @@ export class AgentFramework { */ async addModule(module: Module): Promise { await this.moduleRegistry.addModule(module); + this.syncSubconsciousTools(); this.emitTrace({ type: 'module:added', moduleName: module.name }); } @@ -2471,9 +2474,29 @@ export class AgentFramework { */ async removeModule(name: string): Promise { await this.moduleRegistry.removeModule(name); + this.syncSubconsciousTools(); this.emitTrace({ type: 'module:removed', moduleName: name }); } + /** + * The subconscious's allowed tools are its fixed surface plus whatever + * modules offer it now (Module.getSubconsciousTools). It is created + * before modules are added, so the list its Agent holds is rewritten in + * place whenever modules change. + */ + private syncSubconsciousTools(): void { + const allowed = this.subconsciousAllowedTools; + allowed.splice( + 0, + allowed.length, + ...SUBCONSCIOUS_TOOL_NAMES, + 'think', + 'skip_reply', + 'end_turn', + ...this.moduleRegistry.getAllSubconsciousTools().map((t) => t.name), + ); + } + /** * Get an agent by name. */ @@ -2845,7 +2868,9 @@ export class AgentFramework { ?? this.getAgentRuntimeSettings(agentName).sameRoundThinkTextPolicy, ) : t); - return [...SUBCONSCIOUS_TOOLS, ...basics]; + // Plus what modules offer it (Module.getSubconsciousTools), allowed by + // syncSubconsciousTools. + return [...SUBCONSCIOUS_TOOLS, ...this.moduleRegistry.getAllSubconsciousTools(), ...basics]; } return [...this.getAllTools(), ...(this.toolPresentations.has(agentName) ? presentationTools(this.toolPresentations.get(agentName)!.config.cataloguePath) : [])].map((tool) => { if (tool.name === 'think') { @@ -7227,8 +7252,10 @@ export class AgentFramework { // locus, and bare prose must not fall through to the default channel. // Speech happens only via speak_in_channel (its own marked voice). proseRouting: 'explicit', - allowedTools: [...SUBCONSCIOUS_TOOL_NAMES, 'think', 'skip_reply', 'end_turn'], + // Rewritten in place by syncSubconsciousTools as modules come and go. + allowedTools: this.subconsciousAllowedTools, }; + this.syncSubconsciousTools(); const agent = new Agent(agentConfig, contextManager, this.membrane); this.restoreToolResultGuardSetting(agent); agent.markContextConsumed(); // RFC-006: nothing stored before now is replaceable diff --git a/src/module-registry.ts b/src/module-registry.ts index 374a6e683..81dcf0428 100644 --- a/src/module-registry.ts +++ b/src/module-registry.ts @@ -248,6 +248,17 @@ export class ModuleRegistry { * out of the model-facing tool list and surfaced through the framework's * `utils` meta-tool; dispatch goes through the same handleToolCall. */ + /** Every module's getSubconsciousTools(), prefixed `module--name`. */ + getAllSubconsciousTools(): ToolDefinition[] { + const tools: ToolDefinition[] = []; + for (const module of this.modules.values()) { + for (const tool of module.getSubconsciousTools?.() ?? []) { + tools.push({ ...tool, name: `${module.name}--${tool.name}` }); + } + } + return tools; + } + getAllUtilities(): ToolDefinition[] { const utilities: ToolDefinition[] = []; for (const module of this.modules.values()) { diff --git a/src/types/module.ts b/src/types/module.ts index 52bb6afa5..007e43398 100644 --- a/src/types/module.ts +++ b/src/types/module.ts @@ -75,6 +75,19 @@ export interface Module { */ getUtilities?(): ToolDefinition[]; + /** + * Tools this module offers the subconscious resident (tune-out), whose + * surface is otherwise its own small set: no residents' tool board and no + * `utils`. Same ToolDefinition shape and un-prefixed names as getTools(); + * each appears there as `module--name`, and a call arrives at this + * module's handleToolCall with callerAgentName naming the subconscious. + * For tools every addressee must be able to call, such as an agent's + * answer to a host that asks it directly; residents reach the same tools + * through getTools() or getUtilities(). Optional; omitting it offers the + * subconscious nothing. + */ + getSubconsciousTools?(): ToolDefinition[]; + /** * Handle a tool call. * Tool name is without module prefix. diff --git a/test/subconscious-module-tools.test.ts b/test/subconscious-module-tools.test.ts new file mode 100644 index 000000000..fbea8f724 --- /dev/null +++ b/test/subconscious-module-tools.test.ts @@ -0,0 +1,103 @@ +/** + * Module.getSubconsciousTools: a module can offer tools on the subconscious + * resident's surface, which otherwise has no residents' board and no utils. + * The subconscious is created before modules are added, so the tools it may + * use follow the modules registered now, including ones added or removed at + * runtime, and its call reaches the module under its own name. + */ +import { describe, it, beforeEach, afterEach } from 'node:test'; +import assert from 'node:assert/strict'; +import { tmpdir } from 'node:os'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { join } from 'node:path'; +import { AgentFramework } from '../src/index.js'; +import type { Module, ToolCall, ToolDefinition, ToolResult } from '../src/index.js'; +import { MockMembrane } from './helpers/mock-membrane.js'; + +class AnswerModule implements Module { + readonly calls: ToolCall[] = []; + constructor(readonly name = 'body') {} + async start(): Promise {} + async stop(): Promise {} + getTools(): ToolDefinition[] { return []; } + getUtilities(): ToolDefinition[] { + return [{ name: 'answer', description: 'Answer the host.', inputSchema: { type: 'object', properties: {} } }]; + } + getSubconsciousTools(): ToolDefinition[] { + return [{ name: 'answer', description: 'Answer the host.', inputSchema: { type: 'object', properties: {} } }]; + } + async handleToolCall(call: ToolCall): Promise { + this.calls.push(call); + return { success: true, data: { answered: call.callerAgentName } }; + } + async onProcess(): Promise> { return {}; } +} + +type Internals = { + subconsciousAgentName: string | null; + dispatchToolCall(agentName: string, call: ToolCall): void; +}; + +describe('Module.getSubconsciousTools', () => { + let tempDir: string; + let framework: AgentFramework; + let answers: AnswerModule; + let sub: string; + let quiet: typeof console.log; + + beforeEach(async () => { + tempDir = mkdtempSync(join(tmpdir(), 'subconscious-tools-')); + answers = new AnswerModule(); + framework = await AgentFramework.create({ + storePath: join(tempDir, 'test.chronicle'), + membrane: new MockMembrane().asMembrane(), + agents: [{ name: 'scout', model: 'test-model', systemPrompt: 'You are scout.' }], + subconscious: { enabled: true, systemPrompt: 'You are the Subconscious.' }, + modules: [answers], + }); + sub = (framework as unknown as Internals).subconsciousAgentName!; + quiet = console.log; + console.log = () => {}; + }); + afterEach(async () => { + console.log = quiet; + await framework.stop(); + rmSync(tempDir, { recursive: true, force: true }); + }); + + const surface = (agent: string) => framework.listToolClasses(agent).map((t) => t.tool); + + it('offers them on the subconscious surface, without utils, and not on the residents\' board', async () => { + assert.ok(sub, 'a subconscious exists'); + const subTools = surface(sub); + assert.ok(subTools.includes('body--answer'), `subconscious surface: ${subTools.join(', ')}`); + assert.ok(!subTools.includes('utils'), 'no utils on the subconscious surface'); + const scoutTools = surface('scout'); + assert.ok(!scoutTools.includes('body--answer'), 'not a first-class resident tool'); + assert.ok(scoutTools.includes('utils'), 'the resident reaches it as a utility'); + }); + + it('follows modules added and removed at runtime', async () => { + await framework.addModule(new AnswerModule('late')); + assert.ok(surface(sub).includes('late--answer'), 'added after boot'); + await framework.removeModule('late'); + assert.ok(!surface(sub).includes('late--answer'), 'withdrawn with its module'); + assert.ok(surface(sub).includes('body--answer'), 'the others stay'); + }); + + it("reaches the module under the subconscious's own name, from its model or a puppet", async () => { + // The model path: the subconscious emits the call, which travels as a + // queued tool-call event (the event loop delivers it to the module). + framework.start(); + (framework as unknown as Internals).dispatchToolCall(sub, { id: 'toolu_sub_1', name: 'body--answer', input: {} }); + for (let n = 0; n < 50 && answers.calls.length === 0; n++) await new Promise((r) => setTimeout(r, 5)); + assert.deepEqual(answers.calls.map((c) => ({ name: c.name, caller: c.callerAgentName, origin: c.origin })), [ + { name: 'answer', caller: sub, origin: undefined }, + ]); + + // And the puppet's on-surface check now admits it, marked as the operator's. + const { result } = await framework.puppetToolCall(sub, 'body--answer', {}); + assert.equal(result.success, true, String(result.error)); + assert.deepEqual(answers.calls.map((c) => c.origin), [undefined, 'puppet']); + }); +}); From b83641de91b22ae567f77ccabca30c9294551408 Mon Sep 17 00:00:00 2001 From: Agnes-1009 Date: Wed, 7 Oct 2026 07:49:42 -0400 Subject: [PATCH 35/73] fix(framework): read the lease and subconscious fields safely in prototype-built harnesses MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The first full run of the companion stack (d2435a9, --test-concurrency=4) failed four busy-agent-scheduler tests with "Cannot read properties of undefined (reading 'length')" in tryGrantSafeBoundary. Those tests build the framework with Object.create(AgentFramework.prototype), which leaves every class field undefined. The scheduler code already allows for that ("Truthiness, not `!== null`: prototype-built harnesses … leave the slot undefined"), and the new lease reads didn't. - tryGrantSafeBoundary and the waiting-lease hold in processInferenceRequests read `this.boundaryWaiters?.length`. A missing field reads as nobody waiting. - syncSubconsciousTools returns when its array is missing. No subconscious exists in such a harness. All seventeen prototype-built suites pass with the subconscious and lease suites, 141/141, including the four that failed. The same full run also failed three mcpl-coalescing timing tests: vector 34c, R16 (cancelled at the 120 s timeout) and R17. Ada's full run of her base 287eef2, without this stack, failed the same three under the same host load, so those three predate this stack. Co-Authored-By: Agnes (Claude Opus 5.5) --- src/framework.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/framework.ts b/src/framework.ts index 6e54d455f..6cf557bea 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -2486,6 +2486,7 @@ export class AgentFramework { */ private syncSubconsciousTools(): void { const allowed = this.subconsciousAllowedTools; + if (!allowed) return; // prototype-built harnesses: no field, no subconscious allowed.splice( 0, allowed.length, @@ -6264,7 +6265,9 @@ export class AgentFramework { * lease is granted as soon as the last turn or ephemeral run ends. */ private tryGrantSafeBoundary(): void { - if (this.boundaryWaiters.length === 0 || this.heldLease || this.surgeryHold) return; + // Optional chaining: prototype-built harnesses leave fields undefined, + // which must read as "nobody waiting" (see surgeryHeld below). + if (!this.boundaryWaiters?.length || this.heldLease || this.surgeryHold) return; if (this.ephemeralRuns.size > 0 || this.storeBusyReasons().length > 0) return; const waiter = this.boundaryWaiters.shift()!; const { tokens, release } = this.takeStoreReservation(waiter.verb, 'safe-boundary'); @@ -9068,7 +9071,7 @@ export class AgentFramework { // A registered ephemeral stream's wakes pass too: the lease waits for // its run to end, and holding them would deadlock it. Once the lease is // granted, its reservation (surgeryHold, below) holds everything. - if (this.boundaryWaiters.length > 0 && !this.ephemeralRuns.has(agentName)) { + if (this.boundaryWaiters?.length && !this.ephemeralRuns.has(agentName)) { const held = requests.filter((r) => !isTurnContinuation(r.reason)); if (held.length > 0) { this.pendingRequests.push(...held); From b032920f8dfff57afc29ab5ebb34bb344cd32f99 Mon Sep 17 00:00:00 2001 From: Agnes-1009 Date: Wed, 7 Oct 2026 08:21:32 -0400 Subject: [PATCH 36/73] fix(framework): close four gaps in the lease, delivery and origin contracts (review) Kit's review of the stack through d2435a9 (room-222 #43846) found four gaps, each reproduced with public APIs (/tmp/kit-1008-af-body-probe.mjs). 1. An ephemeral creation in progress was outside the safe boundary. A lease could be granted while createEphemeralAgent was still initializing the store (Kit paused a strategy's async initialize), and the creation then finished under the lease. - createEphemeralAgent gains opts.requestedBy and an admission step. With no lease waiting or held, a creation proceeds. - While a lease waits, a creation proceeds only for a stream the lease is already draining: an agent with a real turn alive (not a lease's reservation token) or a registered ephemeral run. Such a stream may need the new agent to finish, as a sync spawn does. - Any other creation, and every creation while a lease is held, waits until no lease is waiting or held. A held request isn't pending, so it never blocks the grant it waits for. - An admitted creation and its candidate are pending (ephemeralPending) until runEphemeralToCompletion takes the candidate, in the same synchronous step that registers the run, or until cleanup() releases it. The grant waits for every pending one. - Documented: call cleanup() for a candidate that will never run. connectome-host's SubagentModule already does, in finally. 2. A durable deferral whose store write failed was acknowledged and lost. The release flush logged the failure, then ackDeferredWrites emptied the persisted queue after a successful sync. - A sync can now acknowledge only writes that reached a context manager: every landing records its deferredWriteId, and settleUnackedDeferredWrites clears the rest. - A failed write delivered durable (DeferredWrite.durable, carried through re-deferral and persistence) goes back to pending under its own id and seq, so the next boundary retries it. - Other writes keep the established poison policy (logged where they fail, then dropped), which the resume test pins. - This covers every flush site at the one place they all meet. 3. Module delegation lost puppet authorship: a puppeted courier--proxy whose handler called ctx.callTool reached the second tool as origin 'host'. - src/call-provenance.ts adds an AsyncLocalStorage that ModuleRegistry.handleToolCall sets around every module handler: the call's origin and admission, empty for the model. - ctx.callTool inherits from it, including from a callback the handler scheduled after returning. - Only work no tool call started (a module's own timer or event) is 'host'. 4. A reused placement out-param kept an earlier delivery's agent, ids and durability. It is now cleared at the start of each call and filled for that call alone. New tests: - safe-boundary-lease: four cases. - A creation paused in initialize blocks the grant, and so does its candidate until cleanup. - An outsider's creation is held without blocking the grant, and proceeds after the lease. - A draining stream's creation is admitted, and the lease waits for its candidate. - A durable deferral whose write fails at release stays pending, then lands once at the next boundary. - module-targeted-delivery: a reused receipt describes only the latest call. - tool-call-origin, two cases: - puppet authorship survives immediate and delayed delegation; - the model's authorship survives delegation, while a module's own callback is 'host'. The three files pass 27/27. Mutation checks, each caught: - a grant that ignores pending creations fails two cases; - no admission step fails the held-creation case; - acking failed durable writes fails the failure case; - ignoring the ambient actor fails both delegation cases; - not resetting the placement fails the reuse case. Co-Authored-By: Agnes (Claude Opus 5.5) --- src/call-provenance.ts | 19 ++++ src/framework.ts | 147 ++++++++++++++++++++++++-- src/module-registry.ts | 11 +- test/module-targeted-delivery.test.ts | 15 +++ test/safe-boundary-lease.test.ts | 110 ++++++++++++++++++- test/tool-call-origin.test.ts | 43 +++++++- 6 files changed, 333 insertions(+), 12 deletions(-) create mode 100644 src/call-provenance.ts diff --git a/src/call-provenance.ts b/src/call-provenance.ts new file mode 100644 index 000000000..7d208e821 --- /dev/null +++ b/src/call-provenance.ts @@ -0,0 +1,19 @@ +/** + * Who initiated the module tool call whose handler is running, carried + * across that handler's asynchronous work. + * + * ModuleRegistry.handleToolCall runs every module handler inside + * `callProvenance.run(...)`, and ModuleContext.callTool reads it. A module + * that delegates to another tool therefore passes on the actor who started + * the work: the agent's model (no origin), an operator's puppet, or the + * host. That includes delegation from a callback the handler scheduled + * after returning, because AsyncLocalStorage follows the work it started. + * Work that no tool call started has no store: a module's own timer, or its + * event handler, acts for the host. + */ +import { AsyncLocalStorage } from 'node:async_hooks'; +import type { ToolCall } from './types/index.js'; + +export type CallProvenance = Pick; + +export const callProvenance = new AsyncLocalStorage(); diff --git a/src/framework.ts b/src/framework.ts index 6cf557bea..61cc5b920 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -9,6 +9,7 @@ import { MembraneError } from '@animalabs/membrane'; import { ContextManager, PassthroughStrategy, WindowedPassthroughStrategy, OverBudgetError, UncoveredDropError } from '@animalabs/context-manager'; import type { CacheWireReceipt } from './kv-unified-wire.js'; import { SUBCONSCIOUS_TOOLS, SUBCONSCIOUS_TOOL_NAMES, type SubconsciousConfig } from './tune-out/tools.js'; +import { callProvenance } from './call-provenance.js'; import { TuneOutCoordinator, TUNE_OUT_DEFAULTS } from './tune-out/coordinator.js'; import type { MessageId, @@ -482,6 +483,9 @@ interface DeferredWrite { content: ContentBlock[]; metadata?: MessageMetadata; forAgent?: string; + /** Delivered durable (ModuleMessageOptions.durable): if a flush fails to + * store it, ackDeferredWrites returns it to pending instead of dropping it. */ + durable?: boolean; } const bySeq = (a: { seq: number }, b: { seq: number }): number => a.seq - b.seq; @@ -1247,6 +1251,9 @@ export class AgentFramework { * alone would let a hard exit forget an accepted message. */ private unackedDeferredWrites: DeferredWrite[] = []; + /** Ids of handed-off deferred writes that actually reached a context + * manager since the last ack. ackDeferredWrites acknowledges only these. */ + private landedDeferredWrites: Set = new Set(); /** Monotonic order stamp for deferred writes: the durable queue is always * written, restored, drained and flushed in `seq` order, whatever the * pending/un-acked split — a re-deferred entry keeps its place. */ @@ -1452,6 +1459,17 @@ export class AgentFramework { }> = []; /** The lease currently held, and the turn token it reserved per agent. */ private heldLease: { lease: SafeBoundaryLease; tokens: Map } | null = null; + /** + * Ephemeral creations admitted into the store and not yet running or + * released: from createEphemeralAgent's admission until + * runEphemeralToCompletion takes the candidate or its cleanup() runs. A + * safe-boundary lease waits for every one, so nothing initializes or + * starts under it. Requests still held behind a lease are not here. + */ + private ephemeralPending: Set = new Set(); + private ephemeralPendingByAgent: WeakMap = new WeakMap(); + /** Creations held until no lease is waiting or held (admitEphemeralCreation). */ + private boundaryClearedWaiters: Array<() => void> = []; /** Serialize per-server drains so reconnect and an online undo cannot race. */ private discordAwarenessDrains: Map> = new Map(); /** Framework-global inference gate; older generations cannot release it. */ @@ -1519,7 +1537,19 @@ export class AgentFramework { // framework hands to its own module registry. The public // executeToolCall() is shared with model/ephemeral callers and always // stamps agent origin (see there). - callTool: (call) => this.executeToolCallFrom({ ...call, origin: call.origin ?? 'host' }, { kind: 'module' }), + // Who asked travels with a delegated call: inside a module tool + // handler (or work it started) the initiator of that call; with no + // call behind it (a module's own timer or event) the host. + callTool: (call) => { + const ambient = callProvenance.getStore(); + const origin = call.origin ?? (ambient ? ambient.origin : 'host'); + const admission = call.admission ?? ambient?.admission; + const { origin: _o, admission: _a, ...rest } = call; + return this.executeToolCallFrom( + { ...rest, ...(origin ? { origin } : {}), ...(admission ? { admission } : {}) }, + { kind: 'module' }, + ); + }, notifyOps: (kind, agentName, message, data) => this.notifyOps(kind, agentName, message, data), }); } @@ -3452,7 +3482,7 @@ export class AgentFramework { if (this.unackedDeferredWrites.length === 0) return; // Nothing durable to reconcile against unless the queue was persisted. if (!this.deferredWritesPersisted && !this.quiesced) { - this.unackedDeferredWrites = []; + this.settleUnackedDeferredWrites(); return; } try { @@ -3465,11 +3495,35 @@ export class AgentFramework { ); return; } - this.unackedDeferredWrites = []; + this.settleUnackedDeferredWrites(); this.deferredScanFrom.clear(); this.persistDeferredWrites(); } + /** + * Clear the un-acked set once its landed writes are durable. A sync can + * only acknowledge a write that reached a context manager. An entry whose + * store write failed never landed: if it was delivered durable it goes + * back to pending under its own id and seq, so the next boundary retries + * it. Any other keeps the established poison policy (logged where it + * failed, then dropped) so one bad write can't wedge the queue behind it. + */ + private settleUnackedDeferredWrites(): void { + const failedDurable = this.unackedDeferredWrites.filter( + (m) => m.durable && !this.landedDeferredWrites.has(m.id), + ); + this.unackedDeferredWrites = []; + this.landedDeferredWrites.clear(); + if (failedDurable.length > 0) { + this.deferredMessages.push(...failedDurable); + this.deferredMessages.sort(bySeq); + console.error( + `[host-mode] ${failedDurable.length} durable deferred write(s) failed to store; ` + + `kept pending for the next boundary`, + ); + } + } + /** * Move drained entries into the un-acked set and record, per target * agent, the store position they will be appended after — then persist @@ -4207,12 +4261,23 @@ export class AgentFramework { * * Data persists after cleanup for investigation and cross-revert. * Call cleanup() when done to release the ContextManager. + * + * Safe boundaries (runAtSafeBoundary): an admitted creation, and the + * candidate it returns, count as pending until runEphemeralToCompletion + * takes the candidate or cleanup() runs, and a lease waits for every + * pending one. So call cleanup() for a candidate that will never run. + * While a lease waits, a creation proceeds only when `opts.requestedBy` + * names a stream the lease is already draining (an agent with its turn + * alive, or a running ephemeral stream), which may need the new agent to + * finish. Any other creation, and any creation while a lease is held, + * waits until the boundary clears. */ - async createEphemeralAgent(config: AgentConfig): Promise<{ + async createEphemeralAgent(config: AgentConfig, opts?: { requestedBy?: string }): Promise<{ agent: Agent; contextManager: ContextManager; cleanup: () => void; }> { + await this.admitEphemeralCreation(opts?.requestedBy); // Names are Chronicle namespaces and generation identities. Reserve before // opening the namespace so a second creation cannot append to an earlier // generation before runEphemeralToCompletion has a chance to reject it. @@ -4220,6 +4285,10 @@ export class AgentFramework { throw new Error(`Ephemeral agent name \"${config.name}\" is already registered or has been used in this framework`); } this.usedEphemeralAgentNames.add(config.name); + // Synchronously after admission: from here until the candidate runs or + // is cleaned up, a waiting lease isn't granted. + const pending = {}; + this.ephemeralPending.add(pending); try { const namespace = `subagent/${config.name}`; @@ -4236,15 +4305,21 @@ export class AgentFramework { this.restoreToolResultGuardSetting(agent); agent.markContextConsumed(); // RFC-006: nothing stored before now is replaceable this.ephemeralCandidates.set(agent, contextManager); + this.ephemeralPendingByAgent.set(agent, pending); const cleanup = () => { // Don't close the store — it's shared. Just release the CM. // Data persists in the store under the namespace for investigation. + // A candidate that never ran stops holding back safe boundaries. + this.ephemeralPending.delete(pending); + this.tryGrantSafeBoundary(); }; return { agent, contextManager, cleanup }; } catch (error) { this.usedEphemeralAgentNames.delete(config.name); + this.ephemeralPending.delete(pending); + this.tryGrantSafeBoundary(); throw error; } } @@ -4356,7 +4431,11 @@ export class AgentFramework { if (this.agents.has(agent.name) || this.ephemeralRuns.has(agent.name)) { throw new Error(`Ephemeral agent "${agent.name}" is already registered or running`); } - // Register temporarily so the event loop can drive it + // Register temporarily so the event loop can drive it. In the same + // synchronous step the candidate stops being pending and becomes a run, + // which a safe-boundary lease waits for in its place. + const pendingCreation = this.ephemeralPendingByAgent.get(agent); + if (pendingCreation) this.ephemeralPending.delete(pendingCreation); this.agents.set(agent.name, agent); const run: EphemeralRun = { settle: this.createDeferred(), @@ -6240,6 +6319,7 @@ export class AgentFramework { if (at < 0) return; // already granted this.boundaryWaiters.splice(at, 1); reject(withdrawn()); + this.notifyBoundaryCleared(); }; opts.signal.addEventListener('abort', onAbort, { once: true }); } @@ -6268,7 +6348,8 @@ export class AgentFramework { // Optional chaining: prototype-built harnesses leave fields undefined, // which must read as "nobody waiting" (see surgeryHeld below). if (!this.boundaryWaiters?.length || this.heldLease || this.surgeryHold) return; - if (this.ephemeralRuns.size > 0 || this.storeBusyReasons().length > 0) return; + if (this.ephemeralRuns.size > 0 || (this.ephemeralPending?.size ?? 0) > 0) return; + if (this.storeBusyReasons().length > 0) return; const waiter = this.boundaryWaiters.shift()!; const { tokens, release } = this.takeStoreReservation(waiter.verb, 'safe-boundary'); const lease: SafeBoundaryLease = Object.freeze({ @@ -6284,12 +6365,46 @@ export class AgentFramework { release: () => { if (this.heldLease === entry) this.heldLease = null; release(); - // The next waiter may be grantable at once. + // The next waiter may be grantable at once; if none is waiting, + // creations held behind the boundary proceed. this.tryGrantSafeBoundary(); + this.notifyBoundaryCleared(); }, }); } + /** + * Admission for createEphemeralAgent. With no lease waiting or held, a + * creation proceeds. While a lease waits, it proceeds only for a stream + * whose work the lease is already draining (isDrainingStream): such a + * stream may need the new agent to finish, as a sync spawn does. Any other + * creation, and every creation while a lease is held, waits until no lease + * is waiting or held. A held request doesn't count as pending, so it never + * blocks the grant it is waiting for. + */ + private async admitEphemeralCreation(requestedBy?: string): Promise { + for (;;) { + if (!this.boundaryWaiters?.length && !this.heldLease) return; + if (!this.heldLease && requestedBy !== undefined && this.isDrainingStream(requestedBy)) return; + await new Promise((resolve) => this.boundaryClearedWaiters.push(resolve)); + } + } + + /** A stream a waiting lease is draining: a registered ephemeral run, or an + * agent with a real turn alive (not a lease's reservation token). */ + private isDrainingStream(name: string): boolean { + if (this.ephemeralRuns.has(name)) return true; + const token = this.activeTurnTokens.get(name); + return token !== undefined && this.heldLease?.tokens.get(name) !== token; + } + + /** Release creations held by admitEphemeralCreation once no lease is + * waiting or held. */ + private notifyBoundaryCleared(): void { + if (this.boundaryWaiters?.length || this.heldLease) return; + for (const resume of this.boundaryClearedWaiters.splice(0)) resume(); + } + /** The tokens `lease` holds, or a refusal naming why it can't be used. */ private heldLeaseTokens(lease: SafeBoundaryLease, verb: string): Map { if (!this.heldLease || this.heldLease.lease !== lease) { @@ -7428,6 +7543,7 @@ export class AgentFramework { if (deferred.length > 0) { for (const msg of deferred) { agent.getContextManager().addMessage(msg.participant, msg.content, withDeferredWriteId(msg.metadata, msg.id)); + this.landedDeferredWrites.add(msg.id); // Injection guards: tool blocks would corrupt the tool-cycle // structure the membrane enforces, and a message named as the // agent itself would render as an ASSISTANT turn on the wire @@ -9811,6 +9927,7 @@ export class AgentFramework { const id = agent.getContextManager().addMessage( msg.participant, msg.content, withDeferredWriteId(msg.metadata, msg.id), ); + this.landedDeferredWrites.add(msg.id); this.emitTrace({ type: 'message:added', messageId: id, source: 'deferred-flush:turn-start' }); } catch (err) { console.error( @@ -13185,6 +13302,14 @@ export class AgentFramework { bypassDeferralFor?: string; } ): MessageId { + // The placement receipt describes this call only: a reused out-param + // must not keep an earlier delivery's agent, ids or durability. + if (opts?.placement) { + delete opts.placement.agent; + delete opts.placement.messageId; + delete opts.placement.deferredId; + delete opts.placement.durable; + } // Route to the named agent, else the primary (not ephemeral subagents). const agent = opts?.forAgent ? this.agents.get(opts.forAgent) @@ -13242,12 +13367,17 @@ export class AgentFramework { // set: one logical message, one durable identity — never two entries. const id = opts?.deferredWriteId ?? randomUUID(); let seq: number | undefined; + let durable = opts?.durable === true; if (opts?.deferredWriteId) { const prior = this.unackedDeferredWrites.find((m) => m.id === id); seq = prior?.seq; + durable ||= prior?.durable === true; this.unackedDeferredWrites = this.unackedDeferredWrites.filter((m) => m.id !== id); } - this.deferredMessages.push({ id, seq: seq ?? ++this.deferredSeq, participant, content, metadata, forAgent: opts?.forAgent }); + this.deferredMessages.push({ + id, seq: seq ?? ++this.deferredSeq, participant, content, metadata, forAgent: opts?.forAgent, + ...(durable ? { durable: true } : {}), + }); const persisted = this.quiesced || this.deferredWritesPersisted || opts?.durable ? this.persistDeferredWrites({ force: opts?.durable === true }) : false; @@ -13264,6 +13394,7 @@ export class AgentFramework { content, opts?.deferredWriteId ? withDeferredWriteId(metadata, opts.deferredWriteId) : metadata, ); + if (opts?.deferredWriteId) this.landedDeferredWrites.add(opts.deferredWriteId); if (opts?.placement) { opts.placement.agent = agent.name; opts.placement.messageId = stored; opts.placement.durable = true; } return stored; } diff --git a/src/module-registry.ts b/src/module-registry.ts index 81dcf0428..f02733ab7 100644 --- a/src/module-registry.ts +++ b/src/module-registry.ts @@ -25,6 +25,7 @@ import type { TraceEventListener, } from './types/index.js'; import type { Agent } from './agent.js'; +import { callProvenance } from './call-provenance.js'; const MODULE_STATE_PREFIX = 'modules/'; @@ -310,7 +311,15 @@ export class ModuleRegistry { }; try { - return await module.handleToolCall(moduleCall); + // The handler, and any work it starts, runs as this call's initiator, + // so a tool it delegates to through ctx.callTool keeps that actor. + return await callProvenance.run( + { + ...(call.origin ? { origin: call.origin } : {}), + ...(call.admission ? { admission: call.admission } : {}), + }, + () => module.handleToolCall(moduleCall), + ); } catch (error) { return { success: false, diff --git a/test/module-targeted-delivery.test.ts b/test/module-targeted-delivery.test.ts index f61481330..47a0782e1 100644 --- a/test/module-targeted-delivery.test.ts +++ b/test/module-targeted-delivery.test.ts @@ -143,4 +143,19 @@ describe('ModuleContext.addMessage delivery options', () => { assert.equal(id, ''); assert.deepEqual(placement, {}); }); + + it('describes only the latest call in a reused placement receipt', async () => { + const placement: MessagePlacement = {}; + courier.ctx.addMessage('user', [{ type: 'text', text: 'one' }], undefined, { forAgent: 'scout', placement }); + assert.equal(typeof placement.messageId, 'string'); + courier.ctx.addMessage('user', [{ type: 'text', text: 'lost' }], undefined, { forAgent: 'ghost', placement, durable: true }); + assert.deepEqual(placement, {}, 'nothing left from the earlier delivery'); + + courier.ctx.addMessage('user', [{ type: 'text', text: 'two' }], undefined, { forAgent: 'other', placement }); + (framework as unknown as Internals).activeTurnTokens.set('other', 77); + courier.ctx.addMessage('user', [{ type: 'text', text: 'three' }], undefined, { forAgent: 'other', placement }); + assert.equal(placement.messageId, undefined, 'no stale message id beside the deferral'); + assert.equal(typeof placement.deferredId, 'string'); + assert.equal(placement.durable, false); + }); }); diff --git a/test/safe-boundary-lease.test.ts b/test/safe-boundary-lease.test.ts index e77641203..a2c7e3e7f 100644 --- a/test/safe-boundary-lease.test.ts +++ b/test/safe-boundary-lease.test.ts @@ -13,8 +13,8 @@ import assert from 'node:assert/strict'; import { tmpdir } from 'node:os'; import { mkdtempSync, rmSync } from 'node:fs'; import { join } from 'node:path'; -import { AgentFramework } from '../src/index.js'; -import type { InferenceRequest, SafeBoundaryLease } from '../src/index.js'; +import { AgentFramework, PassthroughStrategy } from '../src/index.js'; +import type { InferenceRequest, MessagePlacement, Module, ModuleContext, SafeBoundaryLease, ToolDefinition, ToolResult } from '../src/index.js'; import type { ContentBlock } from '@animalabs/membrane'; import { MockMembrane, createMockResponse } from './helpers/mock-membrane.js'; @@ -30,6 +30,7 @@ type Internals = { ephemeralRuns: Map; deferredMessages: unknown[]; unackedDeferredWrites: unknown[]; + ephemeralPending: Set; addMessage(participant: string, content: ContentBlock[], metadata?: Record, opts?: { forAgent?: string }): string; }; @@ -248,4 +249,109 @@ describe('runAtSafeBoundary', () => { assert.ok(landed, 'landed at release'); assert.equal(typeof landed!.metadata?.deferredWriteId, 'string', 'carries its deferred-write id'); }); + + it('waits for an ephemeral creation in progress, then for its unrun candidate until cleanup', async () => { + let releaseInit!: () => void; + let reachedInit!: () => void; + const initGate = new Promise((r) => { releaseInit = r; }); + const initStarted = new Promise((r) => { reachedInit = r; }); + const strategy = new PassthroughStrategy(); + (strategy as unknown as { initialize: () => Promise }).initialize = async () => { reachedInit(); await initGate; }; + const creating = framework.createEphemeralAgent({ name: 'pending', model: 'test-model', systemPrompt: 'test', strategy }); + await initStarted; + + let granted = false; + const leased = framework.runAtSafeBoundary({ verb: 'after creation' }, async () => { granted = true; }); + await tick(); + assert.equal(granted, false, 'not granted while the creation initializes'); + releaseInit(); + const created = await creating; + i.tryGrantSafeBoundary(); + await tick(); + assert.equal(granted, false, 'nor while its candidate is unrun'); + created.cleanup(); + await leased; + assert.equal(granted, true, 'granted once the candidate is released'); + assert.equal(i.ephemeralPending.size, 0); + }); + + it('holds a creation nobody it is draining asked for, without letting it block the grant', async () => { + i.activeTurnTokens.set('other', 9_999); // a real turn keeps the lease waiting + const order: string[] = []; + const leased = framework.runAtSafeBoundary({ verb: 'held creation' }, async () => { order.push('lease'); }); + await tick(); + const creating = framework.createEphemeralAgent({ name: 'outsider', model: 'test-model', systemPrompt: 'test' }) + .then((c) => { order.push('created'); return c; }); + await tick(); + assert.deepEqual(order, [], 'held behind the waiting lease'); + assert.equal(i.ephemeralPending.size, 0, 'a held request is not pending'); + + i.activeTurnTokens.delete('other'); + i.tryGrantSafeBoundary(); + await leased; + const created = await creating; + assert.deepEqual(order, ['lease', 'created'], 'it proceeds only after the lease'); + created.cleanup(); + }); + + it("admits a creation for a stream it is draining, and waits for that stream's candidate", async () => { + i.activeTurnTokens.set('other', 9_999); // other's real turn is alive: the lease drains it + let granted = false; + const leased = framework.runAtSafeBoundary({ verb: 'draining' }, async () => { granted = true; }); + await tick(); + const created = await framework.createEphemeralAgent( + { name: 'helper', model: 'test-model', systemPrompt: 'test' }, { requestedBy: 'other' }, + ); + assert.equal(i.ephemeralPending.size, 1, 'admitted while the lease waits'); + i.activeTurnTokens.delete('other'); // other's turn ends + i.tryGrantSafeBoundary(); + await tick(); + assert.equal(granted, false, "the lease still waits for other's candidate"); + created.cleanup(); + await leased; + assert.equal(granted, true); + }); + + it('keeps a durable deferral whose store write failed pending for the next boundary', async () => { + class Courier implements Module { + readonly name = 'courier'; + ctx!: ModuleContext; + async start(ctx: ModuleContext): Promise { this.ctx = ctx; } + async stop(): Promise {} + getTools(): ToolDefinition[] { return []; } + async handleToolCall(): Promise { return { success: true }; } + async onProcess(): Promise> { return {}; } + } + const courier = new Courier(); + await framework.addModule(courier as unknown as Module); + const cm = framework.getAgent('scout')!.getContextManager(); + const realAdd = cm.addMessage.bind(cm); + let failing = true; + (cm as unknown as { addMessage: typeof cm.addMessage }).addMessage = ((participant, content, metadata, causedBy) => { + if (failing && (content[0] as { text?: string }).text === 'durable notice') throw new Error('injected append failure'); + return realAdd(participant, content, metadata, causedBy); + }) as typeof cm.addMessage; + + const placement: MessagePlacement = {}; + const quietErr = console.error; + console.error = () => {}; + try { + await framework.runAtSafeBoundary({ verb: 'deliver' }, async () => { + courier.ctx.addMessage('user', [{ type: 'text', text: 'durable notice' }], undefined, { forAgent: 'scout', placement, durable: true }); + }); + } finally { + console.error = quietErr; + } + assert.equal(placement.durable, true); + const landed = () => (cm.getAllMessages() as Array<{ content: Array<{ text?: string }> }>) + .filter((m) => m.content[0]?.text === 'durable notice').length; + assert.equal(landed(), 0, 'the write failed'); + assert.equal(i.deferredMessages.length, 1, 'kept pending, not acknowledged away'); + + failing = false; + await framework.runAtSafeBoundary({ verb: 'next boundary' }, async () => {}); + assert.equal(landed(), 1, 'lands at the next boundary, once'); + assert.equal(i.deferredMessages.length, 0); + assert.equal(i.unackedDeferredWrites.length, 0); + }); }); diff --git a/test/tool-call-origin.test.ts b/test/tool-call-origin.test.ts index 27d77b26a..5e6016606 100644 --- a/test/tool-call-origin.test.ts +++ b/test/tool-call-origin.test.ts @@ -24,12 +24,29 @@ class ProbeModule implements Module { async start(ctx: ModuleContext): Promise { this.ctx = ctx; } async stop(): Promise {} getTools(): ToolDefinition[] { - return [{ name: 'record', description: 'Record the call.', inputSchema: { type: 'object', properties: {} } }]; + return ['record', 'proxy', 'proxy_later'].map((name) => ({ name, description: name, inputSchema: { type: 'object', properties: {} } })); + } + /** Delegated work finished after the handler returned. */ + later?: Promise; + /** A callback the module schedules on its own, outside any tool call. */ + independently(callerAgentName: string): Promise { + return new Promise((resolve) => setImmediate(() => resolve( + this.ctx.callTool!({ id: 'independent', name: 'probe--record', input: {}, callerAgentName }), + ))); } getUtilities(): ToolDefinition[] { return [{ name: 'util_record', description: 'Record the call, as a utility.', inputSchema: { type: 'object', properties: {} } }]; } async handleToolCall(call: ToolCall): Promise { + if (call.name === 'proxy') { + return this.ctx.callTool!({ id: 'nested', name: 'probe--record', input: {}, callerAgentName: call.callerAgentName }); + } + if (call.name === 'proxy_later') { + this.later = new Promise((resolve) => setTimeout(() => resolve( + this.ctx.callTool!({ id: 'nested-later', name: 'probe--record', input: {}, callerAgentName: call.callerAgentName }), + ), 5)); + return { success: true, data: 'scheduled' }; + } this.calls.push(call); return { success: true, data: { recorded: call.name } }; } @@ -131,4 +148,28 @@ describe('tool-call origin', () => { assert.equal(hostRun.success, true, String(hostRun.error)); assert.deepEqual(provenance(probe.calls[1]!), { origin: 'host', admission: undefined }); }); + + it("keeps the puppet's authorship when a module delegates to another tool, now or later", async () => { + await framework.puppetToolCall('scout', 'probe--proxy', {}); + await framework.puppetToolCall('scout', 'probe--proxy_later', {}); + await probe.later; + assert.deepEqual(probe.calls.map((c) => ({ id: c.id, origin: c.origin })), [ + { id: 'nested', origin: 'puppet' }, + { id: 'nested-later', origin: 'puppet' }, + ]); + }); + + it("keeps the model's authorship through delegation, and marks a module's own callback as the host's", async () => { + membrane.pushResponse(createMockResponse([{ type: 'tool_use', id: 'toolu_model_2', name: 'probe--proxy', input: {} }], 'tool_use')); + membrane.pushResponse(createMockResponse([{ type: 'text', text: 'done' }])); + (framework as unknown as Internals).pendingRequests.push( + { agentName: 'scout', reason: 'mcpl:channel-incoming', source: 'test', timestamp: Date.now() } as InferenceRequest, + ); + await framework.runUntilIdle(); + await probe.independently('scout'); + assert.deepEqual(probe.calls.map((c) => ({ id: c.id, origin: c.origin })), [ + { id: 'nested', origin: undefined }, + { id: 'independent', origin: 'host' }, + ]); + }); }); From e7a16907a3f40460ff5a251ac0d0f208e2649b4d Mon Sep 17 00:00:00 2001 From: Agnes-1009 Date: Wed, 7 Oct 2026 08:35:34 -0400 Subject: [PATCH 37/73] fix(framework): atomic creation admission under any store hold; bounded landed-write ids (review) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Kit's recheck of 98f5a61 (room-222 #44202, /tmp/kit-1008-af-body-fixprobe.mjs) found two remaining gaps. 1. Admission had an await between the check and the reservation. createEphemeralAgent awaited admitEphemeralCreation and only then added its pending ticket. An async function yields at that await even when the admission resolves at once, so `createEphemeralAgent(…)` followed by `runAtSafeBoundary(…)` in the same tick let the lease be granted first, and the creation then initialized under it. - tryAdmitEphemeralCreation now checks and takes the ticket in one synchronous step, and admitEphemeralCreation returns that ticket, waiting only when the check fails and re-checking on each wake. So a same-tick lease finds the ticket already pending. - Admission also names the store hold itself (surgeryHold) rather than only a held lease, so a direct rollbackToMessage / suppressMessages holds creations as a lease does. - The reservation's release wakes held creations, by a microtask that re-checks. A name collision after admission releases the ticket before throwing. 2. Landed-write ids leaked for memory-only deferrals. Every flush recorded its id, but ackDeferredWrites returns early when nothing is un-acked, so ordinary traffic grew the set forever. markDeferredWriteLanded now records an id only while that entry is in the un-acked set, the only place settleUnackedDeferredWrites needs it. Tests (safe-boundary-lease, 3 new, 16 in the file): - a creation started in the same tick as a lease request holds the lease until it has initialized and its candidate is released; - a direct surgery's store hold holds a creation, which proceeds at release; - four memory-only deferrals land and leave no ids behind. Lease, delivery, origin, host-quiesce, operator-surgery, surgery-receipt and ephemeral-settle suites: 94/94. Mutation checks, each caught by its case: taking the ticket after a yield, admission ignoring a surgery hold, and recording every landed id. Co-Authored-By: Agnes (Claude Opus 5.5) --- src/framework.ts | 56 +++++++++++++++++++++++--------- test/safe-boundary-lease.test.ts | 43 ++++++++++++++++++++++++ 2 files changed, 84 insertions(+), 15 deletions(-) diff --git a/src/framework.ts b/src/framework.ts index 61cc5b920..4c66d086f 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -3508,6 +3508,12 @@ export class AgentFramework { * it. Any other keeps the established poison policy (logged where it * failed, then dropped) so one bad write can't wedge the queue behind it. */ + /** Record a landed write for the next ack, only while it is un-acked: + * memory-only deferrals never enter that set, so they leave no id behind. */ + private markDeferredWriteLanded(id: string): void { + if (this.unackedDeferredWrites.some((m) => m.id === id)) this.landedDeferredWrites.add(id); + } + private settleUnackedDeferredWrites(): void { const failedDurable = this.unackedDeferredWrites.filter( (m) => m.durable && !this.landedDeferredWrites.has(m.id), @@ -4277,18 +4283,18 @@ export class AgentFramework { contextManager: ContextManager; cleanup: () => void; }> { - await this.admitEphemeralCreation(opts?.requestedBy); + // From admission until the candidate runs or is cleaned up, the ticket + // is pending and a waiting lease isn't granted. + const pending = await this.admitEphemeralCreation(opts?.requestedBy); // Names are Chronicle namespaces and generation identities. Reserve before // opening the namespace so a second creation cannot append to an earlier // generation before runEphemeralToCompletion has a chance to reject it. if (this.usedEphemeralAgentNames.has(config.name) || this.agents.has(config.name)) { + this.ephemeralPending.delete(pending); + this.tryGrantSafeBoundary(); throw new Error(`Ephemeral agent name \"${config.name}\" is already registered or has been used in this framework`); } this.usedEphemeralAgentNames.add(config.name); - // Synchronously after admission: from here until the candidate runs or - // is cleaned up, a waiting lease isn't granted. - const pending = {}; - this.ephemeralPending.add(pending); try { const namespace = `subagent/${config.name}`; @@ -6243,6 +6249,8 @@ export class AgentFramework { if (released) return; released = true; if (this.surgeryHold === hold) this.surgeryHold = null; + // Creations held behind this hold may proceed (unless a lease waits). + queueMicrotask(() => this.notifyBoundaryCleared()); for (const [name, token] of reserved) { if (this.activeTurnTokens.get(name) === token) this.activeTurnTokens.delete(name); } @@ -6382,12 +6390,30 @@ export class AgentFramework { * is waiting or held. A held request doesn't count as pending, so it never * blocks the grant it is waiting for. */ - private async admitEphemeralCreation(requestedBy?: string): Promise { - for (;;) { - if (!this.boundaryWaiters?.length && !this.heldLease) return; - if (!this.heldLease && requestedBy !== undefined && this.isDrainingStream(requestedBy)) return; + private async admitEphemeralCreation(requestedBy?: string): Promise { + // The check and the pending ticket are taken in one synchronous step, + // with no await in between: a lease requested in the same tick as the + // creation must find the ticket already pending. + let ticket = this.tryAdmitEphemeralCreation(requestedBy); + while (!ticket) { await new Promise((resolve) => this.boundaryClearedWaiters.push(resolve)); + ticket = this.tryAdmitEphemeralCreation(requestedBy); } + return ticket; + } + + /** Admit a creation now, returning its pending ticket, or null when it has + * to wait. The store hold (surgeryHold) covers a held lease and a direct + * rollback or suppress alike: nothing initializes under either. */ + private tryAdmitEphemeralCreation(requestedBy?: string): object | null { + const admitted = this.surgeryHold + ? false + : !this.boundaryWaiters?.length + || (requestedBy !== undefined && this.isDrainingStream(requestedBy)); + if (!admitted) return null; + const ticket = {}; + this.ephemeralPending.add(ticket); + return ticket; } /** A stream a waiting lease is draining: a registered ephemeral run, or an @@ -6399,10 +6425,10 @@ export class AgentFramework { } /** Release creations held by admitEphemeralCreation once no lease is - * waiting or held. */ + * waiting and nothing holds the store. Each re-checks on waking. */ private notifyBoundaryCleared(): void { - if (this.boundaryWaiters?.length || this.heldLease) return; - for (const resume of this.boundaryClearedWaiters.splice(0)) resume(); + if (this.boundaryWaiters?.length || this.surgeryHold) return; + for (const resume of this.boundaryClearedWaiters?.splice(0) ?? []) resume(); } /** The tokens `lease` holds, or a refusal naming why it can't be used. */ @@ -7543,7 +7569,7 @@ export class AgentFramework { if (deferred.length > 0) { for (const msg of deferred) { agent.getContextManager().addMessage(msg.participant, msg.content, withDeferredWriteId(msg.metadata, msg.id)); - this.landedDeferredWrites.add(msg.id); + this.markDeferredWriteLanded(msg.id); // Injection guards: tool blocks would corrupt the tool-cycle // structure the membrane enforces, and a message named as the // agent itself would render as an ASSISTANT turn on the wire @@ -9927,7 +9953,7 @@ export class AgentFramework { const id = agent.getContextManager().addMessage( msg.participant, msg.content, withDeferredWriteId(msg.metadata, msg.id), ); - this.landedDeferredWrites.add(msg.id); + this.markDeferredWriteLanded(msg.id); this.emitTrace({ type: 'message:added', messageId: id, source: 'deferred-flush:turn-start' }); } catch (err) { console.error( @@ -13394,7 +13420,7 @@ export class AgentFramework { content, opts?.deferredWriteId ? withDeferredWriteId(metadata, opts.deferredWriteId) : metadata, ); - if (opts?.deferredWriteId) this.landedDeferredWrites.add(opts.deferredWriteId); + if (opts?.deferredWriteId) this.markDeferredWriteLanded(opts.deferredWriteId); if (opts?.placement) { opts.placement.agent = agent.name; opts.placement.messageId = stored; opts.placement.durable = true; } return stored; } diff --git a/test/safe-boundary-lease.test.ts b/test/safe-boundary-lease.test.ts index a2c7e3e7f..95165b542 100644 --- a/test/safe-boundary-lease.test.ts +++ b/test/safe-boundary-lease.test.ts @@ -31,6 +31,8 @@ type Internals = { deferredMessages: unknown[]; unackedDeferredWrites: unknown[]; ephemeralPending: Set; + landedDeferredWrites: Set; + reserveStoreForSurgery(verb: string, agentName: string): () => void; addMessage(participant: string, content: ContentBlock[], metadata?: Record, opts?: { forAgent?: string }): string; }; @@ -354,4 +356,45 @@ describe('runAtSafeBoundary', () => { assert.equal(i.deferredMessages.length, 0); assert.equal(i.unackedDeferredWrites.length, 0); }); + + it('takes the creation ticket before yielding, so a lease asked for in the same tick waits for it', async () => { + const order: string[] = []; + const strategy = new PassthroughStrategy(); + (strategy as unknown as { initialize: () => Promise }).initialize = async () => { + await tick(); + order.push('initialized'); + }; + const creating = framework.createEphemeralAgent({ name: 'same-tick', model: 'test-model', systemPrompt: 'test', strategy }); + const leased = framework.runAtSafeBoundary({ verb: 'same tick' }, async () => { order.push('lease'); }); + const created = await creating; + i.tryGrantSafeBoundary(); + await tick(); + assert.deepEqual(order, ['initialized'], 'the lease waited for the creation and its candidate'); + created.cleanup(); + await leased; + assert.deepEqual(order, ['initialized', 'lease']); + }); + + it("holds a creation behind a direct surgery's store hold, not only behind a lease", async () => { + const release = i.reserveStoreForSurgery('rollback', 'scout'); + let created = false; + const creating = framework.createEphemeralAgent({ name: 'behind-surgery', model: 'test-model', systemPrompt: 'test' }) + .then((c) => { created = true; return c; }); + await tick(); + assert.equal(created, false, 'held while the surgery holds the store'); + release(); + const c = await creating; + assert.equal(created, true, 'proceeds once the hold is released'); + c.cleanup(); + }); + + it('keeps no landed-write ids for memory-only deferrals', async () => { + i.activeTurnTokens.set('other', 9_999); + for (let n = 0; n < 4; n++) i.addMessage('user', [{ type: 'text', text: `ordinary ${n}` }], undefined, { forAgent: 'other' }); + assert.equal(i.deferredMessages.length, 4); + i.activeTurnTokens.delete('other'); + await framework.runAtSafeBoundary({ verb: 'flush' }, async () => {}); + assert.equal(i.deferredMessages.length, 0, 'all landed'); + assert.equal(i.landedDeferredWrites.size, 0, 'and left no ids behind'); + }); }); From a959680667b38db71dbe57e1ce78c878ea98d661 Mon Sep 17 00:00:00 2001 From: Agnes-1009 Date: Wed, 7 Oct 2026 08:57:41 -0400 Subject: [PATCH 38/73] fix(framework): a store reservation also waits for admitted ephemeral creations (review) Kit's recheck of 49d527e (room-222 #44436, /tmp/kit-1008-af-direct-surgery-probe.mjs) found the reverse ordering. Creation already waited for a store hold taken first. A direct rollbackToMessage / suppressMessages taken after a creation had been admitted still went ahead, because reserveStoreForSurgery's busy test (storeBusyReasons) didn't count pending creations; only the lease's grant did. The probe paused a strategy's initialization and rolled back; the rollback succeeded, and the creation then failed with "Branch changed during strategy initialization". storeBusyReasons now reports "N ephemeral creation(s) pending" while any admitted creation or unrun candidate exists. A direct surgery refuses with agent-busy, the lease waits through the same condition, and creation still waits for a hold taken first. Both orders are covered. Tests: - safe-boundary-lease: a new case. A surgery reservation refuses while a creation initializes, and still refuses while its candidate is unrun; after cleanup() it succeeds. The opposite order, a surgery holding a creation until release, was already covered. - operator-surgery: "holds the store against agents admitted AFTER the reservation" created an unrun candidate and then expected a rollback to proceed. It now first asserts the new refusal (agent-busy naming the pending creation). It then releases the candidate from pending with cleanup() and keeps its original checks: a candidate run under the hold is refused with its ticket kept, and wakes are parked. Lease, operator-surgery, surgery-receipt, ephemeral-settle, code-execution, puppet and origin suites: 103/103. Co-Authored-By: Agnes (Claude Opus 5.5) --- src/framework.ts | 5 +++++ test/operator-surgery.test.ts | 9 +++++++++ test/safe-boundary-lease.test.ts | 22 ++++++++++++++++++++++ 3 files changed, 36 insertions(+) diff --git a/src/framework.ts b/src/framework.ts index 4c66d086f..fb14255fc 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -6225,6 +6225,11 @@ export class AgentFramework { for (const name of this.activeTurnTokens.keys()) { if (!this.agents.has(name)) busy.push(`${name} is turn-alive`); } + // An admitted ephemeral creation is initializing against, or about to + // run on, the current branch: a store reservation must wait for it in + // either order (creation also waits for a reservation taken first). + const pending = this.ephemeralPending?.size ?? 0; + if (pending > 0) busy.push(`${pending} ephemeral creation(s) pending`); return busy; } diff --git a/test/operator-surgery.test.ts b/test/operator-surgery.test.ts index f6180a632..6024ed16e 100644 --- a/test/operator-surgery.test.ts +++ b/test/operator-surgery.test.ts @@ -201,6 +201,15 @@ describe('live operator surgery', () => { processInferenceRequests(): Promise; }; try { + // A pending (admitted, unrun) candidate refuses the surgery outright: + // it was created against this branch and may initialize or run on it. + await assert.rejects( + framework.rollbackToMessage('scout', { messageId: ids[1] }), + (e: Error & { code?: string }) => e.code === 'agent-busy' && /ephemeral creation/.test(e.message), + ); + // Released from pending, the candidate can still be run (wrongly) + // later: that admission is what the store hold must refuse. + worker.cleanup(); const rollback = framework.rollbackToMessage('scout', { messageId: ids[1] }); await new Promise((resolve) => setImmediate(resolve)); assert.ok(internals.surgeryHold, 'store hold is up while the switch is awaited'); diff --git a/test/safe-boundary-lease.test.ts b/test/safe-boundary-lease.test.ts index 95165b542..fb5424596 100644 --- a/test/safe-boundary-lease.test.ts +++ b/test/safe-boundary-lease.test.ts @@ -397,4 +397,26 @@ describe('runAtSafeBoundary', () => { assert.equal(i.deferredMessages.length, 0, 'all landed'); assert.equal(i.landedDeferredWrites.size, 0, 'and left no ids behind'); }); + + it('refuses a direct surgery while an admitted creation is pending, in the reverse order too', async () => { + let releaseInit!: () => void; + let reachedInit!: () => void; + const initGate = new Promise((r) => { releaseInit = r; }); + const initStarted = new Promise((r) => { reachedInit = r; }); + const strategy = new PassthroughStrategy(); + (strategy as unknown as { initialize: () => Promise }).initialize = async () => { reachedInit(); await initGate; }; + const creating = framework.createEphemeralAgent({ name: 'first', model: 'test-model', systemPrompt: 'test', strategy }); + await initStarted; + assert.throws( + () => i.reserveStoreForSurgery('rollback', 'scout'), + (e: Error & { code?: string }) => e.code === 'agent-busy' && /ephemeral creation\(s\) pending/.test(e.message), + 'the surgery refuses while the creation initializes', + ); + releaseInit(); + const created = await creating; + assert.throws(() => i.reserveStoreForSurgery('rollback', 'scout'), /ephemeral creation/, 'and while its candidate is unrun'); + created.cleanup(); + const release = i.reserveStoreForSurgery('rollback', 'scout'); // free now + release(); + }); }); From 204dd8899812f25c9983631809192ef6c2899c6c Mon Sep 17 00:00:00 2001 From: Agnes-1009 Date: Wed, 7 Oct 2026 09:21:08 -0400 Subject: [PATCH 39/73] fix(framework): cleanup() withdraws an unrun ephemeral candidate for good (review) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Kit's review of 4a3f7ca (room-222 #44590) asked whether anything legitimately runs a candidate after cleanup(). Since 98f5a61, cleanup() releases the candidate's pending admission, the thing that kept leases and surgeries off the context it was created against. A candidate run after that would resume an object nothing protected meanwhile. No supported caller does it: connectome-host's SubagentModule creates a fresh candidate per attempt and calls cleanup() in that attempt's finally, after its run. - cleanup() now also voids the candidate's generation ticket, if it is still unconsumed, and records the candidate as released. - runEphemeralToCompletion refuses a released candidate with an explicit reason ("was released by cleanup() and can't run; create a new one"), ahead of the generic ticket check. - createEphemeralAgent's doc says cleanup() withdraws the candidate, and that a surgery, like a lease, waits for every pending one. Tests: - operator-surgery "holds the store against agents admitted AFTER the reservation…" (retitled) no longer runs a cleaned-up candidate under the hold. It checks that a pending candidate refuses the rollback and that cleanup() withdraws it (the run is refused, the ticket gone). Then, during the held rollback, a fresh createEphemeralAgent request is held, not admitted, and proceeds once the hold is released. Wakes are still parked. - safe-boundary-lease, two new cases: a cleaned-up candidate can't run and has no ticket. And, as defence in depth, a run attempted under a store hold (reached through the internal takeStoreReservation, since no public path holds the store over a pending candidate any more) is refused before its ticket is consumed. Built and run alone, with the uncommitted R2 work stashed: lease, operator-surgery and ephemeral-settle suites, 45/45. Co-Authored-By: Agnes (Claude Opus 5.5) --- src/framework.ts | 18 +++++++++++++++--- test/operator-surgery.test.ts | 26 +++++++++++++++++--------- test/safe-boundary-lease.test.ts | 26 ++++++++++++++++++++++++++ 3 files changed, 58 insertions(+), 12 deletions(-) diff --git a/src/framework.ts b/src/framework.ts index fb14255fc..2dc25f8da 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -1468,6 +1468,9 @@ export class AgentFramework { */ private ephemeralPending: Set = new Set(); private ephemeralPendingByAgent: WeakMap = new WeakMap(); + /** Candidates withdrawn by their cleanup(): they can never run, since the + * pending admission that protected their pre-run context is gone. */ + private releasedEphemeralCandidates: WeakSet = new WeakSet(); /** Creations held until no lease is waiting or held (admitEphemeralCreation). */ private boundaryClearedWaiters: Array<() => void> = []; /** Serialize per-server drains so reconnect and an online undo cannot race. */ @@ -4270,8 +4273,9 @@ export class AgentFramework { * * Safe boundaries (runAtSafeBoundary): an admitted creation, and the * candidate it returns, count as pending until runEphemeralToCompletion - * takes the candidate or cleanup() runs, and a lease waits for every - * pending one. So call cleanup() for a candidate that will never run. + * takes the candidate or cleanup() runs, and a lease or surgery waits for + * every pending one. So call cleanup() for a candidate that will never + * run; cleanup() withdraws it, and it can't be run afterwards. * While a lease waits, a creation proceeds only when `opts.requestedBy` * names a stream the lease is already draining (an agent with its turn * alive, or a running ephemeral stream), which may need the new agent to @@ -4316,7 +4320,12 @@ export class AgentFramework { const cleanup = () => { // Don't close the store — it's shared. Just release the CM. // Data persists in the store under the namespace for investigation. - // A candidate that never ran stops holding back safe boundaries. + // A candidate that never ran is withdrawn for good: its generation + // ticket goes with its pending admission, so it can't run later on + // a context nothing protected meanwhile, and it stops holding back + // safe boundaries and surgeries. + if (this.ephemeralCandidates.get(agent) === contextManager) this.ephemeralCandidates.delete(agent); + this.releasedEphemeralCandidates.add(agent); this.ephemeralPending.delete(pending); this.tryGrantSafeBoundary(); }; @@ -4416,6 +4425,9 @@ export class AgentFramework { `refusing new ephemeral run for ${agent.name} — resume() first`, ); } + if (this.releasedEphemeralCandidates.has(agent)) { + throw new Error(`Ephemeral agent "${agent.name}" was released by cleanup() and can't run; create a new one`); + } // Only a fresh object returned by createEphemeralAgent may enter this path. // Never overwrite a resident/conversation owner or a concurrent run: their // cleanup is name-keyed and could cancel/deregister the legitimate owner. diff --git a/test/operator-surgery.test.ts b/test/operator-surgery.test.ts index 6024ed16e..44ab94700 100644 --- a/test/operator-surgery.test.ts +++ b/test/operator-surgery.test.ts @@ -181,7 +181,7 @@ describe('live operator surgery', () => { assert.equal(tokens.size, 0, 'and released after'); }); - it('holds the store against agents admitted AFTER the reservation: ephemeral admission is refused (ticket kept) and wakes are parked', async () => { + it('holds the store against agents admitted AFTER the reservation: a pending candidate refuses it, a creation requested under it is held, and wakes are parked', async () => { // Hold the awaited switch open so the surgery is mid-flight. const c = cm() as unknown as { switchBranch: (name: string) => Promise }; const originalSwitch = c.switchBranch; @@ -207,19 +207,24 @@ describe('live operator surgery', () => { framework.rollbackToMessage('scout', { messageId: ids[1] }), (e: Error & { code?: string }) => e.code === 'agent-busy' && /ephemeral creation/.test(e.message), ); - // Released from pending, the candidate can still be run (wrongly) - // later: that admission is what the store hold must refuse. + // cleanup() withdraws the candidate for good: it can't run afterwards. worker.cleanup(); + await assert.rejects( + framework.runEphemeralToCompletion(worker.agent, worker.contextManager), + /released by cleanup\(\)/, + ); + assert.equal(internals.ephemeralCandidates.has(worker.agent), false, 'its generation ticket is gone'); const rollback = framework.rollbackToMessage('scout', { messageId: ids[1] }); await new Promise((resolve) => setImmediate(resolve)); assert.ok(internals.surgeryHold, 'store hold is up while the switch is awaited'); - await assert.rejects( - framework.runEphemeralToCompletion(worker.agent, worker.contextManager), - /under live roll back/, - ); - assert.equal(internals.ephemeralCandidates.get(worker.agent), worker.contextManager, - 'refused BEFORE consuming the generation ticket — a clean retry stays possible'); + // A creation requested now is held, not admitted, until the surgery ends. + let lateCreated = false; + const late = framework.createEphemeralAgent({ + name: 'late-worker', model: 'test-model', systemPrompt: 'Do the task.', allowedTools: 'all', + }).then((c) => { lateCreated = true; return c; }); + await new Promise((resolve) => setImmediate(resolve)); + assert.equal(lateCreated, false, 'no creation is admitted under the hold'); // A wake arriving now is parked by the scheduler, not started. framework.nudgeAgent('scout', 'test'); @@ -232,6 +237,9 @@ describe('live operator surgery', () => { assert.equal(r.messagesRemoved, 3); assert.equal(internals.surgeryHold, null, 'hold released with the reservation'); assert.equal(membrane.calls.length, 0); + const lateWorker = await late; + assert.equal(lateCreated, true, 'the held creation proceeds once the hold is released'); + lateWorker.cleanup(); } finally { c.switchBranch = originalSwitch; worker.cleanup(); diff --git a/test/safe-boundary-lease.test.ts b/test/safe-boundary-lease.test.ts index fb5424596..08c1f2fb0 100644 --- a/test/safe-boundary-lease.test.ts +++ b/test/safe-boundary-lease.test.ts @@ -33,6 +33,8 @@ type Internals = { ephemeralPending: Set; landedDeferredWrites: Set; reserveStoreForSurgery(verb: string, agentName: string): () => void; + takeStoreReservation(verb: string, agentName: string): { release: () => void }; + ephemeralCandidates: Map; addMessage(participant: string, content: ContentBlock[], metadata?: Record, opts?: { forAgent?: string }): string; }; @@ -419,4 +421,28 @@ describe('runAtSafeBoundary', () => { const release = i.reserveStoreForSurgery('rollback', 'scout'); // free now release(); }); + + it('withdraws a cleaned-up candidate for good', async () => { + const created = await framework.createEphemeralAgent({ name: 'withdrawn', model: 'test-model', systemPrompt: 'test' }); + created.cleanup(); + await assert.rejects( + framework.runEphemeralToCompletion(created.agent, created.contextManager), + /released by cleanup\(\) and can't run/, + ); + assert.equal(i.ephemeralCandidates.has(created.agent), false); + }); + + it('still refuses a run under a store hold, keeping its ticket (defence in depth)', async () => { + // No public path holds the store while a candidate is pending; take the + // reservation directly to reach the run-time check behind that rule. + const created = await framework.createEphemeralAgent({ name: 'defended', model: 'test-model', systemPrompt: 'test' }); + const hold = i.takeStoreReservation('rollback', 'scout'); + try { + await assert.rejects(framework.runEphemeralToCompletion(created.agent, created.contextManager), /under live rollback/); + assert.equal(i.ephemeralCandidates.get(created.agent), created.contextManager, 'refused before consuming the ticket'); + } finally { + hold.release(); + created.cleanup(); + } + }); }); From 955a1b9322fad1e7c4f929bc8447f326e6caa4ae Mon Sep 17 00:00:00 2001 From: Agnes-1009 Date: Wed, 7 Oct 2026 09:31:25 -0400 Subject: [PATCH 40/73] feat(framework): operator-change gate and scoped admission for self-change tools (R2, part 1) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Part one of R2 from the operator-change admission design (room-222 #43694; Kit's answers and boundaries, #43896 and #43956). An operator's or a module's mutating agent_settings or tool-presentation call is resolved to an absolute change, and the host's gate decides it before anything runs. The agent's own model calls never reach the gate. With no gate configured, every path behaves as before. FrameworkConfig.operatorChangeGate(change) answers: - `apply`: the framework applies exactly this change now, after revalidating it. - `staged` with a receipt: the host applies it later through applyResolvedOperatorChange. - A gate that throws, or answers anything else, refuses the change (fails closed, Kit's answer 1). Resolution (src/operator-change.ts types; resolveSelfChange): - agent-settings: the input as given, `from` (the touched core settings when resolved, unset as null), and `target`, the concrete values the action leaves them at. - Agent.previewRuntimeSettingsTarget computes the target without changing anything: an update's patch values; a reset's configured values; a cancel's live budget. - Host-managed extension keys the input touches are recorded with their values when resolved. - Invalid input (a non-number budget, an unknown key) isn't resolved. The call runs ungated, and the tool refuses it itself, as before. - tool-presentation: the input, and the targeted tool's presentation entry (name, visible, description) when resolved. Revalidation (revalidateSelfChange; OperatorActionError code 'stale') runs on every application, immediate or delayed (Kit's boundary 2): - The from-state must still hold. For settings, the target recomputed now must also equal the approved one, so a reset applies the values approved, never what the configuration says later. - A change that no longer holds is refused, never re-resolved, and the host restages it. Admission is scoped (Kit's boundary 3), not a skip flag: - activeAdmissions maps an admission id to the change it was registered for, only while applyResolvedOperatorChange (or an applied puppet call) carries it out. - admittedChange lets a call through only for that agent, tool and exact input. An unregistered admission, or one used with any other input, goes to the gate. - A covered call is revalidated synchronously right before it runs, so no await separates the last check from the mutation. Paths: - puppetToolCall asks before anything runs. Staged: nothing executes, no pair is stored (a pair would testify the tool ran), and it returns toolUseId null with the receipt (Kit's answer 4; the return type is now `toolUseId: string | null; staged?`). Apply: revalidate, admit for the call's own duration, run. Gate failure: throws. puppetToolCall gains opts.requester for the descriptor; the old body is puppetToolCallUngated. - Wrapped calls: executeToolCallFrom (module callTool, ephemerals) and dispatchToolCall (a script's inner call can carry an origin) route agent_settings and presentation edits through gatedSelfChange. Origin 'puppet' and 'host' are gated alike (Kit, #43896: a host-initiated body mutation isn't a resident request). The model's own dispatch stays synchronous; only gated calls go async. - applyResolvedOperatorChange(change, { lease, admission }) requires the held lease and revalidates. An operator's change is carried out as the puppet call it was, on the lease's token, so the agent's context records it as before; a module's is applied by the handler directly. It logs operator-change-applied with the host's admission id; staging and refusals log operator-change-staged and operator-change-refused. Still to come in R2: host/command undo-turns (one absolute cut, operation-bound destination), unstick (host-driven steps with operation+step identity and a durable outcome), and hide/undo-messages on Ada's marks branch. Tests: test/operator-change-gate.test.ts, 15 cases: - staging (receipt, nothing run, no pair; descriptor from/target), apply asking once, fail-closed refusal; - reads and the model's own calls pass ungated; - puppet-through-module and module-own changes both gated; - invalid input goes to the tool's own refusal; - an admission covers only its exact call and is revalidated at use; a forged admission is ignored; - delayed application as the puppet act, and a module's directly; - stale settings; a reset's concrete target, and stale when it changes; - the held lease required for both surfaces; - presentation staged with its from-state, then applied under a lease. puppet-tool-call: one assertion tolerates the nullable toolUseId. Mutation checks, each caught: - wrapped calls never gated; a staged puppet still running; - revalidation a no-op; admission ignoring input; - a failed gate applying; application without the lease (caught once a module-surface change was added, since the puppet path checks its lease again downstream); - no revalidation at use; a reset target that isn't the configured value. Sixteen affected suites (gate, runtime settings, presentation, puppet ×2, origin, code-execution, lease, delivery, subconscious, quiesce, operator-surgery, surgery receipt, ephemeral-settle, scheduler, overbudget breaker): 197/197. Co-Authored-By: Agnes (Claude Opus 5.5) --- src/agent.ts | 60 +++++ src/framework.ts | 375 +++++++++++++++++++++++++++++- src/index.ts | 9 + src/operator-change.ts | 102 ++++++++ src/operator-log.ts | 2 +- src/types/framework.ts | 14 ++ test/operator-change-gate.test.ts | 298 ++++++++++++++++++++++++ test/puppet-tool-call.test.ts | 3 +- 8 files changed, 855 insertions(+), 8 deletions(-) create mode 100644 src/operator-change.ts create mode 100644 test/operator-change-gate.test.ts diff --git a/src/agent.ts b/src/agent.ts index 7454f2d8f..d528ee13b 100644 --- a/src/agent.ts +++ b/src/agent.ts @@ -504,6 +504,66 @@ export class Agent { return this.updateRuntimeSettings(overrides); } + /** + * The concrete value each setting a runtime-settings action touches would + * be reported at (getRuntimeSettings) once the action applied, computed + * without changing anything. An operator approves these values, and + * application checks they still hold, so a reset applies what was + * approved rather than whatever the configuration says later. + * + * - update: the patch's values (a decrease reports its target at once). + * - reset: the configured value each key returns to. A key with nothing + * configured keeps its current value; transitionPaceTokens with nothing + * configured reports unset (null) unless a descent it starts needs the + * default pace. + * - cancel: a converging budget stops at the live budget; nothing else + * changes. + */ + previewRuntimeSettingsTarget( + change: + | { action: 'update'; patch: AgentRuntimeSettingsPatch } + | { action: 'reset'; keys?: Array } + | { action: 'cancel' }, + ): Partial> { + const target: Partial> = {}; + if (change.action === 'update') { + const p = change.patch; + if (p.contextBudgetTokens !== undefined) target.contextBudgetTokens = p.contextBudgetTokens; + if (p.tailTokens !== undefined) target.tailTokens = p.tailTokens; + if (p.transitionPaceTokens !== undefined) target.transitionPaceTokens = p.transitionPaceTokens; + if (p.sameRoundThinkTextPolicy !== undefined) target.sameRoundThinkTextPolicy = p.sameRoundThinkTextPolicy; + return target; + } + if (change.action === 'cancel') { + if (this.contextBudgetTargetTokens !== undefined) { + target.contextBudgetTokens = this.contextBudgetTokens ?? DEFAULT_CONTEXT_BUDGET_TOKENS; + } + return target; + } + const keys = new Set(change.keys ?? ['contextBudgetTokens', 'tailTokens', 'transitionPaceTokens', 'sameRoundThinkTextPolicy']); + const hot = this.getHotContextSettings(); + let startsDescent = false; + if (keys.has('contextBudgetTokens')) { + const configured = this.configuredContextBudgetTokens ?? DEFAULT_CONTEXT_BUDGET_TOKENS; + const live = this.contextBudgetTokens ?? DEFAULT_CONTEXT_BUDGET_TOKENS; + startsDescent = configured < live; + target.contextBudgetTokens = configured; + } + if (keys.has('tailTokens') && hot) { + target.tailTokens = this.configuredTailTokens ?? hot.tailTokens; + } + if (keys.has('transitionPaceTokens') && hot) { + const descending = startsDescent || (!keys.has('contextBudgetTokens') && this.contextBudgetTargetTokens !== undefined); + target.transitionPaceTokens = descending && this.configuredTransitionPaceTokens === undefined + ? DEFAULT_TRANSITION_PACE_TOKENS + : this.configuredTransitionPaceTokens ?? null; + } + if (keys.has('sameRoundThinkTextPolicy')) { + target.sameRoundThinkTextPolicy = this.configuredSameRoundThinkTextPolicy ?? DEFAULT_SAME_ROUND_THINK_TEXT_POLICY; + } + return target; + } + private validateRuntimeSettingsPatch(patch: AgentRuntimeSettingsPatch): void { if (Object.keys(patch).length === 0) throw new Error('At least one setting is required'); if (patch.contextBudgetTokens !== undefined) { diff --git a/src/framework.ts b/src/framework.ts index 2dc25f8da..020a89cc3 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -10,6 +10,16 @@ import { ContextManager, PassthroughStrategy, WindowedPassthroughStrategy, OverB import type { CacheWireReceipt } from './kv-unified-wire.js'; import { SUBCONSCIOUS_TOOLS, SUBCONSCIOUS_TOOL_NAMES, type SubconsciousConfig } from './tune-out/tools.js'; import { callProvenance } from './call-provenance.js'; +import { + selfChangeKind, + sameValue, + type AppliedOperatorChange, + type OperatorChangeReceipt, + type ResolvedOperatorChange, + type ResolvedPresentationChange, + type ResolvedSettingsChange, + type RuntimeSettingsValues, +} from './operator-change.js'; import { TuneOutCoordinator, TUNE_OUT_DEFAULTS } from './tune-out/coordinator.js'; import type { MessageId, @@ -1473,6 +1483,11 @@ export class AgentFramework { private releasedEphemeralCandidates: WeakSet = new WeakSet(); /** Creations held until no lease is waiting or held (admitEphemeralCreation). */ private boundaryClearedWaiters: Array<() => void> = []; + /** The host's operator-change gate (FrameworkConfig.operatorChangeGate). */ + private operatorChangeGate: FrameworkConfig['operatorChangeGate'] | null = null; + /** Admissions applyResolvedOperatorChange is carrying out, by admission id: + * each lets exactly its change's tool call through the gate. */ + private activeAdmissions: Map = new Map(); /** Serialize per-server drains so reconnect and an online undo cannot race. */ private discordAwarenessDrains: Map> = new Map(); /** Framework-global inference gate; older generations cannot release it. */ @@ -1670,6 +1685,7 @@ export class AgentFramework { new OperatorLog(operatorLogPath), ); framework.providerHoldHook = config.providerHold; + framework.operatorChangeGate = config.operatorChangeGate ?? null; // Startup reconciliation of the awareness journal, before any delivery. // It derives nothing from branch ancestry: a surgery that crashed after @@ -6459,6 +6475,287 @@ export class AgentFramework { return this.heldLease.tokens; } + // --------------------------------------------------------------------------- + // Operator-change admission (FrameworkConfig.operatorChangeGate) + // --------------------------------------------------------------------------- + + /** Ask the host's gate about one resolved change. Without a gate the answer + * is apply, today's behaviour; a gate that throws or answers nonsense + * refuses the change (fails closed). Staging and refusals are logged. */ + private async askOperatorChangeGate(change: ResolvedOperatorChange): Promise< + | { outcome: 'apply' } + | { outcome: 'staged'; receipt: OperatorChangeReceipt } + | { outcome: 'gate-failed'; error: string } + > { + const gate = this.operatorChangeGate; + if (!gate) return { outcome: 'apply' }; + let answer: { outcome: 'apply' } | { outcome: 'staged'; receipt: OperatorChangeReceipt } | { outcome: 'gate-failed'; error: string }; + try { + const decision = await gate(change); + if (decision?.decision === 'apply') answer = { outcome: 'apply' }; + else if (decision?.decision === 'staged' + && typeof decision.receipt?.id === 'string' && typeof decision.receipt?.text === 'string') { + answer = { outcome: 'staged', receipt: { id: decision.receipt.id, text: decision.receipt.text } }; + } else answer = { outcome: 'gate-failed', error: 'the operator-change gate returned no valid decision' }; + } catch (error) { + answer = { outcome: 'gate-failed', error: error instanceof Error ? error.message : String(error) }; + } + if (answer.outcome !== 'apply') { + this.recordOperatorAction({ + kind: answer.outcome === 'staged' ? 'operator-change-staged' : 'operator-change-refused', + agent: change.agent, + ...(change.requester ? { requester: change.requester } : {}), + params: { changeId: change.id, change: change.kind, surface: change.surface }, + ...(answer.outcome === 'staged' + ? { result: { receipt: answer.receipt.id } } + : { error: `gate failed: ${answer.error}` }), + }); + } + return answer; + } + + /** The change `admission` was registered for (applyResolvedOperatorChange, + * or an applied puppet call), when it is exactly this agent, tool and + * input. Any other admission covers nothing. */ + private admittedChange( + admission: OperatorAdmission | undefined, + agentName: string, + tool: string, + input: unknown, + ): ResolvedSettingsChange | ResolvedPresentationChange | null { + if (!admission) return null; + const change = this.activeAdmissions?.get(admission.id); + if (!change || change.agent !== agentName) return null; + const changeTool = change.kind === 'agent-settings' ? 'agent_settings' : change.tool; + return tool === changeTool && sameValue(change.input, input) ? change : null; + } + + /** The core patch / reset keys and extension keys an agent_settings input touches. */ + private parseSettingsChange(input: Record): { + action: { action: 'update'; patch: AgentRuntimeSettingsPatch } | { action: 'reset'; keys?: Array } | { action: 'cancel' }; + extensionKeys: string[]; + } { + const extensions = [...this.collectAgentSettingsExtensions().values()]; + const extKeyOwner = (key: string) => extensions.find((ext) => ext.keys.includes(key)); + const core: Record = { + context_budget_tokens: 'contextBudgetTokens', + tail_tokens: 'tailTokens', + transition_pace_tokens: 'transitionPaceTokens', + same_round_think_text_policy: 'sameRoundThinkTextPolicy', + }; + if (input.action === 'cancel') return { action: { action: 'cancel' }, extensionKeys: [] }; + if (input.action === 'update') { + const patch: AgentRuntimeSettingsPatch = {}; + const extensionKeys: string[] = []; + for (const [key, value] of Object.entries(input)) { + if (key === 'action' || key === 'immediate' || value === undefined) continue; + const coreKey = core[key]; + if (coreKey === 'sameRoundThinkTextPolicy') { + if (typeof value !== 'string') throw new Error(`${key} must be a string`); + patch.sameRoundThinkTextPolicy = value as AgentRuntimeSettingsPatch['sameRoundThinkTextPolicy']; + } else if (coreKey) { + if (typeof value !== 'number' || !Number.isFinite(value)) throw new Error(`${key} must be a number`); + (patch as Record)[coreKey] = value; + } else if (extKeyOwner(key)) extensionKeys.push(key); + else throw new Error(`Unknown setting: ${key}`); + } + return { action: { action: 'update', patch }, extensionKeys }; + } + if (input.action === 'reset') { + if (input.settings === undefined) { + return { action: { action: 'reset' }, extensionKeys: extensions.flatMap((ext) => (ext.reset ? ext.keys : [])) }; + } + if (!Array.isArray(input.settings)) throw new Error('reset `settings` must be an array'); + const keys: Array = []; + const extensionKeys: string[] = []; + for (const name of input.settings) { + if (typeof name === 'string' && core[name]) keys.push(core[name]); + else if (typeof name === 'string' && extKeyOwner(name)) extensionKeys.push(name); + else throw new Error(`Unknown reset setting: ${String(name)}`); + } + return { action: keys.length > 0 ? { action: 'reset', keys } : { action: 'reset', keys: [] }, extensionKeys }; + } + throw new Error('agent_settings: action must be get, update, reset, or cancel'); + } + + /** The current values of the given settings: core keys from the snapshot + * (unset reads null), extension keys from their owners. */ + private currentSettingsValues(agentName: string, coreKeys: string[], extensionKeys: string[]): { + core: RuntimeSettingsValues; + extensions: Record; + } { + const snapshot = this.getAgentRuntimeSettings(agentName) as unknown as Record; + const core: Record = {}; + for (const key of coreKeys) core[key] = snapshot[key] ?? null; + const extensions: Record = {}; + if (extensionKeys.length > 0) { + for (const ext of this.collectAgentSettingsExtensions().values()) { + const values = ext.get(agentName); + for (const key of extensionKeys) if (ext.keys.includes(key)) extensions[key] = values[key] ?? null; + } + } + return { core: core as RuntimeSettingsValues, extensions }; + } + + /** Resolve an operator's or module's mutating self-change call to the + * absolute change the gate sees. Throws on invalid input. */ + private resolveSelfChange( + agentName: string, + tool: string, + input: Record, + surface: 'puppet' | 'host', + requester?: OperatorRequester, + ): ResolvedSettingsChange | ResolvedPresentationChange { + const agent = this.agents.get(agentName); + if (!agent) throw new Error(`Unknown agent: ${agentName}`); + const base = { + id: randomUUID(), + agent: agentName, + surface, + ...(requester ? { requester } : {}), + resolvedAt: Date.now(), + sourceBranch: this.store.currentBranch().name, + }; + if (tool === 'agent_settings') { + const { action, extensionKeys } = this.parseSettingsChange(input); + const target = agent.previewRuntimeSettingsTarget(action); + const current = this.currentSettingsValues(agentName, Object.keys(target), extensionKeys); + return { + ...base, + kind: 'agent-settings', + input: structuredClone(input), + from: current.core, + target, + ...(extensionKeys.length > 0 ? { extensions: current.extensions } : {}), + }; + } + const name = (input as { name?: unknown }).name; + const entry = typeof name === 'string' + ? this.inspectToolPresentation(agentName)?.entries.find((e) => e.name === name) + : undefined; + if (!entry) throw new Error(`Tool presentation has no entry ${JSON.stringify(name)} for ${agentName}`); + return { + ...base, + kind: 'tool-presentation', + tool: tool as ResolvedPresentationChange['tool'], + input: structuredClone(input), + from: { name: entry.name, visible: entry.visible, description: entry.description }, + }; + } + + /** Refuse (stale) a self-change whose from-state or target no longer holds. */ + private revalidateSelfChange(change: ResolvedSettingsChange | ResolvedPresentationChange): void { + const stale = (what: string) => { + throw new OperatorActionError('stale', `Change ${change.id} for ${change.agent} no longer holds: ${what}`); + }; + if (!this.agents.has(change.agent)) throw new OperatorActionError('unknown-agent', `Unknown agent: ${change.agent}`); + if (change.kind === 'agent-settings') { + const { action, extensionKeys } = this.parseSettingsChange(change.input); + const current = this.currentSettingsValues(change.agent, Object.keys(change.from), extensionKeys); + if (!sameValue(current.core, change.from)) { + stale(`its settings moved from ${JSON.stringify(change.from)} to ${JSON.stringify(current.core)}`); + } + if (change.extensions && !sameValue(current.extensions, change.extensions)) { + stale(`its host-managed settings moved from ${JSON.stringify(change.extensions)} to ${JSON.stringify(current.extensions)}`); + } + const target = this.agents.get(change.agent)!.previewRuntimeSettingsTarget(action); + if (!sameValue(target, change.target)) { + stale(`it would now set ${JSON.stringify(target)}, not the approved ${JSON.stringify(change.target)}`); + } + return; + } + const entry = this.inspectToolPresentation(change.agent)?.entries.find((e) => e.name === change.from.name); + const now = entry ? { name: entry.name, visible: entry.visible, description: entry.description } : null; + if (!sameValue(now, change.from)) { + stale(`the presentation of ${change.from.name} moved from ${JSON.stringify(change.from)} to ${JSON.stringify(now)}`); + } + } + + /** + * Run a self-change tool call (agent_settings, tool presentation) through + * the gate when an operator or a module made it. The agent's own model + * calls (no origin), reads, and calls carrying an admission registered for + * exactly this change run as before. Otherwise the call is resolved and + * the gate decides: staged returns its receipt as the call's error; apply + * revalidates and runs exactly that call; a failed gate refuses it. + */ + private async gatedSelfChange(agentName: string, call: ToolCall, run: () => ToolResult): Promise { + if (!this.operatorChangeGate || call.origin === undefined) return run(); + if (!selfChangeKind(call.name, call.input)) return run(); + const input = (call.input ?? {}) as Record; + const admitted = this.admittedChange(call.admission, agentName, call.name, input); + if (admitted) { + // The last check before the change runs, with no await in between. + try { + this.revalidateSelfChange(admitted); + } catch (error) { + return { success: false, isError: true, error: error instanceof Error ? error.message : String(error) }; + } + return run(); + } + let change: ResolvedSettingsChange | ResolvedPresentationChange; + try { + change = this.resolveSelfChange(agentName, call.name, input, call.origin); + } catch { + return run(); // invalid input: the handler reports its own error + } + const answer = await this.askOperatorChangeGate(change); + if (answer.outcome === 'staged') { + return { success: false, isError: true, error: answer.receipt.text, data: { staged: answer.receipt } }; + } + if (answer.outcome === 'gate-failed') { + return { success: false, isError: true, error: `Operator change refused: ${answer.error}` }; + } + try { + this.revalidateSelfChange(change); + } catch (error) { + return { success: false, isError: true, error: error instanceof Error ? error.message : String(error) }; + } + return run(); + } + + /** + * Apply a staged change at a safe boundary, under the host's held lease. + * Revalidates first and refuses with OperatorActionError('stale') when the + * change no longer holds, so the host can restage it. `admission` is the + * host's id for the admitted change; for the change's duration it lets + * exactly this change's tool call through the gate, and nothing else. + * An operator's (puppet) change is carried out as the puppet call it was, + * on the lease's token, so the agent's context records it as before; a + * module's is applied by the handler directly. + */ + async applyResolvedOperatorChange( + change: ResolvedOperatorChange, + opts: { lease: SafeBoundaryLease; admission: OperatorAdmission }, + ): Promise { + this.heldLeaseTokens(opts.lease, `apply ${change.kind} for ${change.agent}`); + this.revalidateSelfChange(change); + const tool = change.kind === 'agent-settings' ? 'agent_settings' : change.tool; + this.activeAdmissions.set(opts.admission.id, change); + try { + let result: ToolResult; + if (change.surface === 'puppet') { + result = (await this.puppetToolCall(change.agent, tool, change.input, { lease: opts.lease, admission: opts.admission })).result; + } else { + const call: ToolCall = { id: `admitted-${change.id}`, name: tool, input: change.input, callerAgentName: change.agent, origin: 'host', admission: opts.admission }; + result = change.kind === 'agent-settings' + ? this.runAgentSettingsToolCall(change.agent, call) + : this.editToolPresentation(change.agent, call); + } + this.recordOperatorAction({ + kind: 'operator-change-applied', + agent: change.agent, + ...(change.requester ? { requester: change.requester } : {}), + params: { changeId: change.id, change: change.kind, surface: change.surface, admission: opts.admission.id }, + ...(result.isError ? { error: result.error ?? 'failed' } : { result: { applied: true } }), + }); + if (result.isError) throw new OperatorActionError('failed', result.error ?? `${change.kind} failed`); + return { kind: change.kind, result: result.data } as AppliedOperatorChange; + } finally { + this.activeAdmissions.delete(opts.admission.id); + } + } + /** * After a failed surgery has put the source back: retire its prepared * awareness batch, but only when the store is confirmed on the source. If @@ -11835,6 +12132,55 @@ export class AgentFramework { * then applies instead of staging again. */ async puppetToolCall( + agentName: string, + toolName: string, + input: Record, + opts?: { lease?: SafeBoundaryLease; admission?: OperatorAdmission; requester?: OperatorRequester }, + ): Promise<{ toolUseId: string | null; result: ToolResult; staged?: OperatorChangeReceipt }> { + if (!this.agents.get(agentName)) throw new Error(`Unknown agent: ${agentName}`); + // An operator's mutating self-change (agent_settings update/reset/cancel, + // a tool-presentation edit) goes through the host's gate before anything + // runs. Staged: nothing executes and no pair is stored, since a pair + // would testify the tool ran; the receipt comes back with toolUseId null. + // Apply: the call proceeds as exactly the resolved change, revalidated, + // and admitted for its own duration only. + if ( + this.operatorChangeGate && + selfChangeKind(toolName, input) && + !this.admittedChange(opts?.admission, agentName, toolName, input) + ) { + let change: ResolvedSettingsChange | ResolvedPresentationChange | undefined; + try { + change = this.resolveSelfChange(agentName, toolName, input, 'puppet', opts?.requester); + } catch { + change = undefined; // invalid input: the tool refuses it itself, as before + } + if (change) { + const answer = await this.askOperatorChangeGate(change); + if (answer.outcome === 'staged') { + return { + toolUseId: null, + result: { success: false, isError: true, error: answer.receipt.text }, + staged: answer.receipt, + }; + } + if (answer.outcome === 'gate-failed') { + throw new Error(`puppet refused: the operator-change gate failed (${answer.error})`); + } + this.revalidateSelfChange(change); + const admission: OperatorAdmission = { id: change.id }; + this.activeAdmissions.set(admission.id, change); + try { + return await this.puppetToolCallUngated(agentName, toolName, input, { ...opts, admission }); + } finally { + this.activeAdmissions.delete(admission.id); + } + } + } + return this.puppetToolCallUngated(agentName, toolName, input, opts); + } + + private async puppetToolCallUngated( agentName: string, toolName: string, input: Record, @@ -11988,7 +12334,10 @@ export class AgentFramework { } private async executeToolCallFrom(call: ToolCall, origin: ChannelToolOrigin): Promise { - if (isPresentationTool(call.name)) return this.editToolPresentation(call.callerAgentName ?? '__ephemeral__', call); + if (isPresentationTool(call.name)) { + const agentName = call.callerAgentName ?? '__ephemeral__'; + return this.gatedSelfChange(agentName, call, () => this.editToolPresentation(agentName, call)); + } // Client-side programmatic tool calling for promise-based callers // (SubagentModule ephemerals). Keyed by callerAgentName so each ephemeral // gets its own interpreter state. @@ -12034,7 +12383,8 @@ export class AgentFramework { // route a puppeted agent_settings fell through to module-name parsing // and failed as "Invalid tool name format". if (call.name === 'agent_settings') { - return this.runAgentSettingsToolCall(call.callerAgentName ?? '__ephemeral__', call); + const agentName = call.callerAgentName ?? '__ephemeral__'; + return this.gatedSelfChange(agentName, call, () => this.runAgentSettingsToolCall(agentName, call)); } // Module tools @@ -13092,8 +13442,15 @@ export class AgentFramework { // Enrich call with caller identity so modules can resolve the calling agent const enrichedCall: ToolCall = { ...call, callerAgentName: agentName }; if (isPresentationTool(call.name)) { - const result = this.editToolPresentation(agentName, enrichedCall); - this.pushEvent({type:'tool-result',callId:call.id,agentName,moduleName:'tool-presentation',result}); + const push = (result: ToolResult) => + this.pushEvent({type:'tool-result',callId:call.id,agentName,moduleName:'tool-presentation',result}); + // Only a call an operator or module made (a script's inner call can + // carry one) waits on the gate; the model's own call stays synchronous. + if (enrichedCall.origin !== undefined && this.operatorChangeGate) { + void this.gatedSelfChange(agentName, enrichedCall, () => this.editToolPresentation(agentName, enrichedCall)).then(push); + } else { + push(this.editToolPresentation(agentName, enrichedCall)); + } return; } @@ -16516,14 +16873,20 @@ export class AgentFramework { } private dispatchAgentSettingsToolCall(agentName: string, call: ToolCall): void { - const result = this.runAgentSettingsToolCall(agentName, call); - this.pushEvent({ + const push = (result: ToolResult) => this.pushEvent({ type: 'tool-result', callId: call.id, agentName, moduleName: 'agent', result, }); + // Only a call an operator or module made (a script's inner call can carry + // one) waits on the gate; the model's own call stays synchronous. + if (call.origin !== undefined && this.operatorChangeGate) { + void this.gatedSelfChange(agentName, call, () => this.runAgentSettingsToolCall(agentName, call)).then(push); + } else { + push(this.runAgentSettingsToolCall(agentName, call)); + } } /** diff --git a/src/index.ts b/src/index.ts index 4fb06cbe7..408fcd77e 100644 --- a/src/index.ts +++ b/src/index.ts @@ -4,6 +4,15 @@ export * from './types/index.js'; // Core classes export { AgentFramework, BudgetPreflightError, ResumeBlockedError } from './framework.js'; export type { RuntimeSettingsPreview, HostModeStatus, SafeBoundaryLease, SafeBoundaryOptions } from './framework.js'; +export type { + OperatorChangeDecision, + OperatorChangeReceipt, + ResolvedOperatorChange, + ResolvedSettingsChange, + ResolvedPresentationChange, + RuntimeSettingsValues, + AppliedOperatorChange, +} from './operator-change.js'; export { Agent } from './agent.js'; export type { StartStreamResult } from './agent.js'; export { ProcessQueueImpl } from './queue.js'; diff --git a/src/operator-change.ts b/src/operator-change.ts new file mode 100644 index 000000000..aa65d026b --- /dev/null +++ b/src/operator-change.ts @@ -0,0 +1,102 @@ +/** + * Operator-change admission: the framework resolves each of its own operator + * mutations to an absolute change and asks the host's gate before acting + * (FrameworkConfig.operatorChangeGate). The gate answers `apply` (act now, + * on exactly this change) or `staged` (the host will apply it later, at a + * safe boundary, through AgentFramework.applyResolvedOperatorChange). + * + * A change is resolved once. Application, immediate or delayed, revalidates + * it against stable identity (the active branch, the settings it was + * resolved from) and applies exactly what was resolved, never the relative + * command again: a change that no longer holds is refused as `stale` and the + * host restages it. + */ +import type { OperatorRequester } from './operator-log.js'; + +/** What the host's gate decided for one resolved change. */ +export type OperatorChangeDecision = + | { decision: 'apply' } + | { decision: 'staged'; receipt: OperatorChangeReceipt }; + +/** The host's receipt for a staged change, shown to whoever asked. */ +export interface OperatorChangeReceipt { + id: string; + text: string; +} + +/** Settings an agent_settings action touches, by snapshot key. null = unset. */ +export type RuntimeSettingsValues = Partial< + Record<'contextBudgetTokens' | 'tailTokens' | 'transitionPaceTokens' | 'sameRoundThinkTextPolicy', number | string | null> +>; + +interface ResolvedOperatorChangeBase { + /** The framework's id for this resolution. */ + id: string; + agent: string; + /** Where the change came from: an operator's host/command, an operator + * acting through the agent's own tools (puppet), or a module. */ + surface: 'host-command' | 'puppet' | 'host'; + requester?: OperatorRequester; + /** Epoch ms. */ + resolvedAt: number; + /** The active branch when resolved: the stable identity application + * checks. Appending messages (a notice, an answer) doesn't change it. */ + sourceBranch: string; +} + +/** agent_settings update / reset / cancel, from an operator or a module. */ +export interface ResolvedSettingsChange extends ResolvedOperatorChangeBase { + kind: 'agent-settings'; + /** The agent_settings input as given. */ + input: Record; + /** The touched core settings when resolved (getRuntimeSettings). */ + from: RuntimeSettingsValues; + /** The concrete values the action leaves them at (Agent.previewRuntimeSettingsTarget). + * Application applies the action only if it still produces exactly these. */ + target: RuntimeSettingsValues; + /** Host-managed (extension) keys the input touches, with their values when + * resolved. Their owners apply them; application checks they haven't moved. */ + extensions?: Record; +} + +/** set_tool_visibility / set_tool_description, from an operator or a module. */ +export interface ResolvedPresentationChange extends ResolvedOperatorChangeBase { + kind: 'tool-presentation'; + tool: 'set_tool_visibility' | 'set_tool_description'; + input: Record; + /** The targeted tool's presentation entry when resolved. */ + from: { name: string; visible: boolean; description: string }; +} + +export type ResolvedOperatorChange = + | ResolvedSettingsChange + | ResolvedPresentationChange; + +/** What applyResolvedOperatorChange did. */ +export type AppliedOperatorChange = + | { kind: 'agent-settings'; result: unknown } + | { kind: 'tool-presentation'; result: unknown }; + +/** The self-change kind a tool call is, when it mutates the agent's own body: + * agent_settings update/reset/cancel, or a tool-presentation edit. */ +export function selfChangeKind(tool: string, input: unknown): 'agent-settings' | 'tool-presentation' | null { + if (tool === 'agent_settings') { + const action = (input as { action?: unknown } | null | undefined)?.action; + return action === 'update' || action === 'reset' || action === 'cancel' ? 'agent-settings' : null; + } + if (tool === 'set_tool_visibility' || tool === 'set_tool_description') return 'tool-presentation'; + return null; +} + +/** Order-independent structural equality for tool inputs and settings values. */ +export function sameValue(a: unknown, b: unknown): boolean { + return stableJson(a) === stableJson(b); +} + +function stableJson(value: unknown): string { + if (value === undefined) return 'null'; + if (value === null || typeof value !== 'object') return JSON.stringify(value); + if (Array.isArray(value)) return `[${value.map(stableJson).join(',')}]`; + const keys = Object.keys(value as Record).sort(); + return `{${keys.map((k) => `${JSON.stringify(k)}:${stableJson((value as Record)[k])}`).join(',')}}`; +} diff --git a/src/operator-log.ts b/src/operator-log.ts index 690c89e36..1112bc546 100644 --- a/src/operator-log.ts +++ b/src/operator-log.ts @@ -178,7 +178,7 @@ export interface SurgeryMarksPreview { * lets surfaces distinguish "agent busy — quiesce first" from bad input. */ export class OperatorActionError extends Error { constructor( - readonly code: 'unknown-agent' | 'agent-busy' | 'unknown-message' | 'invalid' | 'failed', + readonly code: 'unknown-agent' | 'agent-busy' | 'unknown-message' | 'invalid' | 'failed' | 'stale', message: string, options?: { cause?: unknown }, ) { diff --git a/src/types/framework.ts b/src/types/framework.ts index 92d70fe99..00dbb3d3c 100644 --- a/src/types/framework.ts +++ b/src/types/framework.ts @@ -192,6 +192,20 @@ export interface FrameworkConfig { */ providerHold?: ProviderHoldHook; + /** + * The host's gate for the framework's own operator mutations: host/command + * undo, hide and unstick, and an operator's or module's mutating + * agent_settings / tool-presentation calls. Each is resolved to an absolute + * change first; the gate answers `apply` (the framework applies exactly + * that change now, after revalidating it) or `staged` with a receipt (the + * host applies it later through applyResolvedOperatorChange). The agent's + * own model calls never reach it. A gate that throws refuses the change. + * Absent: every operator path behaves as it always has. + */ + operatorChangeGate?: ( + change: import('../operator-change.js').ResolvedOperatorChange, + ) => Promise; + /** Interval for periodic store sync in milliseconds (default: 1000ms, 0 to disable) */ syncIntervalMs?: number; diff --git a/test/operator-change-gate.test.ts b/test/operator-change-gate.test.ts new file mode 100644 index 000000000..773dbaf31 --- /dev/null +++ b/test/operator-change-gate.test.ts @@ -0,0 +1,298 @@ +/** + * Operator-change admission for self-change tools (agent_settings, + * tool presentation) made by an operator (puppet) or a module (host). + * + * The call is resolved to an absolute change (from-state and concrete + * target) and the host's gate decides: staged returns a receipt and runs + * nothing (a puppet stores no pair); apply runs exactly that change after + * revalidating it; a failed gate refuses. The agent's own model calls never + * reach the gate. applyResolvedOperatorChange later applies a staged change + * under a held lease, refusing it as stale if it no longer holds, and its + * admission lets only that change's call through. + */ +import { describe, it, beforeEach, afterEach } from 'node:test'; +import assert from 'node:assert/strict'; +import { tmpdir } from 'node:os'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { join } from 'node:path'; +import { AgentFramework, AutobiographicalStrategy, WorkspaceModule } from '../src/index.js'; +import type { + InferenceRequest, + Module, + ModuleContext, + OperatorChangeDecision, + ResolvedOperatorChange, + ToolCall, + ToolDefinition, + ToolResult, +} from '../src/index.js'; +import { MockMembrane, createMockResponse } from './helpers/mock-membrane.js'; + +class Mover implements Module { + readonly name = 'mover'; + ctx!: ModuleContext; + async start(ctx: ModuleContext): Promise { this.ctx = ctx; } + async stop(): Promise {} + getTools(): ToolDefinition[] { + return [{ name: 'set_budget', description: 'Change the caller budget.', inputSchema: { type: 'object', properties: {} } }]; + } + async handleToolCall(call: ToolCall): Promise { + return this.ctx.callTool!({ + id: 'delegated', name: 'agent_settings', input: { action: 'update', context_budget_tokens: 130_000 }, callerAgentName: call.callerAgentName, + }); + } + async onProcess(): Promise> { return {}; } +} + +type Internals = { pendingRequests: InferenceRequest[]; activeAdmissions: Map }; + +describe('operator-change gate: self-change tools', () => { + let tempDir: string; + let membrane: MockMembrane; + let framework: AgentFramework; + let mover: Mover; + let asked: ResolvedOperatorChange[]; + let decide: (change: ResolvedOperatorChange) => Promise; + let quiet: typeof console.log; + + const staged = async (change: ResolvedOperatorChange): Promise => + ({ decision: 'staged', receipt: { id: `rev-${asked.length}`, text: `staged ${change.kind} as rev-${asked.length}; waiting for scout` } }); + + beforeEach(async () => { + tempDir = mkdtempSync(join(tmpdir(), 'operator-gate-')); + membrane = new MockMembrane(); + mover = new Mover(); + asked = []; + decide = staged; + framework = await AgentFramework.create({ + storePath: join(tempDir, 'test.chronicle'), + membrane: membrane.asMembrane(), + agents: [{ name: 'scout', model: 'test-model', systemPrompt: 'You are scout.', maxTokens: 1000 }], + modules: [mover], + operatorChangeGate: async (change) => { asked.push(structuredClone(change)); return decide(change); }, + }); + quiet = console.log; + console.log = () => {}; + }); + afterEach(async () => { + console.log = quiet; + await framework.stop(); + rmSync(tempDir, { recursive: true, force: true }); + }); + + const budget = () => framework.getAgentRuntimeSettings('scout').contextBudgetTokens; + const pairs = () => (framework.getAgent('scout')!.getContextManager().getAllMessages() as Array<{ content: Array<{ type: string }> }>) + .filter((m) => m.content[0]?.type === 'tool_use').length; + + it("stages an operator's settings change: a receipt, nothing run, no pair stored", async () => { + const before = budget(); + const out = await framework.puppetToolCall('scout', 'agent_settings', { action: 'update', context_budget_tokens: 120_000 }); + assert.equal(out.toolUseId, null); + assert.equal(out.staged?.id, 'rev-1'); + assert.match(String(out.result.error), /staged agent-settings as rev-1/); + assert.equal(budget(), before, 'unchanged'); + assert.equal(pairs(), 0, 'no pair testifies the tool ran'); + + assert.equal(asked.length, 1); + const change = asked[0]!; + assert.equal(change.kind, 'agent-settings'); + assert.equal(change.surface, 'puppet'); + assert.equal(change.agent, 'scout'); + assert.deepEqual(change.kind === 'agent-settings' && { from: change.from, target: change.target }, + { from: { contextBudgetTokens: before }, target: { contextBudgetTokens: 120_000 } }); + assert.ok(framework.getOperatorLog().some((e) => e.kind === 'operator-change-staged')); + }); + + it('applies at once on apply, asking only once even though the call goes through two checks', async () => { + decide = async () => ({ decision: 'apply' }); + const out = await framework.puppetToolCall('scout', 'agent_settings', { action: 'update', context_budget_tokens: 120_000 }); + assert.equal(out.result.success, true, String(out.result.error)); + assert.equal(typeof out.toolUseId, 'string'); + assert.equal(budget(), 120_000); + assert.equal(pairs(), 1); + assert.equal(asked.length, 1); + assert.equal((framework as unknown as Internals).activeAdmissions.size, 0, 'the admission ended with the call'); + }); + + it('refuses when the gate fails, applying nothing', async () => { + decide = async () => { throw new Error('journal unavailable'); }; + const before = budget(); + await assert.rejects( + framework.puppetToolCall('scout', 'agent_settings', { action: 'update', context_budget_tokens: 120_000 }), + /operator-change gate failed \(journal unavailable\)/, + ); + assert.equal(budget(), before); + assert.equal(pairs(), 0); + assert.ok(framework.getOperatorLog().some((e) => e.kind === 'operator-change-refused')); + }); + + it('lets reads, and the agent\'s own model calls, through without asking', async () => { + const read = await framework.puppetToolCall('scout', 'agent_settings', { action: 'get' }); + assert.equal(read.result.success, true); + membrane.pushResponse(createMockResponse([{ type: 'tool_use', id: 'toolu_own_1', name: 'agent_settings', input: { action: 'update', context_budget_tokens: 140_000 } }], 'tool_use')); + membrane.pushResponse(createMockResponse([{ type: 'text', text: 'done' }])); + (framework as unknown as Internals).pendingRequests.push( + { agentName: 'scout', reason: 'mcpl:channel-incoming', source: 'test', timestamp: Date.now() } as InferenceRequest, + ); + await framework.runUntilIdle(); + assert.equal(budget(), 140_000, "the agent's own change applies"); + assert.equal(asked.length, 0, 'the gate was never asked'); + }); + + it("stages a puppet's change made through a module, and a module's own change", async () => { + const before = budget(); + const viaModule = await framework.puppetToolCall('scout', 'mover--set_budget', {}); + assert.match(String(viaModule.result.error), /staged agent-settings/, 'the wrapper sees the receipt'); + const own = await mover.ctx.callTool!({ id: 'own', name: 'agent_settings', input: { action: 'update', context_budget_tokens: 150_000 }, callerAgentName: 'scout' }); + assert.match(String(own.error), /staged agent-settings/); + assert.equal(budget(), before); + assert.deepEqual(asked.map((c) => c.surface), ['puppet', 'host']); + }); + + it("lets invalid input through ungated, to the tool's own refusal", async () => { + const out = await framework.puppetToolCall('scout', 'agent_settings', { action: 'update', context_budget_tokens: 'lots' }); + assert.equal(asked.length, 0, 'nothing to resolve, so nothing staged'); + assert.equal(out.result.isError, true); + }); + + it('admits only the exact call its admission was registered for, and revalidates it as it runs', async () => { + await framework.puppetToolCall('scout', 'agent_settings', { action: 'update', context_budget_tokens: 120_000 }); + const change = asked[0]!; + const admissions = (framework as unknown as Internals).activeAdmissions; + admissions.set('adm-1', change); + try { + // Same admission, different input: not covered, so the gate is asked. + const other = await framework.puppetToolCall('scout', 'agent_settings', { action: 'update', context_budget_tokens: 125_000 }, { admission: { id: 'adm-1' } }); + assert.equal(other.toolUseId, null); + assert.equal(asked.length, 2); + // The exact call, after the settings moved: refused at the moment it would run. + framework.updateAgentRuntimeSettings('scout', { contextBudgetTokens: 110_000 }); + const covered = await framework.puppetToolCall('scout', 'agent_settings', { action: 'update', context_budget_tokens: 120_000 }, { admission: { id: 'adm-1' } }); + assert.equal(asked.length, 2, 'covered: not asked again'); + assert.match(String(covered.result.error), /no longer holds/); + assert.equal(budget(), 110_000, 'nothing applied'); + } finally { + admissions.delete('adm-1'); + } + }); + + it('ignores an admission nobody registered', async () => { + const out = await framework.puppetToolCall('scout', 'agent_settings', { action: 'update', context_budget_tokens: 120_000 }, { admission: { id: 'forged' } }); + assert.equal(out.toolUseId, null, 'still staged'); + assert.equal(asked.length, 1); + }); + + it('applies a staged change under a lease as the puppet call it was, without asking again', async () => { + await framework.puppetToolCall('scout', 'agent_settings', { action: 'update', context_budget_tokens: 120_000 }); + const change = asked[0]!; + const applied = await framework.runAtSafeBoundary({ verb: 'apply rev-1' }, (lease) => + framework.applyResolvedOperatorChange(change, { lease, admission: { id: 'rev-1' } })); + assert.equal(applied.kind, 'agent-settings'); + assert.equal(budget(), 120_000); + assert.equal(pairs(), 1, 'stored as the puppet act'); + assert.equal(asked.length, 1, 'not asked again'); + assert.ok(framework.getOperatorLog().some((e) => e.kind === 'operator-change-applied' && (e.params as { admission?: string })?.admission === 'rev-1')); + assert.equal((framework as unknown as Internals).activeAdmissions.size, 0); + }); + + it('refuses a staged change as stale when the settings moved since', async () => { + await framework.puppetToolCall('scout', 'agent_settings', { action: 'update', context_budget_tokens: 120_000 }); + const change = asked[0]!; + framework.updateAgentRuntimeSettings('scout', { contextBudgetTokens: 110_000 }); // the agent changed it meanwhile + await assert.rejects( + framework.runAtSafeBoundary({ verb: 'apply rev-1' }, (lease) => + framework.applyResolvedOperatorChange(change, { lease, admission: { id: 'rev-1' } })), + (e: Error & { code?: string }) => e.code === 'stale' && /moved from/.test(e.message), + ); + assert.equal(budget(), 110_000, 'the newer state stands'); + }); + + it('resolves a reset to the concrete values it restores, and refuses if those change', async () => { + framework.updateAgentRuntimeSettings('scout', { contextBudgetTokens: 120_000 }); + await framework.puppetToolCall('scout', 'agent_settings', { action: 'reset', settings: ['context_budget_tokens'] }); + const change = asked[0]!; + assert.equal(change.kind === 'agent-settings' && change.target.contextBudgetTokens, 100_000, 'the configured default, concretely'); + const agent = framework.getAgent('scout')! as unknown as { previewRuntimeSettingsTarget: (c: unknown) => unknown }; + const real = agent.previewRuntimeSettingsTarget.bind(agent); + agent.previewRuntimeSettingsTarget = () => ({ contextBudgetTokens: 90_000 }); // the configuration says otherwise now + try { + await assert.rejects( + framework.runAtSafeBoundary({ verb: 'apply' }, (lease) => + framework.applyResolvedOperatorChange(change, { lease, admission: { id: 'rev-1' } })), + (e: Error & { code?: string }) => e.code === 'stale' && /not the approved/.test(e.message), + ); + } finally { + agent.previewRuntimeSettingsTarget = real; + } + assert.equal(budget(), 120_000, 'nothing applied'); + }); + + it("needs the held lease to apply, whether the change was an operator's or a module's", async () => { + await framework.puppetToolCall('scout', 'agent_settings', { action: 'update', context_budget_tokens: 120_000 }); + await mover.ctx.callTool!({ id: 'own', name: 'agent_settings', input: { action: 'update', context_budget_tokens: 150_000 }, callerAgentName: 'scout' }); + const [puppeted, moduleOwn] = asked; + assert.deepEqual([puppeted!.surface, moduleOwn!.surface], ['puppet', 'host']); + let stale: Parameters[1]>[0] | null = null; + await framework.runAtSafeBoundary({ verb: 'first' }, async (lease) => { stale = lease; }); + const before = budget(); + for (const change of [puppeted!, moduleOwn!]) { + await assert.rejects( + framework.applyResolvedOperatorChange(change, { lease: stale!, admission: { id: 'rev-x' } }), + /not the lease currently held/, + ); + } + assert.equal(budget(), before, 'neither applied'); + }); + + it("applies a module's staged change directly, under the lease", async () => { + await mover.ctx.callTool!({ id: 'own', name: 'agent_settings', input: { action: 'update', context_budget_tokens: 150_000 }, callerAgentName: 'scout' }); + const change = asked[0]!; + await framework.runAtSafeBoundary({ verb: 'apply' }, (lease) => + framework.applyResolvedOperatorChange(change, { lease, admission: { id: 'rev-m' } })); + assert.equal(budget(), 150_000); + assert.equal(pairs(), 0, 'a module change stores no pair'); + assert.equal(asked.length, 1); + }); +}); + +describe('operator-change gate: tool presentation', () => { + it('stages an operator\'s presentation edit with its from-state, and applies it later under a lease', async () => { + const dir = mkdtempSync(join(tmpdir(), 'operator-gate-presentation-')); + const workspace = new WorkspaceModule({ mounts: [{ name: 'board', path: dir, mode: 'read-write', watch: 'never' }] }); + const asked: ResolvedOperatorChange[] = []; + const framework = await AgentFramework.create({ + storePath: join(dir, 'store'), + membrane: new MockMembrane().asMembrane(), + agents: [{ + name: 'ada', model: 'test', systemPrompt: '.', + strategy: new AutobiographicalStrategy({ adaptiveResolution: true, foldingStrategy: 'kv-stable', recentWindowTokens: 30000, kvStableReachTokens: 8000 }), + toolPresentation: { path: join(dir, 'tools.json'), cataloguePath: 'board/tools.md' }, + }], + modules: [workspace], + operatorChangeGate: async (change) => { + asked.push(structuredClone(change)); + return { decision: 'staged', receipt: { id: 'rev-p', text: 'staged' } }; + }, + }); + const quiet = console.log; + console.log = () => {}; + try { + const visible = () => framework.inspectToolPresentation('ada')!.entries.find((e) => e.name === 'workspace--glob')!.visible; + const out = await framework.puppetToolCall('ada', 'set_tool_visibility', { name: 'workspace--glob', visible: false }); + assert.equal(out.toolUseId, null); + assert.equal(visible(), true, 'not applied'); + const change = asked[0]!; + assert.equal(change.kind, 'tool-presentation'); + assert.deepEqual(change.kind === 'tool-presentation' && { name: change.from.name, visible: change.from.visible }, { name: 'workspace--glob', visible: true }); + + await framework.runAtSafeBoundary({ verb: 'apply rev-p' }, (lease) => + framework.applyResolvedOperatorChange(change, { lease, admission: { id: 'rev-p' } })); + assert.equal(visible(), false, 'applied under the lease'); + assert.equal(asked.length, 1); + } finally { + console.log = quiet; + await framework.stop(); + rmSync(dir, { recursive: true, force: true }); + } + }); +}); diff --git a/test/puppet-tool-call.test.ts b/test/puppet-tool-call.test.ts index 08e34088a..59e9f77a8 100644 --- a/test/puppet-tool-call.test.ts +++ b/test/puppet-tool-call.test.ts @@ -109,7 +109,8 @@ test('puppetToolCall executes with agent provenance and stores the pair', async 'princess', 'mcpl--eido--look', {}, ); - assert.match(toolUseId, /^toolu_01[A-Za-z0-9]{22}$/, 'anthropic-shaped id'); + // null only for a change the operator-change gate staged; none here. + assert.match(toolUseId ?? '', /^toolu_01[A-Za-z0-9]{22}$/, 'anthropic-shaped id'); assert.equal(result.success, true); assert.equal(executed.length, 1); assert.equal(executed[0].callerAgentName, 'princess', 'executes AS the agent'); From b3410fd9f06fa6f18584d76b45447d2a2c3be178 Mon Sep 17 00:00:00 2001 From: Agnes-1009 Date: Wed, 7 Oct 2026 10:01:18 -0400 Subject: [PATCH 41/73] fix+feat(framework): self-change gate review fixes; admitted undo-turns as one cut (R2) Two parts: Kit's four findings on a79d7bb (room-222 #44862, /tmp/kit-1008-r2-probe.mjs), and host/command undo by turns under the gate (Kit's answer 3, #43896). Review fixes, all in the common self-change path, so direct, wrapped and delayed application agree: 1. Resolution failed open. Both catches fell through to the mutating handler. The resolver rejected context_budget_tokens "120000" while the handler's Number(...) accepted it, so a puppet changed the budget with zero gate calls. - parseAgentSettingsInput is now the one parser for the handler and for resolution: Number coercion, unknown update keys ignored, extension slices by owner, reset names and reset-all. runAgentSettingsToolCall is rewritten onto it, with identical outputs (the runtime-settings, guard and spill suites, 71/71, pass unchanged). - Resolution also runs Agent.checkRuntimeSettingsPatch, the new public form of the handler's own validation. - Any resolution failure now refuses the change without running it: no execution, no gate call, and for a puppet no pair (toolUseId null). This covers input the handler would reject, a failing extension reader, and an extension that can't preview. 2. Branch identity wasn't checked. revalidateSelfChange now requires the active branch to be the descriptor's sourceBranch, for settings and presentation alike. 3. An immediate apply ignored a held lease: a module's own callTool got `apply` and changed the budget while another runAtSafeBoundary callback held the store. - leaseScope (src/call-provenance.ts), an AsyncLocalStorage, is set around the lease callback. - storeHoldAgainstCaller names a store hold this call isn't running under, and every gated apply (immediate or admitted) refuses while one exists. - It refuses rather than waits, so a wrapped call never waits on its own outer turn or token. The lease holder's own work, including a module call it makes, still applies. 4. Extension resets had no resolved target. A staged reset captured `from {knob: 7}`, `target {}`, and a later default change applied 9 under the original admission. - AgentSettingsExtension gains an optional preview(agent, {update patch | reset keys}): the values its keys would have, normalized as update would store them, with a reset's concrete defaults. - The descriptor's extensions are now { from, target }, and revalidation compares both. - An extension without preview can't be resolved, so a gated operator or module change to it is refused. The agent's own calls are unaffected. - The framework's own extensions implement it: tool_result_inline_max_chars (validated and floored, reset to null) and tool_result_guard (reset to ToolResultGuard.configuredEnabled, a new getter). Undo by turns (ResolvedUndoTurnsChange; host/command with a gate): - Resolution: the exact checkpoints to undo, newest first, with requestedTurns (there may be fewer than requested). With nothing to undo, nothing is asked. - Staged: { ok:false, code:'staged', error: receipt text, staged:{id} }, and nothing changes. Gate failure: code 'gate-failed'. - Apply now: exactly that change, under reserveStoreForSurgery('undo'). That requires every agent sharing the store to be idle, as Ada's hide guard does. The awareness sync runs after, as host/command undo always has. Delayed: applyResolvedOperatorChange under the held lease; a sync failure there is logged (the outbox owns delivery). - applyUndoTurns is one absolute cut at the oldest staged checkpoint's sequenceBefore, onto `undo//op-`: - The cut drops the staged turns and everything after them, so a notice and answer turn added since don't make it stale. Their checkpoints go too. - It keeps the oldest undone turn's input, exactly as the per-turn loop does: a checkpoint is taken after the input landed. - One redo entry returns to the source tip with all of it. - Revalidation: the active branch must still be the source, and every staged checkpoint still present; otherwise 'stale'. - Crash recovery: nothing is written on the source, and every step is idempotent. Checkpoint lists are branch-scoped, so the destination saves the kept list and the untouched source tip keeps every checkpoint (the cut's redo entry re-adds none). On retry, the destination tells what happened: - active: the cut was already applied; reported as alreadyApplied, nothing changed; - created but never written (head at its branchPoint): finished; - written to (it records framework/state operatorCut {changeId}, which moves its head) but no longer active: stale, never re-applied, as after a redo. - RedoEntry now carries checkpoints[] (the ones redo re-adds): one for undoLastTurn, none for a cut. The undo:completed trace gains an optional `turns`, and the operator log gets 'undo-turns' with changeId, requested, resolved, turnIndexes and the admission. - Without a gate, host/command keeps its per-turn loop. Tests: test/operator-change-gate.test.ts, 29 cases (14 new). - Self-change: a numeric string staged; unresolvable input refused unasked via puppet and module; a mixed core+extension change resolved through each owner (normalized) and applied later; a failing reader and a preview-less extension refused; an extension reset stale once the default moved; a settings change stale once the branch moved (presentation too); a foreign lease's hold refused while the holder's own call applies. - Undo: staged as exact checkpoints with nothing changed; applied now as one cut with one redo entry; applied later past an answer turn, with redo restoring everything; an already-applied cut recognized and a redone-away one refused; an interrupted cut finished; stale on a moved branch and on a vanished checkpoint; fewer turns than requested; gate failure; nothing to undo. Mutation checks, twelve, each caught: - an unresolvable change run (puppet; wrapped); - no branch check; - apply ignoring a foreign hold; lease scope ignored; - an extension without preview allowed; extension targets not compared; - no already-applied recognition; a used destination re-applied; an interrupted cut not finished; - checkpoints not revalidated; redo re-adding the cut's checkpoints. Twenty affected suites: 300/300. mcpl-coalescing's undo case passes. Co-Authored-By: Agnes (Claude Opus 5.5) --- src/agent.ts | 5 + src/call-provenance.ts | 8 + src/framework.ts | 667 +++++++++++++++++++++--------- src/operator-change.ts | 32 +- src/tool-result-guard.ts | 2 + src/types/module.ts | 16 + src/types/trace.ts | 3 + test/operator-change-gate.test.ts | 320 +++++++++++++- 8 files changed, 845 insertions(+), 208 deletions(-) diff --git a/src/agent.ts b/src/agent.ts index d528ee13b..a32f1aa39 100644 --- a/src/agent.ts +++ b/src/agent.ts @@ -564,6 +564,11 @@ export class Agent { return target; } + /** Throw if updateRuntimeSettings would reject this patch; changes nothing. */ + checkRuntimeSettingsPatch(patch: AgentRuntimeSettingsPatch): void { + this.validateRuntimeSettingsPatch(patch); + } + private validateRuntimeSettingsPatch(patch: AgentRuntimeSettingsPatch): void { if (Object.keys(patch).length === 0) throw new Error('At least one setting is required'); if (patch.contextBudgetTokens !== undefined) { diff --git a/src/call-provenance.ts b/src/call-provenance.ts index 7d208e821..c9ac9b293 100644 --- a/src/call-provenance.ts +++ b/src/call-provenance.ts @@ -17,3 +17,11 @@ import type { ToolCall } from './types/index.js'; export type CallProvenance = Pick; export const callProvenance = new AsyncLocalStorage(); + +/** + * The lease whose callback (runAtSafeBoundary) the current work runs + * inside, carried across its asynchronous work. A module's own timer or an + * unrelated caller is outside it even while that lease holds the store, so + * a mutation can tell the lease holder's work from everyone else's. + */ +export const leaseScope = new AsyncLocalStorage(); diff --git a/src/framework.ts b/src/framework.ts index 020a89cc3..b2be74143 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -9,7 +9,7 @@ import { MembraneError } from '@animalabs/membrane'; import { ContextManager, PassthroughStrategy, WindowedPassthroughStrategy, OverBudgetError, UncoveredDropError } from '@animalabs/context-manager'; import type { CacheWireReceipt } from './kv-unified-wire.js'; import { SUBCONSCIOUS_TOOLS, SUBCONSCIOUS_TOOL_NAMES, type SubconsciousConfig } from './tune-out/tools.js'; -import { callProvenance } from './call-provenance.js'; +import { callProvenance, leaseScope } from './call-provenance.js'; import { selfChangeKind, sameValue, @@ -18,6 +18,7 @@ import { type ResolvedOperatorChange, type ResolvedPresentationChange, type ResolvedSettingsChange, + type ResolvedUndoTurnsChange, type RuntimeSettingsValues, } from './operator-change.js'; import { TuneOutCoordinator, TUNE_OUT_DEFAULTS } from './tune-out/coordinator.js'; @@ -502,6 +503,88 @@ const bySeq = (a: { seq: number }, b: { seq: number }): number => a.seq - b.seq; /** Stamp a deferred write's durable id into the metadata it is stored with * (boot recovery dedups replays by it). Idempotent for an already-stamped * message. */ +/** agent_settings input, parsed once for the handler and for operator-change + * resolution alike, so the two can never disagree about what it changes. */ +type ParsedAgentSettings = + | { action: 'get' } + | { action: 'cancel' } + | { + action: 'update'; + /** Core keys, coerced as the handler always has (Number(...)), plus `immediate`. */ + patch: AgentRuntimeSettingsPatch; + /** Each extension's slice of the input (only its keys). */ + extUpdates: Map>; + } + | { + action: 'reset'; + /** Core keys to reset; undefined with resetsAll, or when only extension keys are named. */ + keys?: Array; + resetsAll: boolean; + /** Extensions to reset, each with its keys (undefined: all of them, on reset-all). */ + extResets: Map; + }; + +const AGENT_SETTINGS_CORE_NAMES: Record = { + context_budget_tokens: 'contextBudgetTokens', + tail_tokens: 'tailTokens', + transition_pace_tokens: 'transitionPaceTokens', + same_round_think_text_policy: 'sameRoundThinkTextPolicy', +}; + +function parseAgentSettingsInput(input: Record, extensions: AgentSettingsExtension[]): ParsedAgentSettings { + switch (input.action) { + case 'get': + return { action: 'get' }; + case 'cancel': + return { action: 'cancel' }; + case 'update': { + const patch: AgentRuntimeSettingsPatch = {}; + if (input.context_budget_tokens !== undefined) patch.contextBudgetTokens = Number(input.context_budget_tokens); + if (input.tail_tokens !== undefined) patch.tailTokens = Number(input.tail_tokens); + if (input.transition_pace_tokens !== undefined) patch.transitionPaceTokens = Number(input.transition_pace_tokens); + if (input.same_round_think_text_policy !== undefined) { + patch.sameRoundThinkTextPolicy = input.same_round_think_text_policy as AgentRuntimeSettingsPatch['sameRoundThinkTextPolicy']; + } + // Immediate budget decrease: skip the paced descent; the next compile + // plans straight at the new budget (one-shot fold-down). + if (input.immediate !== undefined) patch.immediate = Boolean(input.immediate); + const extUpdates = new Map>(); + for (const ext of extensions) { + const slice: Record = {}; + for (const key of ext.keys) if (input[key] !== undefined) slice[key] = input[key]; + if (Object.keys(slice).length > 0) extUpdates.set(ext, slice); + } + return { action: 'update', patch, extUpdates }; + } + case 'reset': { + const extResets = new Map(); + if (input.settings === undefined) { + // Reset-all covers every extension that can reset. + for (const ext of extensions) if (ext.reset) extResets.set(ext, undefined); + return { action: 'reset', resetsAll: true, extResets }; + } + if (!Array.isArray(input.settings)) throw new Error('reset `settings` must be an array'); + const keys: Array = []; + for (const name of input.settings) { + if (typeof name === 'string' && AGENT_SETTINGS_CORE_NAMES[name]) { + keys.push(AGENT_SETTINGS_CORE_NAMES[name]); + continue; + } + const owner = extensions.find((ext) => typeof name === 'string' && ext.keys.includes(name)); + if (!owner) throw new Error(`Unknown reset setting: ${String(name)}`); + // An extension without reset semantics is skipped, as reset-all skips it. + if (!owner.reset) continue; + const list = extResets.get(owner) ?? []; + list.push(name as string); + extResets.set(owner, list); + } + return { action: 'reset', ...(keys.length > 0 ? { keys } : {}), resetsAll: false, extResets }; + } + default: + throw new Error('agent_settings: action must be get, update, reset, or cancel'); + } +} + function withDeferredWriteId(metadata: MessageMetadata | undefined, id: string): MessageMetadata { return { ...(metadata ?? {}), deferredWriteId: id } as MessageMetadata; } @@ -680,7 +763,11 @@ export class ResumeBlockedError extends Error { interface RedoEntry { branchName: string; - checkpoint: TurnCheckpoint; + /** Checkpoints redo re-adds after switching back, oldest first. A single + * undo saved its source without the checkpoint, so redo re-adds it; an + * admitted multi-turn cut writes nothing on its source, whose tip still + * holds them all, so it re-adds none. */ + checkpoints: TurnCheckpoint[]; } interface InferenceToolSnapshot { @@ -2778,6 +2865,20 @@ export class AgentFramework { } return { tool_result_inline_max_chars: null }; }, + preview: (agentName: string, change) => { + if (change.action === 'reset') { + return { + tool_result_inline_max_chars: !change.keys || change.keys.includes('tool_result_inline_max_chars') + ? null + : this.toolResultInlineMaxCharsOverride.get(agentName) ?? null, + }; + } + const n = Number(change.patch.tool_result_inline_max_chars); + if (!Number.isFinite(n) || n < 1000) { + throw new Error('tool_result_inline_max_chars must be a number >= 1000'); + } + return { tool_result_inline_max_chars: Math.floor(n) }; + }, }; } @@ -2824,6 +2925,13 @@ export class AgentFramework { return set(name, patch.tool_result_guard); }, reset: (name) => set(name, undefined), + preview: (name, change) => { + const guard = this.agents.get(name)?.toolResultGuard; + if (!guard) throw new Error(`Unknown agent: ${name}`); + if (change.action === 'reset') return { tool_result_guard: guard.configuredEnabled }; + if (typeof change.patch.tool_result_guard !== 'boolean') throw new Error('tool_result_guard must be a boolean'); + return { tool_result_guard: change.patch.tool_result_guard }; + }, }; } @@ -5276,8 +5384,12 @@ export class AgentFramework { ): Promise<{ ok: boolean; error?: string; - /** Refusal code from live surgery (e.g. 'agent-busy'), when applicable. */ + /** Refusal code from live surgery (e.g. 'agent-busy'), when applicable; + * 'staged' when the operator-change gate staged the change, with its + * receipt text as `error`; 'gate-failed' when the gate refused it. */ code?: string; + /** For a staged change: the host's receipt id. */ + staged?: { id: string }; undone?: number; requested?: number; /** For quiesce/resume/maintain/host-status: the host mode snapshot. */ @@ -5622,6 +5734,9 @@ export class AgentFramework { } const requested = Math.max(1, Math.min(20, Math.floor(params.turns ?? 1))); + if (this.operatorChangeGate) { + return this.gatedUndoTurns(agentName, requested, hostCommandRequester(serverId, params)); + } const turnMarks: DiscordAwarenessMarks = params.marks === 'addressed' || params.marks === 'all' ? { scope: params.marks } : 'none'; @@ -6073,7 +6188,7 @@ export class AgentFramework { redoStack = []; this.redoStacks.set(agentName, redoStack); } - redoStack.push({ branchName: currentBranch.name, checkpoint }); + redoStack.push({ branchName: currentBranch.name, checkpoints: [checkpoint] }); this.emitTrace({ type: 'undo:completed', @@ -6122,7 +6237,7 @@ export class AgentFramework { return { redone: false }; } - const { branchName, checkpoint } = redoStack.pop()!; + const { branchName, checkpoints: restored } = redoStack.pop()!; const currentBranch = this.store.currentBranch(); this.store.switchBranch(branchName); @@ -6133,9 +6248,9 @@ export class AgentFramework { (wsRedo as any).materializeMount('_config').catch(() => {}); } - // Restore the checkpoint + // Restore the checkpoints the undo removed const checkpoints = this.getTurnCheckpoints(agentName); - checkpoints.push(checkpoint); + checkpoints.push(...restored); this.saveTurnCheckpoints(agentName, checkpoints); this.emitTrace({ @@ -6373,7 +6488,8 @@ export class AgentFramework { `${opts.requester ? `, for ${opts.requester.name ?? opts.requester.id ?? 'operator'} via ${opts.requester.via}` : ''})`, ); try { - return await fn(held.lease); + // The callback, and the work it starts, runs inside this lease's scope. + return await leaseScope.run(held.lease, () => fn(held.lease)); } finally { held.release(); console.log(`[safe-boundary] ${held.lease.verb}: released after ${Date.now() - held.lease.since}ms`); @@ -6525,76 +6641,73 @@ export class AgentFramework { ): ResolvedSettingsChange | ResolvedPresentationChange | null { if (!admission) return null; const change = this.activeAdmissions?.get(admission.id); - if (!change || change.agent !== agentName) return null; + if (!change || change.agent !== agentName || change.kind === 'undo-turns') return null; const changeTool = change.kind === 'agent-settings' ? 'agent_settings' : change.tool; return tool === changeTool && sameValue(change.input, input) ? change : null; } - /** The core patch / reset keys and extension keys an agent_settings input touches. */ - private parseSettingsChange(input: Record): { - action: { action: 'update'; patch: AgentRuntimeSettingsPatch } | { action: 'reset'; keys?: Array } | { action: 'cancel' }; - extensionKeys: string[]; + /** + * What an agent_settings change would do, resolved without changing + * anything: the touched core settings now (`from`) and after (`target`), + * and the touched extension keys the same way, from each owner's preview. + * Shares the handler's parser, so it reads the input exactly as the + * handler would. Throws when the change can't be resolved: input the + * handler would reject, an extension that can't preview, a failing reader. + */ + private resolveSettingsEffect(agentName: string, input: Record): { + from: RuntimeSettingsValues; + target: RuntimeSettingsValues; + extensions?: { from: Record; target: Record }; } { - const extensions = [...this.collectAgentSettingsExtensions().values()]; - const extKeyOwner = (key: string) => extensions.find((ext) => ext.keys.includes(key)); - const core: Record = { - context_budget_tokens: 'contextBudgetTokens', - tail_tokens: 'tailTokens', - transition_pace_tokens: 'transitionPaceTokens', - same_round_think_text_policy: 'sameRoundThinkTextPolicy', - }; - if (input.action === 'cancel') return { action: { action: 'cancel' }, extensionKeys: [] }; - if (input.action === 'update') { - const patch: AgentRuntimeSettingsPatch = {}; - const extensionKeys: string[] = []; - for (const [key, value] of Object.entries(input)) { - if (key === 'action' || key === 'immediate' || value === undefined) continue; - const coreKey = core[key]; - if (coreKey === 'sameRoundThinkTextPolicy') { - if (typeof value !== 'string') throw new Error(`${key} must be a string`); - patch.sameRoundThinkTextPolicy = value as AgentRuntimeSettingsPatch['sameRoundThinkTextPolicy']; - } else if (coreKey) { - if (typeof value !== 'number' || !Number.isFinite(value)) throw new Error(`${key} must be a number`); - (patch as Record)[coreKey] = value; - } else if (extKeyOwner(key)) extensionKeys.push(key); - else throw new Error(`Unknown setting: ${key}`); - } - return { action: { action: 'update', patch }, extensionKeys }; - } - if (input.action === 'reset') { - if (input.settings === undefined) { - return { action: { action: 'reset' }, extensionKeys: extensions.flatMap((ext) => (ext.reset ? ext.keys : [])) }; + const agent = this.agents.get(agentName); + if (!agent) throw new Error(`Unknown agent: ${agentName}`); + const parsed = parseAgentSettingsInput(input, [...this.collectAgentSettingsExtensions().values()]); + if (parsed.action === 'get') throw new Error('agent_settings get changes nothing'); + let coreAction: Parameters[0] | null = null; + if (parsed.action === 'cancel') coreAction = { action: 'cancel' }; + else if (parsed.action === 'update') { + if (Object.keys(parsed.patch).length > 0) { + agent.checkRuntimeSettingsPatch(parsed.patch); + coreAction = { action: 'update', patch: parsed.patch }; } - if (!Array.isArray(input.settings)) throw new Error('reset `settings` must be an array'); - const keys: Array = []; - const extensionKeys: string[] = []; - for (const name of input.settings) { - if (typeof name === 'string' && core[name]) keys.push(core[name]); - else if (typeof name === 'string' && extKeyOwner(name)) extensionKeys.push(name); - else throw new Error(`Unknown reset setting: ${String(name)}`); + } else if (parsed.resetsAll || parsed.keys) { + coreAction = { action: 'reset', ...(parsed.keys ? { keys: parsed.keys } : {}) }; + } + const target = coreAction ? agent.previewRuntimeSettingsTarget(coreAction) : {}; + const snapshot = agent.getRuntimeSettings() as unknown as Record; + const from: Record = {}; + for (const key of Object.keys(target)) from[key] = snapshot[key] ?? null; + + const extFrom: Record = {}; + const extTarget: Record = {}; + const visit = ( + ext: AgentSettingsExtension, + change: { action: 'update'; patch: Record } | { action: 'reset'; keys?: string[] }, + keys: string[], + ) => { + if (!ext.preview) { + throw new Error( + `${keys.join(', ')} ${keys.length === 1 ? 'is' : 'are'} managed by a settings extension that ` + + `can't preview a change, so an operator change to ${keys.length === 1 ? 'it' : 'them'} can't be resolved`, + ); } - return { action: keys.length > 0 ? { action: 'reset', keys } : { action: 'reset', keys: [] }, extensionKeys }; - } - throw new Error('agent_settings: action must be get, update, reset, or cancel'); - } - - /** The current values of the given settings: core keys from the snapshot - * (unset reads null), extension keys from their owners. */ - private currentSettingsValues(agentName: string, coreKeys: string[], extensionKeys: string[]): { - core: RuntimeSettingsValues; - extensions: Record; - } { - const snapshot = this.getAgentRuntimeSettings(agentName) as unknown as Record; - const core: Record = {}; - for (const key of coreKeys) core[key] = snapshot[key] ?? null; - const extensions: Record = {}; - if (extensionKeys.length > 0) { - for (const ext of this.collectAgentSettingsExtensions().values()) { - const values = ext.get(agentName); - for (const key of extensionKeys) if (ext.keys.includes(key)) extensions[key] = values[key] ?? null; + const now = ext.get(agentName); + const next = ext.preview(agentName, change); + for (const key of keys) { + extFrom[key] = now[key] ?? null; + extTarget[key] = next[key] ?? null; } + }; + if (parsed.action === 'update') { + for (const [ext, slice] of parsed.extUpdates) visit(ext, { action: 'update', patch: slice }, Object.keys(slice)); + } else if (parsed.action === 'reset') { + for (const [ext, keys] of parsed.extResets) visit(ext, { action: 'reset', ...(keys ? { keys } : {}) }, keys ?? ext.keys); } - return { core: core as RuntimeSettingsValues, extensions }; + return { + from: from as RuntimeSettingsValues, + target, + ...(Object.keys(extTarget).length > 0 ? { extensions: { from: extFrom, target: extTarget } } : {}), + }; } /** Resolve an operator's or module's mutating self-change call to the @@ -6617,17 +6730,7 @@ export class AgentFramework { sourceBranch: this.store.currentBranch().name, }; if (tool === 'agent_settings') { - const { action, extensionKeys } = this.parseSettingsChange(input); - const target = agent.previewRuntimeSettingsTarget(action); - const current = this.currentSettingsValues(agentName, Object.keys(target), extensionKeys); - return { - ...base, - kind: 'agent-settings', - input: structuredClone(input), - from: current.core, - target, - ...(extensionKeys.length > 0 ? { extensions: current.extensions } : {}), - }; + return { ...base, kind: 'agent-settings', input: structuredClone(input), ...this.resolveSettingsEffect(agentName, input) }; } const name = (input as { name?: unknown }).name; const entry = typeof name === 'string' @@ -6649,18 +6752,27 @@ export class AgentFramework { throw new OperatorActionError('stale', `Change ${change.id} for ${change.agent} no longer holds: ${what}`); }; if (!this.agents.has(change.agent)) throw new OperatorActionError('unknown-agent', `Unknown agent: ${change.agent}`); + const branch = this.store.currentBranch().name; + if (branch !== change.sourceBranch) stale(`the active branch moved from ${change.sourceBranch} to ${branch}`); if (change.kind === 'agent-settings') { - const { action, extensionKeys } = this.parseSettingsChange(change.input); - const current = this.currentSettingsValues(change.agent, Object.keys(change.from), extensionKeys); - if (!sameValue(current.core, change.from)) { - stale(`its settings moved from ${JSON.stringify(change.from)} to ${JSON.stringify(current.core)}`); + let now: ReturnType; + try { + now = this.resolveSettingsEffect(change.agent, change.input); + } catch (error) { + stale(`it can no longer be resolved (${error instanceof Error ? error.message : String(error)})`); + } + if (!sameValue(now!.from, change.from)) { + stale(`its settings moved from ${JSON.stringify(change.from)} to ${JSON.stringify(now!.from)}`); } - if (change.extensions && !sameValue(current.extensions, change.extensions)) { - stale(`its host-managed settings moved from ${JSON.stringify(change.extensions)} to ${JSON.stringify(current.extensions)}`); + if (!sameValue(now!.target, change.target)) { + stale(`it would now set ${JSON.stringify(now!.target)}, not the approved ${JSON.stringify(change.target)}`); } - const target = this.agents.get(change.agent)!.previewRuntimeSettingsTarget(action); - if (!sameValue(target, change.target)) { - stale(`it would now set ${JSON.stringify(target)}, not the approved ${JSON.stringify(change.target)}`); + if (!sameValue(now!.extensions ?? null, change.extensions ?? null)) { + stale( + `its host-managed settings would now go from ${JSON.stringify(now!.extensions?.from ?? {})} ` + + `to ${JSON.stringify(now!.extensions?.target ?? {})}, not the approved ` + + `${JSON.stringify(change.extensions?.from ?? {})} to ${JSON.stringify(change.extensions?.target ?? {})}`, + ); } return; } @@ -6683,35 +6795,242 @@ export class AgentFramework { if (!this.operatorChangeGate || call.origin === undefined) return run(); if (!selfChangeKind(call.name, call.input)) return run(); const input = (call.input ?? {}) as Record; + const refused = (error: string): ToolResult => ({ success: false, isError: true, error }); const admitted = this.admittedChange(call.admission, agentName, call.name, input); + let change: ResolvedSettingsChange | ResolvedPresentationChange; if (admitted) { - // The last check before the change runs, with no await in between. + change = admitted; + } else { + // A change that can't be resolved is refused, never run ungated: the + // resolver shares the handler's parser, so whatever it can't resolve + // the gate could not have decided. try { - this.revalidateSelfChange(admitted); + change = this.resolveSelfChange(agentName, call.name, input, call.origin); } catch (error) { - return { success: false, isError: true, error: error instanceof Error ? error.message : String(error) }; + return refused(`Operator change refused: it couldn't be resolved (${error instanceof Error ? error.message : String(error)})`); } - return run(); - } - let change: ResolvedSettingsChange | ResolvedPresentationChange; + const answer = await this.askOperatorChangeGate(change); + if (answer.outcome === 'staged') { + return { success: false, isError: true, error: answer.receipt.text, data: { staged: answer.receipt } }; + } + if (answer.outcome === 'gate-failed') return refused(`Operator change refused: ${answer.error}`); + } + // Permission to apply isn't a safe boundary: while the store is held by + // a reservation this call isn't running under, refuse rather than wait + // (the holder may be waiting for this very call's turn). + const held = this.storeHoldAgainstCaller(); + if (held) return refused(`Operator change refused: the store is held by ${held}; retry when it completes`); + // The last check before the change runs, with no await in between. try { - change = this.resolveSelfChange(agentName, call.name, input, call.origin); - } catch { - return run(); // invalid input: the handler reports its own error + this.revalidateSelfChange(change); + } catch (error) { + return refused(error instanceof Error ? error.message : String(error)); } + return run(); + } + + /** The store hold this work isn't running under, described; null when the + * store is free, or held by the lease whose callback this work runs in. */ + private storeHoldAgainstCaller(): string | null { + if (!this.surgeryHold) return null; + const scoped = leaseScope.getStore(); + if (scoped && this.heldLease?.lease === scoped) return null; + return `${this.surgeryHold.verb} for ${this.surgeryHold.agentName}`; + } + + /** + * host/command undo by turns when the host has an operator-change gate: + * resolve the exact checkpoints, ask, and either return the receipt + * (staged, nothing changed) or apply exactly that change now as one cut, + * under a store reservation. Without a gate, host/command keeps its + * per-turn loop. + */ + private async gatedUndoTurns(agentName: string, requested: number, requester: OperatorRequester): Promise<{ + ok: boolean; + error?: string; + code?: string; + staged?: { id: string }; + undone?: number; + requested?: number; + lastVisible?: { participant?: string; role?: string; preview?: string } | null; + }> { + const agent = this.agents.get(agentName)!; + if (agent.state.status !== 'idle') { + return { ok: false, error: `Cannot undo while agent is ${agent.state.status}` }; + } + const change = this.resolveUndoTurns(agentName, requested, requester); + if (!change) return { ok: true, undone: 0, requested, lastVisible: null }; const answer = await this.askOperatorChangeGate(change); if (answer.outcome === 'staged') { - return { success: false, isError: true, error: answer.receipt.text, data: { staged: answer.receipt } }; + return { ok: false, code: 'staged', error: answer.receipt.text, staged: { id: answer.receipt.id }, requested }; } if (answer.outcome === 'gate-failed') { - return { success: false, isError: true, error: `Operator change refused: ${answer.error}` }; + return { ok: false, code: 'gate-failed', error: `Operator change refused: ${answer.error}`, requested }; } + let applied: ReturnType; try { - this.revalidateSelfChange(change); + const release = this.reserveStoreForSurgery('undo', agentName); + try { + applied = this.applyUndoTurns(change); + } finally { + release(); + } } catch (error) { - return { success: false, isError: true, error: error instanceof Error ? error.message : String(error) }; + return { + ok: false, + error: error instanceof Error ? error.message : String(error), + ...(error instanceof OperatorActionError ? { code: error.code } : {}), + requested, + }; + } + console.error( + `[host-command] undo agent=${agentName} requested=${requested} undone=${applied.undone} ` + + `as one cut (${change.id}) by=${requester.name ?? requester.id ?? requester.via}`, + ); + await this.syncDiscordAwarenessMarkers(); + return { ok: true, undone: applied.undone, requested, lastVisible: await this.lastVisiblePreview(agentName) }; + } + + /** Resolve host/command undo by turns to the exact checkpoints it would + * undo, newest first; null when there is nothing to undo. */ + private resolveUndoTurns(agentName: string, requested: number, requester?: OperatorRequester): ResolvedUndoTurnsChange | null { + const checkpoints = this.getTurnCheckpoints(agentName); + if (checkpoints.length === 0) return null; + const taken = checkpoints.slice(-requested).reverse(); + return { + id: randomUUID(), + agent: agentName, + surface: 'host-command', + ...(requester ? { requester } : {}), + resolvedAt: Date.now(), + sourceBranch: this.store.currentBranch().name, + kind: 'undo-turns', + requestedTurns: requested, + checkpoints: taken.map((c) => ({ turnIndex: c.turnIndex, sequenceBefore: c.sequenceBefore, branchName: c.branchName })), + }; + } + + /** The branch an admitted undo-turns change cuts to: named for the change, + * so a recovering host can recognize a cut that already happened. */ + private undoTurnsDestination(change: ResolvedUndoTurnsChange): string { + return `undo/${change.agent}/op-${change.id}`; + } + + /** + * Apply a resolved undo-turns change as one absolute cut. The caller holds + * the store (a lease, or a surgery reservation). Revalidates against + * stable identity: the active branch must still be the source branch, and + * every staged checkpoint must still be in the agent's list (appending a + * notice or an answer changes neither). Then it branches at the oldest + * staged checkpoint's sequenceBefore, which drops the staged turns and + * everything after them, and removes those checkpoints and any newer + * ones. One redo entry restores the source tip and all of them. If the + * active branch is already this change's destination, the cut happened + * before (a host retrying after a crash) and nothing changes. + */ + private applyUndoTurns(change: ResolvedUndoTurnsChange, admission?: OperatorAdmission): { + requested: number; + undone: number; + fromBranch: string; + toBranch: string; + alreadyApplied?: true; + } { + const stale = (what: string): never => { + throw new OperatorActionError('stale', `Undo ${change.id} for ${change.agent} no longer holds: ${what}`); + }; + if (!this.agents.has(change.agent)) throw new OperatorActionError('unknown-agent', `Unknown agent: ${change.agent}`); + const destination = this.undoTurnsDestination(change); + const undone = change.checkpoints.length; + const current = this.store.currentBranch().name; + if (current === destination) { + return { requested: change.requestedTurns, undone, fromBranch: change.sourceBranch, toBranch: destination, alreadyApplied: true }; + } + // The destination exists if an earlier attempt got as far as creating + // it. Never written to (its head still at its branch point), that + // attempt was interrupted before switching, and this one finishes it. + // Written to, the cut was applied and the active branch has left it + // since (a redo, a later undo): refused, never re-applied. + const dest = this.store.listBranches().find((b) => b.name === destination); + const created = dest !== undefined; + if (dest && dest.head !== (dest.branchPoint ?? dest.head)) { + stale(`its cut was already applied as ${destination}, and the active branch has moved since to ${current}`); + } + if (current !== change.sourceBranch) stale(`the active branch moved from ${change.sourceBranch} to ${current}`); + const list = this.getTurnCheckpoints(change.agent); + const at = (c: ResolvedUndoTurnsChange['checkpoints'][number]) => list.findIndex((k) => + k.turnIndex === c.turnIndex && k.sequenceBefore === c.sequenceBefore && k.branchName === c.branchName); + for (const c of change.checkpoints) { + if (at(c) < 0) stale(`turn ${c.turnIndex} is no longer an undoable checkpoint`); + } + const oldest = change.checkpoints[change.checkpoints.length - 1]!; + const cutAt = at(oldest); + const removed = list.slice(cutAt); + + // Checkpoint lists are branch-scoped state, and nothing is written on the + // source: the destination, cut before the oldest staged turn recorded + // itself, inherits the older checkpoints (saved there explicitly), and + // the source tip keeps every checkpoint for redo to return to. So each + // step is idempotent and a retry after a crash finishes the cut. + const kept = list.slice(0, cutAt); + if (!created) this.store.createBranchAt(destination, change.sourceBranch, oldest.sequenceBefore); + this.store.switchBranch(destination); + // Mark the destination as used, naming the change that made it: this + // write is what moves its head off its branch point (see above), so a + // later retry can tell a completed cut from an interrupted one. + const framework = this.store.getStateJson(FRAMEWORK_STATE_ID); + this.store.setStateJson(FRAMEWORK_STATE_ID, { + ...(framework && typeof framework === 'object' ? framework as Record : {}), + operatorCut: { changeId: change.id, at: Date.now() }, + }); + this.saveTurnCheckpoints(change.agent, kept); + const ws = this.moduleRegistry.getModule('workspace'); + if (ws && 'materializeMount' in ws) { + (ws as unknown as { materializeMount(name: string): Promise }).materializeMount('_config').catch(() => {}); + } + let redoStack = this.redoStacks.get(change.agent); + if (!redoStack) { + redoStack = []; + this.redoStacks.set(change.agent, redoStack); + } + // The source tip still holds the removed checkpoints: redo re-adds none. + redoStack.push({ branchName: change.sourceBranch, checkpoints: [] }); + + this.emitTrace({ + type: 'undo:completed', + agentName: change.agent, + turnIndex: oldest.turnIndex, + fromBranch: change.sourceBranch, + toBranch: destination, + turns: undone, + }); + this.recordOperatorAction({ + kind: 'undo-turns', + agent: change.agent, + ...(change.requester ? { requester: change.requester } : {}), + params: { + changeId: change.id, + requested: change.requestedTurns, + resolved: undone, + turnIndexes: change.checkpoints.map((c) => c.turnIndex), + ...(admission ? { admission: admission.id } : {}), + }, + result: { sourceBranch: change.sourceBranch, targetBranch: destination, checkpointsRemoved: removed.length }, + }); + return { requested: change.requestedTurns, undone, fromBranch: change.sourceBranch, toBranch: destination }; + } + + /** After an admitted undo: reconcile awareness markers as host/command + * undo always has. The cut stands either way; the durable outbox owns + * eventual delivery, so a sync failure is logged, not thrown. */ + private async syncAwarenessAfterUndo(agentName: string): Promise { + try { + await this.syncDiscordAwarenessMarkers(); + } catch (error) { + console.error( + `[operator-change] awareness sync after undo for ${agentName} failed; the outbox retries: ` + + `${error instanceof Error ? error.message : String(error)}`, + ); } - return run(); } /** @@ -6729,6 +7048,11 @@ export class AgentFramework { opts: { lease: SafeBoundaryLease; admission: OperatorAdmission }, ): Promise { this.heldLeaseTokens(opts.lease, `apply ${change.kind} for ${change.agent}`); + if (change.kind === 'undo-turns') { + const applied = this.applyUndoTurns(change, opts.admission); + if (!applied.alreadyApplied) await this.syncAwarenessAfterUndo(change.agent); + return { kind: 'undo-turns', ...applied }; + } this.revalidateSelfChange(change); const tool = change.kind === 'agent-settings' ? 'agent_settings' : change.tool; this.activeAdmissions.set(opts.admission.id, change); @@ -12149,32 +12473,40 @@ export class AgentFramework { selfChangeKind(toolName, input) && !this.admittedChange(opts?.admission, agentName, toolName, input) ) { - let change: ResolvedSettingsChange | ResolvedPresentationChange | undefined; + let change: ResolvedSettingsChange | ResolvedPresentationChange; try { change = this.resolveSelfChange(agentName, toolName, input, 'puppet', opts?.requester); - } catch { - change = undefined; // invalid input: the tool refuses it itself, as before + } catch (error) { + // Refused, never run ungated, and no pair: the tool didn't run. + return { + toolUseId: null, + result: { + success: false, + isError: true, + error: `Operator change refused: it couldn't be resolved (${error instanceof Error ? error.message : String(error)})`, + }, + }; } - if (change) { - const answer = await this.askOperatorChangeGate(change); - if (answer.outcome === 'staged') { - return { - toolUseId: null, - result: { success: false, isError: true, error: answer.receipt.text }, - staged: answer.receipt, - }; - } - if (answer.outcome === 'gate-failed') { - throw new Error(`puppet refused: the operator-change gate failed (${answer.error})`); - } - this.revalidateSelfChange(change); - const admission: OperatorAdmission = { id: change.id }; - this.activeAdmissions.set(admission.id, change); - try { - return await this.puppetToolCallUngated(agentName, toolName, input, { ...opts, admission }); - } finally { - this.activeAdmissions.delete(admission.id); - } + const answer = await this.askOperatorChangeGate(change); + if (answer.outcome === 'staged') { + return { + toolUseId: null, + result: { success: false, isError: true, error: answer.receipt.text }, + staged: answer.receipt, + }; + } + if (answer.outcome === 'gate-failed') { + throw new Error(`puppet refused: the operator-change gate failed (${answer.error})`); + } + // Revalidated here and again where the change runs (gatedSelfChange's + // admitted path), right before the handler, with no await between. + this.revalidateSelfChange(change); + const admission: OperatorAdmission = { id: change.id }; + this.activeAdmissions.set(admission.id, change); + try { + return await this.puppetToolCallUngated(agentName, toolName, input, { ...opts, admission }); + } finally { + this.activeAdmissions.delete(admission.id); } } return this.puppetToolCallUngated(agentName, toolName, input, opts); @@ -16905,15 +17237,7 @@ export class AgentFramework { }); let result: ToolResult; try { - const input = (call.input ?? {}) as { - action?: unknown; - context_budget_tokens?: unknown; - tail_tokens?: unknown; - transition_pace_tokens?: unknown; - same_round_think_text_policy?: unknown; - immediate?: unknown; - settings?: unknown; - } & Record; + const input = (call.input ?? {}) as Record; const extensions = this.collectAgentSettingsExtensions(); /** Extension values merged flat alongside the core snapshot. */ const extGet = (): Record => { @@ -16921,7 +17245,8 @@ export class AgentFramework { for (const ext of extensions.values()) Object.assign(out, ext.get(agentName)); return out; }; - switch (input.action) { + const parsed = parseAgentSettingsInput(input, [...extensions.values()]); + switch (parsed.action) { case 'get': result = { success: true, @@ -16932,32 +17257,13 @@ export class AgentFramework { result = { success: true, data: this.cancelAgentRuntimeSettingsTransition(agentName) }; break; case 'update': { - const patch: AgentRuntimeSettingsPatch = {}; - if (input.context_budget_tokens !== undefined) { - patch.contextBudgetTokens = Number(input.context_budget_tokens); - } - if (input.tail_tokens !== undefined) patch.tailTokens = Number(input.tail_tokens); - if (input.transition_pace_tokens !== undefined) { - patch.transitionPaceTokens = Number(input.transition_pace_tokens); - } - if (input.same_round_think_text_policy !== undefined) { - patch.sameRoundThinkTextPolicy = input.same_round_think_text_policy as AgentRuntimeSettingsPatch['sameRoundThinkTextPolicy']; - } - // Immediate budget decrease: skip the paced descent; the next - // compile plans straight at the new budget (one-shot fold-down). - if (input.immediate !== undefined) patch.immediate = Boolean(input.immediate); + const { patch } = parsed; // Route extension-owned keys to their modules; apply the core patch // only when it touches core keys (an extension-only update must not // disturb a converging budget transition). const extResults: Record = {}; - for (const ext of extensions.values()) { - const slice: Record = {}; - for (const key of ext.keys) { - if (input[key] !== undefined) slice[key] = input[key]; - } - if (Object.keys(slice).length > 0) { - Object.assign(extResults, ext.update(agentName, slice)); - } + for (const [ext, slice] of parsed.extUpdates) { + Object.assign(extResults, ext.update(agentName, slice)); } const coreTouched = Object.keys(patch).length > 0; const core = coreTouched @@ -16980,47 +17286,14 @@ export class AgentFramework { break; } case 'reset': { - let keys: Array | undefined; - const extResetKeys = new Map(); - const resetsAllSettings = input.settings === undefined; - if (input.settings !== undefined) { - if (!Array.isArray(input.settings)) throw new Error('reset `settings` must be an array'); - const names: Record = { - context_budget_tokens: 'contextBudgetTokens', - tail_tokens: 'tailTokens', - transition_pace_tokens: 'transitionPaceTokens', - same_round_think_text_policy: 'sameRoundThinkTextPolicy', - }; - keys = []; - for (const name of input.settings) { - if (typeof name === 'string' && names[name]) { - keys.push(names[name]); - continue; - } - const owner = [...extensions.values()].find( - (ext) => typeof name === 'string' && ext.keys.includes(name), - ); - if (!owner) throw new Error(`Unknown reset setting: ${String(name)}`); - const list = extResetKeys.get(owner) ?? []; - list.push(name as string); - extResetKeys.set(owner, list); - } - if (keys.length === 0) keys = undefined; - } + const { keys, resetsAll } = parsed; const touchedSameRoundThinkTextPolicy = - resetsAllSettings || keys?.includes('sameRoundThinkTextPolicy') === true; + resetsAll || keys?.includes('sameRoundThinkTextPolicy') === true; const extResults: Record = {}; - if (input.settings === undefined) { - // Reset-all covers extensions too. - for (const ext of extensions.values()) { - if (ext.reset) Object.assign(extResults, ext.reset(agentName)); - } - } else { - for (const [ext, list] of extResetKeys) { - if (ext.reset) Object.assign(extResults, ext.reset(agentName, list)); - } + for (const [ext, list] of parsed.extResets) { + Object.assign(extResults, list === undefined ? ext.reset!(agentName) : ext.reset!(agentName, list)); } - const coreTouched = input.settings === undefined || keys !== undefined; + const coreTouched = resetsAll || keys !== undefined; const core = coreTouched ? this.resetAgentRuntimeSettings(agentName, keys) : this.getAgentRuntimeSettings(agentName); @@ -17040,8 +17313,6 @@ export class AgentFramework { }; break; } - default: - throw new Error('agent_settings: action must be get, update, reset, or cancel'); } } catch (error) { result = { diff --git a/src/operator-change.ts b/src/operator-change.ts index aa65d026b..db099b563 100644 --- a/src/operator-change.ts +++ b/src/operator-change.ts @@ -54,9 +54,10 @@ export interface ResolvedSettingsChange extends ResolvedOperatorChangeBase { /** The concrete values the action leaves them at (Agent.previewRuntimeSettingsTarget). * Application applies the action only if it still produces exactly these. */ target: RuntimeSettingsValues; - /** Host-managed (extension) keys the input touches, with their values when - * resolved. Their owners apply them; application checks they haven't moved. */ - extensions?: Record; + /** Host-managed (extension) keys the input touches: their values when + * resolved, and the values the change leaves them at, from each owner's + * preview (AgentSettingsExtension.preview). Application checks both. */ + extensions?: { from: Record; target: Record }; } /** set_tool_visibility / set_tool_description, from an operator or a module. */ @@ -68,14 +69,35 @@ export interface ResolvedPresentationChange extends ResolvedOperatorChangeBase { from: { name: string; visible: boolean; description: string }; } +/** host/command undo by turns: the exact turn checkpoints it undoes. */ +export interface ResolvedUndoTurnsChange extends ResolvedOperatorChangeBase { + kind: 'undo-turns'; + /** Turns asked for. `checkpoints` holds fewer when history is shorter. */ + requestedTurns: number; + /** The checkpoints it undoes, newest first. Application makes one cut at + * the oldest one's sequenceBefore, onto a branch named for this change. */ + checkpoints: Array<{ turnIndex: number; sequenceBefore: number; branchName: string }>; +} + export type ResolvedOperatorChange = | ResolvedSettingsChange - | ResolvedPresentationChange; + | ResolvedPresentationChange + | ResolvedUndoTurnsChange; /** What applyResolvedOperatorChange did. */ export type AppliedOperatorChange = | { kind: 'agent-settings'; result: unknown } - | { kind: 'tool-presentation'; result: unknown }; + | { kind: 'tool-presentation'; result: unknown } + | { + kind: 'undo-turns'; + requested: number; + undone: number; + fromBranch: string; + toBranch: string; + /** The cut had already been applied (the active branch is its + * destination): nothing changed this time. */ + alreadyApplied?: true; + }; /** The self-change kind a tool call is, when it mutates the agent's own body: * agent_settings update/reset/cancel, or a tool-presentation edit. */ diff --git a/src/tool-result-guard.ts b/src/tool-result-guard.ts index 33c9d17a1..bf3f3f2fd 100644 --- a/src/tool-result-guard.ts +++ b/src/tool-result-guard.ts @@ -53,6 +53,8 @@ export class ToolResultGuard { get enabled(): boolean { return this.override ?? this.configured; } setOverride(value: boolean | undefined): void { this.override = value; } get settingOverride(): boolean | undefined { return this.override; } + /** The recipe's value: what a reset of the setting restores. */ + get configuredEnabled(): boolean { return this.configured; } get hasPending(): boolean { return this.pending !== undefined; } /** True only once the pending batch's originals were actually put on the * wire. Guard effects (retry suppression, refusal claim, prose buffering) diff --git a/src/types/module.ts b/src/types/module.ts index 007e43398..71c99b7df 100644 --- a/src/types/module.ts +++ b/src/types/module.ts @@ -154,6 +154,22 @@ export interface AgentSettingsExtension { /** Restore the listed keys (or all when omitted) to defaults and return the * new values. Optional — extensions without reset semantics are skipped. */ reset?(agentName: string, keys?: string[]): Record; + /** + * The values this extension's keys would have after an update (`patch` + * holds only its keys) or a reset (`keys`, or all when omitted), computed + * without changing anything: normalized as update would store them, and a + * reset's concrete defaults. Throw for input update would reject. + * + * An operator-gated change (FrameworkConfig.operatorChangeGate) approves + * these values, and the framework refuses to apply it once they no longer + * hold. Without preview, an operator or module change to these keys can't + * be resolved and is refused while a gate is configured; the agent's own + * calls are unaffected. + */ + preview?( + agentName: string, + change: { action: 'update'; patch: Record } | { action: 'reset'; keys?: string[] }, + ): Record; } /** diff --git a/src/types/trace.ts b/src/types/trace.ts index bf7b2715d..03e7df980 100644 --- a/src/types/trace.ts +++ b/src/types/trace.ts @@ -272,9 +272,12 @@ export type TraceEvent = | (TraceEventBase & { type: 'undo:completed'; agentName: string; + /** The (oldest) turn undone. */ turnIndex: number; fromBranch: string; toBranch: string; + /** Turns undone by one cut (an admitted multi-turn undo); absent for one. */ + turns?: number; }) | (TraceEventBase & { type: 'redo:completed'; diff --git a/test/operator-change-gate.test.ts b/test/operator-change-gate.test.ts index 773dbaf31..6ced82066 100644 --- a/test/operator-change-gate.test.ts +++ b/test/operator-change-gate.test.ts @@ -17,6 +17,7 @@ import { mkdtempSync, rmSync } from 'node:fs'; import { join } from 'node:path'; import { AgentFramework, AutobiographicalStrategy, WorkspaceModule } from '../src/index.js'; import type { + AgentSettingsExtension, InferenceRequest, Module, ModuleContext, @@ -44,6 +45,37 @@ class Mover implements Module { async onProcess(): Promise> { return {}; } } +/** A synthetic host settings extension: `knob`, with a configurable default. */ +class Knobs implements Module { + readonly name = 'knobs'; + value = 7; + defaultValue = 3; + failGet = false; + canPreview = true; + async start(): Promise {} + async stop(): Promise {} + getTools(): ToolDefinition[] { return []; } + async handleToolCall(): Promise { return { success: true }; } + async onProcess(): Promise> { return {}; } + getAgentSettingsExtension(): AgentSettingsExtension { + const read = () => { + if (this.failGet) throw new Error('knob store unavailable'); + return { knob: this.value }; + }; + return { + properties: { knob: { type: 'number' } }, + keys: ['knob'], + get: read, + update: (_agent, patch) => { this.value = Number(patch.knob); return read(); }, + reset: () => { this.value = this.defaultValue; return read(); }, + ...(this.canPreview ? { + preview: (_agent: string, change: { action: 'update'; patch: Record } | { action: 'reset'; keys?: string[] }) => + change.action === 'update' ? { knob: Number(change.patch.knob) } : { knob: this.defaultValue }, + } : {}), + }; + } +} + type Internals = { pendingRequests: InferenceRequest[]; activeAdmissions: Map }; describe('operator-change gate: self-change tools', () => { @@ -51,6 +83,7 @@ describe('operator-change gate: self-change tools', () => { let membrane: MockMembrane; let framework: AgentFramework; let mover: Mover; + let knobs: Knobs; let asked: ResolvedOperatorChange[]; let decide: (change: ResolvedOperatorChange) => Promise; let quiet: typeof console.log; @@ -62,13 +95,14 @@ describe('operator-change gate: self-change tools', () => { tempDir = mkdtempSync(join(tmpdir(), 'operator-gate-')); membrane = new MockMembrane(); mover = new Mover(); + knobs = new Knobs(); asked = []; decide = staged; framework = await AgentFramework.create({ storePath: join(tempDir, 'test.chronicle'), membrane: membrane.asMembrane(), agents: [{ name: 'scout', model: 'test-model', systemPrompt: 'You are scout.', maxTokens: 1000 }], - modules: [mover], + modules: [mover, knobs], operatorChangeGate: async (change) => { asked.push(structuredClone(change)); return decide(change); }, }); quiet = console.log; @@ -149,10 +183,99 @@ describe('operator-change gate: self-change tools', () => { assert.deepEqual(asked.map((c) => c.surface), ['puppet', 'host']); }); - it("lets invalid input through ungated, to the tool's own refusal", async () => { - const out = await framework.puppetToolCall('scout', 'agent_settings', { action: 'update', context_budget_tokens: 'lots' }); - assert.equal(asked.length, 0, 'nothing to resolve, so nothing staged'); - assert.equal(out.result.isError, true); + it('reads input exactly as the handler does: a numeric string is a change, and it is staged', async () => { + const before = budget(); + const out = await framework.puppetToolCall('scout', 'agent_settings', { action: 'update', context_budget_tokens: '120000' }); + assert.equal(out.toolUseId, null, 'staged, not run'); + assert.equal(budget(), before); + assert.deepEqual(asked[0]!.kind === 'agent-settings' && asked[0]!.target, { contextBudgetTokens: 120_000 }); + }); + + it("refuses a change it can't resolve, without running it or asking", async () => { + const before = budget(); + const viaPuppet = await framework.puppetToolCall('scout', 'agent_settings', { action: 'update', context_budget_tokens: 'lots' }); + assert.equal(viaPuppet.toolUseId, null, 'no pair: nothing ran'); + assert.match(String(viaPuppet.result.error), /couldn't be resolved/); + const viaModule = await mover.ctx.callTool!({ id: 'bad', name: 'agent_settings', input: { action: 'update', context_budget_tokens: 'lots' }, callerAgentName: 'scout' }); + assert.match(String(viaModule.error), /couldn't be resolved/); + assert.equal(asked.length, 0); + assert.equal(budget(), before); + assert.equal(pairs(), 0); + }); + + it('resolves a mixed core and extension change through each owner, and applies both later', async () => { + await framework.puppetToolCall('scout', 'agent_settings', { + action: 'update', context_budget_tokens: 120_000, tool_result_inline_max_chars: 4321.7, knob: 11, + }); + const change = asked[0]!; + assert.deepEqual(change.kind === 'agent-settings' && change.extensions, { + from: { tool_result_inline_max_chars: null, knob: 7 }, + target: { tool_result_inline_max_chars: 4321, knob: 11 }, + }, 'normalized by the owner'); + await framework.runAtSafeBoundary({ verb: 'apply' }, (lease) => + framework.applyResolvedOperatorChange(change, { lease, admission: { id: 'rev-1' } })); + assert.equal(budget(), 120_000); + assert.equal(knobs.value, 11); + }); + + it("refuses a change to an extension whose reader fails, or that can't preview", async () => { + knobs.failGet = true; + const failing = await framework.puppetToolCall('scout', 'agent_settings', { action: 'update', knob: 11 }); + assert.match(String(failing.result.error), /couldn't be resolved \(knob store unavailable\)/); + knobs.failGet = false; + knobs.canPreview = false; + const opaque = await framework.puppetToolCall('scout', 'agent_settings', { action: 'reset', settings: ['knob'] }); + assert.match(String(opaque.result.error), /can't preview a change/); + assert.equal(asked.length, 0); + assert.equal(knobs.value, 7, 'nothing changed'); + }); + + it("refuses an extension reset once the owner's default moved, rather than replaying it", async () => { + await framework.puppetToolCall('scout', 'agent_settings', { action: 'reset', settings: ['knob'] }); + const change = asked[0]!; + assert.deepEqual(change.kind === 'agent-settings' && change.extensions, { from: { knob: 7 }, target: { knob: 3 } }); + knobs.defaultValue = 9; // the current value stays 7 + await assert.rejects( + framework.runAtSafeBoundary({ verb: 'apply' }, (lease) => + framework.applyResolvedOperatorChange(change, { lease, admission: { id: 'rev-1' } })), + (e: Error & { code?: string }) => e.code === 'stale' && /host-managed settings/.test(e.message), + ); + assert.equal(knobs.value, 7); + }); + + it('refuses a staged change once the active branch moved, even with the settings unchanged', async () => { + await framework.puppetToolCall('scout', 'agent_settings', { action: 'update', context_budget_tokens: 120_000 }); + const change = asked[0]!; + const store = framework.getStore(); + store.createBranch('changed'); + store.switchBranch('changed'); + await assert.rejects( + framework.runAtSafeBoundary({ verb: 'apply' }, (lease) => + framework.applyResolvedOperatorChange(change, { lease, admission: { id: 'rev-1' } })), + (e: Error & { code?: string }) => e.code === 'stale' && /active branch moved/.test(e.message), + ); + }); + + it("refuses to apply while the store is held by a lease it isn't running under, and applies inside one", async () => { + decide = async () => ({ decision: 'apply' }); + const before = budget(); + let releaseLease!: () => void; + const holding = new Promise((r) => { releaseLease = r; }); + let granted!: () => void; + const isGranted = new Promise((r) => { granted = r; }); + const leased = framework.runAtSafeBoundary({ verb: 'held elsewhere' }, async () => { granted(); await holding; }); + await isGranted; + const outside = await mover.ctx.callTool!({ id: 'own', name: 'agent_settings', input: { action: 'update', context_budget_tokens: 150_000 }, callerAgentName: 'scout' }); + assert.match(String(outside.error), /the store is held by held elsewhere/); + assert.equal(budget(), before, 'permission to apply is not a safe boundary'); + releaseLease(); + await leased; + + await framework.runAtSafeBoundary({ verb: 'mine' }, async () => { + const inside = await mover.ctx.callTool!({ id: 'own2', name: 'agent_settings', input: { action: 'update', context_budget_tokens: 150_000 }, callerAgentName: 'scout' }); + assert.equal(inside.success, true, String(inside.error)); + }); + assert.equal(budget(), 150_000, 'the lease holder can apply under its own lease'); }); it('admits only the exact call its admission was registered for, and revalidates it as it runs', async () => { @@ -285,6 +408,16 @@ describe('operator-change gate: tool presentation', () => { assert.equal(change.kind, 'tool-presentation'); assert.deepEqual(change.kind === 'tool-presentation' && { name: change.from.name, visible: change.from.visible }, { name: 'workspace--glob', visible: true }); + const store = framework.getStore(); + const main = store.currentBranch().name; + store.createBranch('elsewhere'); + store.switchBranch('elsewhere'); + await assert.rejects( + framework.runAtSafeBoundary({ verb: 'apply rev-p' }, (lease) => + framework.applyResolvedOperatorChange(change, { lease, admission: { id: 'rev-p' } })), + (e: Error & { code?: string }) => e.code === 'stale' && /active branch moved/.test(e.message), + ); + store.switchBranch(main); await framework.runAtSafeBoundary({ verb: 'apply rev-p' }, (lease) => framework.applyResolvedOperatorChange(change, { lease, admission: { id: 'rev-p' } })); assert.equal(visible(), false, 'applied under the lease'); @@ -296,3 +429,180 @@ describe('operator-change gate: tool presentation', () => { } }); }); + +describe('operator-change gate: host/command undo by turns', () => { + let tempDir: string; + let membrane: MockMembrane; + let framework: AgentFramework; + let asked: ResolvedOperatorChange[]; + let decide: (change: ResolvedOperatorChange) => Promise; + let quiet: { log: typeof console.log; error: typeof console.error }; + type HostInternals = { + pendingRequests: InferenceRequest[]; + redoStacks: Map>; + getTurnCheckpoints(agent: string): Array<{ turnIndex: number }>; + saveTurnCheckpoints(agent: string, list: unknown[]): void; + handleHostCommand(serverId: string, params: Record): Promise>; + addMessage(participant: string, content: unknown[]): string; + }; + const host = () => framework as unknown as HostInternals; + + beforeEach(async () => { + tempDir = mkdtempSync(join(tmpdir(), 'operator-gate-undo-')); + membrane = new MockMembrane(); + asked = []; + decide = async (change) => ({ decision: 'staged', receipt: { id: 'rev-u', text: `staged ${change.kind}` } }); + framework = await AgentFramework.create({ + storePath: join(tempDir, 'test.chronicle'), + membrane: membrane.asMembrane(), + agents: [{ name: 'scout', model: 'test-model', systemPrompt: 'You are scout.', maxTokens: 1000 }], + modules: [], + operatorChangeGate: async (change) => { asked.push(structuredClone(change)); return decide(change); }, + }); + quiet = { log: console.log, error: console.error }; + console.log = () => {}; + console.error = () => {}; + }); + afterEach(async () => { + console.log = quiet.log; + console.error = quiet.error; + await framework.stop(); + rmSync(tempDir, { recursive: true, force: true }); + }); + + const turn = async (text: string) => { + host().addMessage('user', [{ type: 'text', text }]); + membrane.pushResponse(createMockResponse([{ type: 'text', text: `reply to ${text}` }])); + host().pendingRequests.push({ agentName: 'scout', reason: 'mcpl:channel-incoming', source: 'test', timestamp: Date.now() } as InferenceRequest); + await framework.runUntilIdle(); + }; + const texts = () => (framework.getAgent('scout')!.getContextManager().getAllMessages() as Array<{ content: Array<{ text?: string }> }>) + .map((m) => m.content[0]?.text ?? ''); + const branch = () => framework.getStore().currentBranch().name; + const undo = (turns: number) => host().handleHostCommand('discord', { command: 'undo', agentName: 'scout', turns, requesterName: 'nissa' }); + + it('stages an undo as the exact checkpoints it would undo, changing nothing', async () => { + await turn('one'); await turn('two'); await turn('three'); + const before = { branch: branch(), texts: texts(), checkpoints: host().getTurnCheckpoints('scout').length }; + const r = await undo(2); + assert.equal(r.ok, false); + assert.equal(r.code, 'staged'); + assert.deepEqual(r.staged, { id: 'rev-u' }); + assert.deepEqual({ branch: branch(), texts: texts(), checkpoints: host().getTurnCheckpoints('scout').length }, before); + const change = asked[0]!; + assert.equal(change.kind, 'undo-turns'); + assert.equal(change.kind === 'undo-turns' && change.requestedTurns, 2); + assert.deepEqual(change.kind === 'undo-turns' && change.checkpoints.map((c) => c.turnIndex), [2, 1], 'newest first'); + assert.equal(change.sourceBranch, before.branch); + }); + + it('applies an admitted multi-turn undo as one cut with one redo entry', async () => { + decide = async () => ({ decision: 'apply' }); + await turn('one'); await turn('two'); await turn('three'); + const source = branch(); + const r = await undo(2); + assert.equal(r.ok, true, String(r.error)); + assert.deepEqual([r.undone, r.requested], [2, 2]); + assert.equal(branch(), `undo/scout/op-${asked[0]!.id}`, 'the destination is named for the change'); + // A turn's checkpoint is taken after its input landed, so undoing turns + // keeps the oldest undone turn's input, exactly as the per-turn loop does. + assert.deepEqual(texts(), ['one', 'reply to one', 'two'], 'turns two and three are undone'); + assert.equal(host().getTurnCheckpoints('scout').length, 1); + assert.equal(host().redoStacks.get('scout')!.length, 1, 'one redo entry'); + + const redone = framework.redo('scout'); + assert.equal(redone.toBranch, source, 'one redo restores the source tip'); + assert.deepEqual(texts(), ['one', 'reply to one', 'two', 'reply to two', 'three', 'reply to three']); + assert.equal(host().getTurnCheckpoints('scout').length, 3, 'and every checkpoint the cut removed'); + }); + + it('applies a staged undo later under a lease, dropping the answer turn added since without going stale', async () => { + await turn('one'); await turn('two'); await turn('three'); + await undo(2); + const change = asked[0]!; + await turn('notice and answer'); // same branch: a newer turn, not a conflict + const applied = await framework.runAtSafeBoundary({ verb: 'apply rev-u' }, (lease) => + framework.applyResolvedOperatorChange(change, { lease, admission: { id: 'rev-u' } })); + assert.deepEqual(applied.kind === 'undo-turns' && [applied.requested, applied.undone], [2, 2]); + assert.deepEqual(texts(), ['one', 'reply to one', 'two'], 'the staged turns and the answer turn are undone'); + assert.equal(host().getTurnCheckpoints('scout').length, 1, 'staged and newer checkpoints removed'); + framework.redo('scout'); + assert.equal(texts().at(-1), 'reply to notice and answer', 'one redo restores the source tip, answer turn included'); + assert.equal(host().getTurnCheckpoints('scout').length, 4, 'with every checkpoint'); + assert.ok(framework.getOperatorLog().some((e) => e.kind === 'undo-turns' && (e.params as { admission?: string }).admission === 'rev-u')); + }); + + it('finishes a cut an earlier attempt created but never switched to', async () => { + await turn('one'); await turn('two'); + await undo(1); + const change = asked[0]!; + // As if a crash came between creating the destination and switching to it. + const store = framework.getStore(); + const oldest = change.kind === 'undo-turns' ? change.checkpoints.at(-1)! : null; + store.createBranchAt(`undo/scout/op-${change.id}`, change.sourceBranch, oldest!.sequenceBefore); + const applied = await framework.runAtSafeBoundary({ verb: 'retry' }, (lease) => + framework.applyResolvedOperatorChange(change, { lease, admission: { id: 'rev-u' } })); + assert.equal(applied.kind === 'undo-turns' && applied.alreadyApplied, undefined, 'applied now'); + assert.equal(branch(), `undo/scout/op-${change.id}`); + assert.deepEqual(texts(), ['one', 'reply to one', 'two']); + }); + + it('recognizes a cut that was already applied, and refuses one whose branch moved', async () => { + await turn('one'); await turn('two'); + await undo(1); + const change = asked[0]!; + const apply = () => framework.runAtSafeBoundary({ verb: 'apply' }, (lease) => + framework.applyResolvedOperatorChange(change, { lease, admission: { id: 'rev-u' } })); + await apply(); + const after = { branch: branch(), texts: texts() }; + const again = await apply(); + assert.equal(again.kind === 'undo-turns' && again.alreadyApplied, true, 'a retry changes nothing'); + assert.deepEqual({ branch: branch(), texts: texts() }, after); + + framework.redo('scout'); // the active branch moves away from the destination + await assert.rejects(apply(), (e: Error & { code?: string }) => e.code === 'stale' && /already applied/.test(e.message)); + }); + + it('refuses a staged undo whose branch moved or whose checkpoint is gone', async () => { + await turn('one'); await turn('two'); + await undo(1); + const change = asked[0]!; + framework.undoLastTurn('scout'); // someone else undid it meanwhile + await assert.rejects( + framework.runAtSafeBoundary({ verb: 'apply' }, (lease) => + framework.applyResolvedOperatorChange(change, { lease, admission: { id: 'rev-u' } })), + (e: Error & { code?: string }) => e.code === 'stale' && /active branch moved/.test(e.message), + ); + }); + + it('refuses a staged undo whose checkpoint is gone though the branch stayed', async () => { + await turn('one'); await turn('two'); + await undo(1); + const change = asked[0]!; + host().saveTurnCheckpoints('scout', host().getTurnCheckpoints('scout').slice(0, 1)); // the staged one is gone + await assert.rejects( + framework.runAtSafeBoundary({ verb: 'apply' }, (lease) => + framework.applyResolvedOperatorChange(change, { lease, admission: { id: 'rev-u' } })), + (e: Error & { code?: string }) => e.code === 'stale' && /no longer an undoable checkpoint/.test(e.message), + ); + }); + + it('resolves fewer turns than requested when history is shorter, and reports both', async () => { + decide = async () => ({ decision: 'apply' }); + await turn('only'); + const r = await undo(3); + assert.deepEqual([r.ok, r.undone, r.requested], [true, 1, 3]); + assert.equal(asked[0]!.kind === 'undo-turns' && asked[0]!.checkpoints.length, 1); + }); + + it('refuses when the gate fails, and asks nothing when there is nothing to undo', async () => { + const empty = await undo(1); + assert.deepEqual([empty.ok, empty.undone], [true, 0]); + assert.equal(asked.length, 0); + await turn('one'); + decide = async () => { throw new Error('journal unavailable'); }; + const r = await undo(1); + assert.deepEqual([r.ok, r.code], [false, 'gate-failed']); + assert.equal(host().getTurnCheckpoints('scout').length, 1, 'nothing undone'); + }); +}); From 491b0f3afa8a1b3f5821977e400daee7b70e8dee Mon Sep 17 00:00:00 2001 From: Agnes-1009 Date: Wed, 7 Oct 2026 10:51:37 -0400 Subject: [PATCH 42/73] fix(framework): turn-owned apply boundary, retryable undo bookkeeping, unawaited undo delivery, durable immediate receipts (review) Kit's findings on af1e085 (room-222 #45167, /tmp/kit-1008-r2-undo-probe.mjs) and on R4 (#45325, /tmp/kit-1008-immediate-durability.mjs). 1. The apply boundary was only the store hold. A puppetToolCall can own the agent's live turn token with no store hold; while it waited, an unrelated module's gated agent_settings call got `apply` and changed the budget. - applyBoundaryAgainstCaller replaces storeHoldAgainstCaller. It refuses while the store is held by a reservation this work isn't running under, and also while the agent has a live turn that isn't the caller's own. - Ownership: turnScope (src/call-provenance.ts), an AsyncLocalStorage that puppetToolCallUngated sets around its execution with the token it holds; or being inside the lease whose tokens reserve the agent. - Ownership holds only while that exact token is the agent's active one (Kit, #45228). A delegated callback that fires after its puppet returned keeps its actor (callProvenance) but not permission to mutate during a later turn. - It refuses, never waits, so a wrapped call can't deadlock on its own outer token. 2. An undo's completion was claimed from destination-active alone. With a failure injected at the operatorCut write, the retry returned alreadyApplied but redo() found nothing. - finishUndoTurnsCut makes everything after the switch idempotent: the marker (written if absent); the destination's checkpoints (only those older than the cut); the one redo entry (RedoEntry.changeId, pushed once); and the trace and log (once per process, completedOperatorCuts). - A retry on the destination finishes what is missing, without cutting again, before reporting. 3. Undo had re-coupled delivery: gatedUndoTurns awaited syncDiscordAwarenessMarkers after cutting (a thrown sync failed the command although the branch had changed), and applyResolvedOperatorChange awaited the drain inside the host's held lease. - Both now start #246's deliverDiscordAwarenessInBackground (verb widened to 'undo') while the reservation or lease is still held, which installs the MCPL gate synchronously, and never await it. A delivery failure is logged and alerted, never this undo's. - This is the interim form on #246. With the rebase onto Ada's approved marks head, undo-turns moves to her explicit marks: refs frozen at staging, the before/after snapshot marking only their removed intersection, and later cut arrivals reported unmarked (June, room-203 #45297; Ada, #45349). 4. R4 had claimed durability for an immediate store. A durable delivery to an idle agent returned durable:true, but Chronicle persists only on sync(), so a SIGKILL before the next sync lost it. - storeDurably passes a requested-durable immediate delivery through the crash-recovery queue: queued and persisted under a fresh write id with its boot-scan position, stored stamped with that id, then acknowledged by a sync. - A failed sync leaves it queued for exactly-once replay (deduplicated by id), so durable:true means "on disk, or queued for exactly-once replay". If the queue can't be persisted, a direct sync decides, and its failure reports durable:false. - An ordinary immediate store now reports durable:false: stored, but not proven durable. The MessagePlacement and ModuleMessageOptions docs say so. Tests: - operator-change-gate (35): - another call owning the live turn refuses a module's apply; - a puppet's own delegated change applies, but its late callback during a later turn is refused; - for each seam (marker, checkpoints, log), a failure there leaves the cut landed, and the retry finishes the bookkeeping with one log record, one redo entry, and the one-step redo to the source; - an already-complete cut applied again adds no redo entry and no log record; - delivery that rejects or never resolves fails neither undo form, and neither waits for it. - module-targeted-delivery (10): - an ordinary immediate store reports durable:false; - a durable one is acknowledged through the queue; - with its sync failing it stays queued and still says durable; - a child process SIGKILLed right after a durable immediate delivery leaves the message present exactly once on reopen. Mutation checks, eight, each caught: - live turns ignored; ownership ignoring the exact token; the puppet setting no turn scope; - a retry not finishing the bookkeeping; the redo entry pushed per attempt; the log record per attempt; - the durable immediate path off; delivery awaited inside the lease (the test times out). Twenty-two affected suites: 321/321. mcpl-coalescing's undo case passes. Also in this commit: - shedNewestTurn is split into planShedExchange (pure planning, with an optional walk-back anchor and 'anchor-missing') and its application. Behaviour is unchanged (refusal-rewind and inference-exhaustion-rewind, 12/12). It's groundwork for unstick's planned steps. - index.ts exports ResolvedUndoTurnsChange. Separately observed, not changed: puppetToolCall crashes in spillOrTruncate on a module result with no `data`. This is pre-existing and unrelated to the gate; the test tool returns data. Co-Authored-By: Agnes (Claude Opus 5.5) --- src/call-provenance.ts | 9 ++ src/framework.ts | 214 +++++++++++++++++++------- src/index.ts | 1 + src/types/module.ts | 20 ++- test/module-targeted-delivery.test.ts | 82 +++++++++- test/operator-change-gate.test.ts | 118 +++++++++++++- 6 files changed, 372 insertions(+), 72 deletions(-) diff --git a/src/call-provenance.ts b/src/call-provenance.ts index c9ac9b293..81e78533a 100644 --- a/src/call-provenance.ts +++ b/src/call-provenance.ts @@ -25,3 +25,12 @@ export const callProvenance = new AsyncLocalStorage(); * a mutation can tell the lease holder's work from everyone else's. */ export const leaseScope = new AsyncLocalStorage(); + +/** + * The agent turn the current work runs as, carried across its asynchronous + * work: puppetToolCall sets it around its execution with the token it holds + * (its own, or the lease's). A gated change for that agent counts this turn + * as the caller's own rather than as a busy agent; anyone else's call finds + * the agent busy. + */ +export const turnScope = new AsyncLocalStorage<{ agent: string; token: number }>(); diff --git a/src/framework.ts b/src/framework.ts index b2be74143..baea6660a 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -9,7 +9,7 @@ import { MembraneError } from '@animalabs/membrane'; import { ContextManager, PassthroughStrategy, WindowedPassthroughStrategy, OverBudgetError, UncoveredDropError } from '@animalabs/context-manager'; import type { CacheWireReceipt } from './kv-unified-wire.js'; import { SUBCONSCIOUS_TOOLS, SUBCONSCIOUS_TOOL_NAMES, type SubconsciousConfig } from './tune-out/tools.js'; -import { callProvenance, leaseScope } from './call-provenance.js'; +import { callProvenance, leaseScope, turnScope } from './call-provenance.js'; import { selfChangeKind, sameValue, @@ -763,6 +763,9 @@ export class ResumeBlockedError extends Error { interface RedoEntry { branchName: string; + /** The admitted undo-turns change that pushed it, so a retry that finishes + * that cut's bookkeeping pushes it once. */ + changeId?: string; /** Checkpoints redo re-adds after switching back, oldest first. A single * undo saved its source without the checkpoint, so redo re-adds it; an * admitted multi-turn cut writes nothing on its source, whose tip still @@ -1575,6 +1578,8 @@ export class AgentFramework { /** Admissions applyResolvedOperatorChange is carrying out, by admission id: * each lets exactly its change's tool call through the gate. */ private activeAdmissions: Map = new Map(); + /** Admitted undo cuts whose bookkeeping finished in this process. */ + private completedOperatorCuts: Set = new Set(); /** Serialize per-server drains so reconnect and an online undo cannot race. */ private discordAwarenessDrains: Map> = new Map(); /** Framework-global inference gate; older generations cannot release it. */ @@ -5127,8 +5132,29 @@ export class AgentFramework { * host command. */ private shedNewestTurn(agent: Agent): RewindRecord | null { + const plan = this.planShedExchange(agent); + if (!plan || plan === 'anchor-missing') return null; + const cm = agent.getContextManager(); + for (const id of plan.removedIds) cm.removeMessage(id); + return { kind: plan.kind, descriptor: plan.descriptor, removedIds: plan.removedIds, discordRef: plan.discordRef }; + } + + /** + * Plan shedNewestTurn's removal without changing anything: the newest + * non-system message and its complete exchange, with `anchor`, the id of + * the message just before that exchange (null when none). Given `atOrBefore`, + * only messages at or before that id are considered, so an operation can + * walk back from where it stopped whatever arrived since; 'anchor-missing' + * when that id is no longer on the branch. Null when nothing is eligible. + */ + private planShedExchange(agent: Agent, atOrBefore?: MessageId | string): (RewindRecord & { anchor: MessageId | null }) | 'anchor-missing' | null { const cm = agent.getContextManager(); const all = cm.getAllMessages(); + let start = all.length - 1; + if (atOrBefore !== undefined) { + start = all.findIndex((m) => m.id === atOrBefore); + if (start < 0) return 'anchor-missing'; + } const typeOf = (b: unknown) => (b as { type?: string }).type; const hasBlock = (m: { content?: unknown } | undefined, t: string) => Array.isArray(m?.content) && (m!.content as unknown[]).some((b) => typeOf(b) === t); @@ -5143,7 +5169,7 @@ export class AgentFramework { // the latest assistant message cannot be modified"). We do NOT add a marker // here; the caller keeps one consolidated marker (updateRewindMarker). let idx = -1; - for (let i = all.length - 1; i >= 0; i--) { + for (let i = start; i >= 0; i--) { const meta = (all[i].metadata ?? {}) as { system?: unknown }; if (meta.system) continue; idx = i; break; @@ -5188,8 +5214,9 @@ export class AgentFramework { ? { channelId: String(md.channelId), messageId: String(md.messageId) } : undefined; - for (const id of removedIds) cm.removeMessage(id); - return { kind, descriptor, removedIds, discordRef }; + const first = Math.min(...removedIds.map((id) => all.findIndex((m) => m.id === id))); + const anchor = first > 0 ? all[first - 1].id : null; + return { kind, descriptor, removedIds, discordRef, anchor }; } /** @@ -6816,10 +6843,11 @@ export class AgentFramework { if (answer.outcome === 'gate-failed') return refused(`Operator change refused: ${answer.error}`); } // Permission to apply isn't a safe boundary: while the store is held by - // a reservation this call isn't running under, refuse rather than wait - // (the holder may be waiting for this very call's turn). - const held = this.storeHoldAgainstCaller(); - if (held) return refused(`Operator change refused: the store is held by ${held}; retry when it completes`); + // a reservation this call isn't running under, or the agent has a live + // turn that isn't this call's own, refuse rather than wait (the holder + // may be waiting for this very call's turn). + const busy = this.applyBoundaryAgainstCaller(agentName); + if (busy) return refused(`Operator change refused: ${busy}; retry when it completes`); // The last check before the change runs, with no await in between. try { this.revalidateSelfChange(change); @@ -6829,13 +6857,25 @@ export class AgentFramework { return run(); } - /** The store hold this work isn't running under, described; null when the - * store is free, or held by the lease whose callback this work runs in. */ - private storeHoldAgainstCaller(): string | null { - if (!this.surgeryHold) return null; + /** + * Why a gated change to `agentName` can't apply now from this caller, or + * null when it can. The store must not be held by a reservation this work + * isn't running under (only the lease whose callback this is), and the + * agent must have no live turn other than the caller's own: the turn a + * puppet call runs as (turnScope), or the lease's reservation of it. + */ + private applyBoundaryAgainstCaller(agentName: string): string | null { const scoped = leaseScope.getStore(); - if (scoped && this.heldLease?.lease === scoped) return null; - return `${this.surgeryHold.verb} for ${this.surgeryHold.agentName}`; + const inHeldLease = scoped !== undefined && this.heldLease?.lease === scoped; + if (this.surgeryHold && !inHeldLease) { + return `the store is held by ${this.surgeryHold.verb} for ${this.surgeryHold.agentName}`; + } + const token = this.activeTurnTokens.get(agentName); + if (token === undefined) return null; + const own = turnScope.getStore(); + if (own && own.agent === agentName && own.token === token) return null; + if (inHeldLease && this.heldLease!.tokens.get(agentName) === token) return null; + return `${agentName} has a live turn that isn't this call's`; } /** @@ -6872,6 +6912,9 @@ export class AgentFramework { const release = this.reserveStoreForSurgery('undo', agentName); try { applied = this.applyUndoTurns(change); + // Gate installed before release; delivery drains in the background + // and a failure there is never reported as this undo's. + this.deliverDiscordAwarenessInBackground('undo', agentName); } finally { release(); } @@ -6887,7 +6930,6 @@ export class AgentFramework { `[host-command] undo agent=${agentName} requested=${requested} undone=${applied.undone} ` + `as one cut (${change.id}) by=${requester.name ?? requester.id ?? requester.via}`, ); - await this.syncDiscordAwarenessMarkers(); return { ok: true, undone: applied.undone, requested, lastVisible: await this.lastVisiblePreview(agentName) }; } @@ -6943,6 +6985,9 @@ export class AgentFramework { const undone = change.checkpoints.length; const current = this.store.currentBranch().name; if (current === destination) { + // The cut happened. Finish whatever bookkeeping an interrupted attempt + // left undone, without cutting again, and only then report it applied. + this.finishUndoTurnsCut(change, admission); return { requested: change.requestedTurns, undone, fromBranch: change.sourceBranch, toBranch: destination, alreadyApplied: true }; } // The destination exists if an earlier attempt got as far as creating @@ -6963,45 +7008,57 @@ export class AgentFramework { if (at(c) < 0) stale(`turn ${c.turnIndex} is no longer an undoable checkpoint`); } const oldest = change.checkpoints[change.checkpoints.length - 1]!; - const cutAt = at(oldest); - const removed = list.slice(cutAt); // Checkpoint lists are branch-scoped state, and nothing is written on the // source: the destination, cut before the oldest staged turn recorded - // itself, inherits the older checkpoints (saved there explicitly), and - // the source tip keeps every checkpoint for redo to return to. So each - // step is idempotent and a retry after a crash finishes the cut. - const kept = list.slice(0, cutAt); + // itself, keeps only the older checkpoints, and the source tip keeps + // every checkpoint for redo to return to. if (!created) this.store.createBranchAt(destination, change.sourceBranch, oldest.sequenceBefore); this.store.switchBranch(destination); - // Mark the destination as used, naming the change that made it: this - // write is what moves its head off its branch point (see above), so a - // later retry can tell a completed cut from an interrupted one. + this.finishUndoTurnsCut(change, admission); + return { requested: change.requestedTurns, undone, fromBranch: change.sourceBranch, toBranch: destination }; + } + + /** + * The bookkeeping after an admitted undo cut, once the destination is + * active. Every step is idempotent, so a retry after any failure finishes + * exactly what is missing and none twice: + * - the marker naming this change on the destination, which moves its head + * off its branch point (how applyUndoTurns tells a used cut); + * - the destination's checkpoints: only those older than the cut; + * - the one redo entry back to the source tip; + * - the trace and the operator-log record, once per process. + */ + private finishUndoTurnsCut(change: ResolvedUndoTurnsChange, admission?: OperatorAdmission): void { + const destination = this.undoTurnsDestination(change); + const oldest = change.checkpoints[change.checkpoints.length - 1]!; const framework = this.store.getStateJson(FRAMEWORK_STATE_ID); - this.store.setStateJson(FRAMEWORK_STATE_ID, { - ...(framework && typeof framework === 'object' ? framework as Record : {}), - operatorCut: { changeId: change.id, at: Date.now() }, - }); - this.saveTurnCheckpoints(change.agent, kept); - const ws = this.moduleRegistry.getModule('workspace'); - if (ws && 'materializeMount' in ws) { - (ws as unknown as { materializeMount(name: string): Promise }).materializeMount('_config').catch(() => {}); + const state = (framework && typeof framework === 'object' ? framework : {}) as Record; + if ((state.operatorCut as { changeId?: unknown } | undefined)?.changeId !== change.id) { + this.store.setStateJson(FRAMEWORK_STATE_ID, { ...state, operatorCut: { changeId: change.id, at: Date.now() } }); } + this.saveTurnCheckpoints( + change.agent, + this.getTurnCheckpoints(change.agent).filter((c) => c.sequenceBefore < oldest.sequenceBefore), + ); + this.materializeConfigMountAfterBranchSwitch(); let redoStack = this.redoStacks.get(change.agent); if (!redoStack) { redoStack = []; this.redoStacks.set(change.agent, redoStack); } // The source tip still holds the removed checkpoints: redo re-adds none. - redoStack.push({ branchName: change.sourceBranch, checkpoints: [] }); - + if (!redoStack.some((entry) => entry.changeId === change.id)) { + redoStack.push({ branchName: change.sourceBranch, checkpoints: [], changeId: change.id }); + } + if (this.completedOperatorCuts.has(change.id)) return; this.emitTrace({ type: 'undo:completed', agentName: change.agent, turnIndex: oldest.turnIndex, fromBranch: change.sourceBranch, toBranch: destination, - turns: undone, + turns: change.checkpoints.length, }); this.recordOperatorAction({ kind: 'undo-turns', @@ -7010,27 +7067,13 @@ export class AgentFramework { params: { changeId: change.id, requested: change.requestedTurns, - resolved: undone, + resolved: change.checkpoints.length, turnIndexes: change.checkpoints.map((c) => c.turnIndex), ...(admission ? { admission: admission.id } : {}), }, - result: { sourceBranch: change.sourceBranch, targetBranch: destination, checkpointsRemoved: removed.length }, + result: { sourceBranch: change.sourceBranch, targetBranch: destination }, }); - return { requested: change.requestedTurns, undone, fromBranch: change.sourceBranch, toBranch: destination }; - } - - /** After an admitted undo: reconcile awareness markers as host/command - * undo always has. The cut stands either way; the durable outbox owns - * eventual delivery, so a sync failure is logged, not thrown. */ - private async syncAwarenessAfterUndo(agentName: string): Promise { - try { - await this.syncDiscordAwarenessMarkers(); - } catch (error) { - console.error( - `[operator-change] awareness sync after undo for ${agentName} failed; the outbox retries: ` + - `${error instanceof Error ? error.message : String(error)}`, - ); - } + this.completedOperatorCuts.add(change.id); } /** @@ -7050,7 +7093,10 @@ export class AgentFramework { this.heldLeaseTokens(opts.lease, `apply ${change.kind} for ${change.agent}`); if (change.kind === 'undo-turns') { const applied = this.applyUndoTurns(change, opts.admission); - if (!applied.alreadyApplied) await this.syncAwarenessAfterUndo(change.agent); + // Installs the MCPL data-plane gate now, under the lease, and drains in + // the background: remote delivery is never awaited inside the lease, + // and its failure is never this undo's. + if (!applied.alreadyApplied) this.deliverDiscordAwarenessInBackground('undo', change.agent); return { kind: 'undo-turns', ...applied }; } this.revalidateSelfChange(change); @@ -12582,14 +12628,16 @@ export class AgentFramework { let ownedToEnd = false; try { const started = Date.now(); - const result = await this.executeToolCall({ + // The call, and anything it delegates, runs as this turn: a gated + // change it makes to this agent is the caller's own, not a busy agent. + const result = await turnScope.run({ agent: agentName, token: turnToken }, () => this.executeToolCall({ id: toolUseId, name: toolName, input, callerAgentName: agentName, origin: 'puppet', ...(opts?.admission ? { admission: opts.admission } : {}), - }); + })); const durationMs = Date.now() - started; // Store the pair through the same shapes the ordinary path uses. Build @@ -14121,13 +14169,67 @@ export class AgentFramework { return '' as MessageId; // Deferred — flushed at the target's next boundary } + if (opts?.durable && !opts.deferredWriteId) { + return this.storeDurably(agent, participant, content, metadata, opts.placement); + } const stored = agent.getContextManager().addMessage( participant, content, opts?.deferredWriteId ? withDeferredWriteId(metadata, opts.deferredWriteId) : metadata, ); if (opts?.deferredWriteId) this.markDeferredWriteLanded(opts.deferredWriteId); - if (opts?.placement) { opts.placement.agent = agent.name; opts.placement.messageId = stored; opts.placement.durable = true; } + // Stored, but not yet proven durable: Chronicle persists on sync(). + if (opts?.placement) { opts.placement.agent = agent.name; opts.placement.messageId = stored; opts.placement.durable = false; } + return stored; + } + + /** + * Store a message at once with a durable receipt (ModuleMessageOptions + * .durable). Chronicle persists only on sync(), so a stored id alone + * proves nothing across a hard kill. The write goes through the + * crash-recovery queue the deferred path uses: queued and persisted under + * a fresh write id, with its boot-scan position, before it is written; + * stored stamped with that id; then acknowledged by a sync. If the sync + * fails the entry stays queued and a restart replays it once, + * deduplicated by the id, so durable means "on disk, or queued for + * exactly-once replay". If the queue can't be persisted, a direct sync + * decides, and its failure is reported as not durable. + */ + private storeDurably( + agent: Agent, + participant: string, + content: ContentBlock[], + metadata: MessageMetadata | undefined, + placement?: { agent?: string; messageId?: MessageId; deferredId?: string; durable?: boolean }, + ): MessageId { + const cm = agent.getContextManager(); + const id = randomUUID(); + let count = 0; + try { + const counted = cm as unknown as { getMessageCount?: () => number; getAllMessages: () => unknown[] }; + count = typeof counted.getMessageCount === 'function' ? counted.getMessageCount() : counted.getAllMessages().length; + } catch { count = 0; } + const prevScan = this.deferredScanFrom.get(agent.name); + this.deferredScanFrom.set(agent.name, prevScan === undefined ? count : Math.min(prevScan, count)); + this.unackedDeferredWrites.push({ + id, seq: ++this.deferredSeq, participant, content, metadata, forAgent: agent.name, durable: true, + }); + const queued = this.persistDeferredWrites({ force: true }); + if (!queued) this.unackedDeferredWrites = this.unackedDeferredWrites.filter((m) => m.id !== id); + const stored = cm.addMessage(participant, content, withDeferredWriteId(metadata, id)); + let durable = queued; + if (queued) { + this.markDeferredWriteLanded(id); + this.ackDeferredWrites(); // a failed sync leaves it queued: still durable + } else { + try { + this.store.sync(); + durable = true; + } catch (err) { + console.error(`[module-delivery] durable message for ${agent.name} stored but its sync failed:`, err); + } + } + if (placement) { placement.agent = agent.name; placement.messageId = stored; placement.durable = durable; } return stored; } diff --git a/src/index.ts b/src/index.ts index 408fcd77e..c924be5b5 100644 --- a/src/index.ts +++ b/src/index.ts @@ -12,6 +12,7 @@ export type { ResolvedPresentationChange, RuntimeSettingsValues, AppliedOperatorChange, + ResolvedUndoTurnsChange, } from './operator-change.js'; export { Agent } from './agent.js'; export type { StartStreamResult } from './agent.js'; diff --git a/src/types/module.ts b/src/types/module.ts index 71c99b7df..f859fdaf0 100644 --- a/src/types/module.ts +++ b/src/types/module.ts @@ -186,11 +186,13 @@ export interface ModuleMessageOptions { /** Out-param: filled with where the message went. */ placement?: MessagePlacement; /** - * If the message has to be deferred, keep it in the persisted - * crash-recovery queue (the one quiesce uses) rather than in memory only, - * so a restart before its flush replays it once (deduplicated by its - * deferred-write id). Deferral itself is unchanged: the same guard decides - * when it is held and when it lands. + * Make the delivery durable before the receipt says so. Deferred: keep it + * in the persisted crash-recovery queue (the one quiesce uses) rather than + * in memory only, so a restart before its flush replays it once + * (deduplicated by its deferred-write id). Stored at once: pass it through + * that queue and acknowledge it with a store sync, so a hard kill right + * after the call neither loses nor duplicates it. Deferral itself is + * unchanged: the same guard decides when it is held and when it lands. */ durable?: boolean; } @@ -205,9 +207,11 @@ export interface MessagePlacement { * `metadata.deferredWriteId` on the message when it lands. */ deferredId?: string; /** - * True when it was stored, or deferred with its queue persisted. False - * when it is deferred in memory only: not asked to be durable, or the - * persist failed or exceeded its size cap. + * True only when the delivery is proven durable: asked for with `durable`, + * and either on disk (synced) or held in the persisted recovery queue for + * exactly-once replay. False otherwise: stored without asking (it reaches + * disk at the next sync), deferred in memory only, or a persist or sync + * failed. */ durable?: boolean; } diff --git a/test/module-targeted-delivery.test.ts b/test/module-targeted-delivery.test.ts index 47a0782e1..767f781f6 100644 --- a/test/module-targeted-delivery.test.ts +++ b/test/module-targeted-delivery.test.ts @@ -4,15 +4,18 @@ * went, and ask for a deferral to survive a restart. * * Deferral is unchanged: the same guard decides when a message is held and - * when it lands. `durable` only keeps a held message in the persisted - * recovery queue (the one quiesce uses) instead of memory, so it replays - * exactly once after a crash. + * when it lands. `durable` keeps a held message in the persisted recovery + * queue (the one quiesce uses) instead of memory, and passes a message stored + * at once through that queue to a sync, so either replays exactly once after + * a crash. Only a delivery proven durable says so. */ import { describe, it, beforeEach, afterEach } from 'node:test'; import assert from 'node:assert/strict'; import { tmpdir } from 'node:os'; -import { existsSync, mkdtempSync, readFileSync, rmSync } from 'node:fs'; +import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; import { join } from 'node:path'; +import { spawnSync } from 'node:child_process'; +import { fileURLToPath } from 'node:url'; import { AgentFramework } from '../src/index.js'; import type { MessagePlacement, Module, ModuleContext, ToolDefinition, ToolResult } from '../src/index.js'; import { MockMembrane } from './helpers/mock-membrane.js'; @@ -75,10 +78,35 @@ describe('ModuleContext.addMessage delivery options', () => { const placement: MessagePlacement = {}; const id = courier.ctx.addMessage('user', [{ type: 'text', text: 'for other' }], undefined, { forAgent: 'other', placement }); assert.ok(id, 'stored, with an id'); - assert.deepEqual(placement, { agent: 'other', messageId: id, durable: true }); + assert.deepEqual(placement, { agent: 'other', messageId: id, durable: false }, 'stored, not yet proven durable'); assert.deepEqual(texts(framework, 'other').map((m) => m.text), ['for other']); }); + it('passes a durable message stored at once through the recovery queue to a sync', async () => { + const placement: MessagePlacement = {}; + const id = courier.ctx.addMessage('user', [{ type: 'text', text: 'proven' }], undefined, { forAgent: 'other', placement, durable: true }); + assert.deepEqual(placement, { agent: 'other', messageId: id, durable: true }); + const internals = framework as unknown as { unackedDeferredWrites: unknown[]; deferredMessages: unknown[] }; + assert.equal(internals.unackedDeferredWrites.length, 0, 'acknowledged by the sync'); + assert.equal(internals.deferredMessages.length, 0); + assert.deepEqual(texts(framework, 'other').map((m) => m.text), ['proven'], 'stored once'); + }); + + it('keeps a durable message stored at once queued when its sync fails, and still says durable', async () => { + const store = framework.getStore() as unknown as { sync: () => void }; + const realSync = store.sync.bind(store); + store.sync = () => { throw new Error('injected sync failure'); }; + const placement: MessagePlacement = {}; + try { + courier.ctx.addMessage('user', [{ type: 'text', text: 'queued' }], undefined, { forAgent: 'other', placement, durable: true }); + } finally { + store.sync = realSync; + } + assert.equal(placement.durable, true, 'held in the persisted queue for exactly-once replay'); + const internals = framework as unknown as { unackedDeferredWrites: Array<{ id: string }> }; + assert.equal(internals.unackedDeferredWrites.length, 1, 'still queued, awaiting a sync'); + }); + it("holds a targeted message for the target's turn even when the primary is idle", async () => { // The guard is the target's: other mid-turn defers it although scout, // the primary, has no turn alive. @@ -159,3 +187,47 @@ describe('ModuleContext.addMessage delivery options', () => { assert.equal(placement.durable, false); }); }); + +describe('ModuleContext.addMessage durable delivery across a hard kill', () => { + it('keeps a durable message stored at once, exactly once, when the process is killed right after the call', () => { + const dir = mkdtempSync(join(tmpdir(), 'module-delivery-kill-')); + const index = fileURLToPath(new URL('../src/index.js', import.meta.url)); + const mock = fileURLToPath(new URL('./helpers/mock-membrane.js', import.meta.url)); + const script = join(dir, 'child.mjs'); + writeFileSync(script, ` + import { AgentFramework } from ${JSON.stringify(index)}; + import { MockMembrane } from ${JSON.stringify(mock)}; + import { writeFileSync } from 'node:fs'; + import { join } from 'node:path'; + const [mode, dir] = process.argv.slice(2); + const courier = { name: 'courier', async start(c) { this.ctx = c; }, async stop() {}, getTools() { return []; }, + async handleToolCall() { return { success: true }; }, async onProcess() { return {}; } }; + console.log = () => {}; console.error = () => {}; + const fw = await AgentFramework.create({ storePath: join(dir, 'store'), membrane: new MockMembrane().asMembrane(), + agents: [{ name: 'scout', model: 'test-model', systemPrompt: 'test' }], modules: [courier] }); + if (mode === 'deliver') { + fw.getStore().sync(); + const placement = {}; + courier.ctx.addMessage('user', [{ type: 'text', text: 'DURABLE_IMMEDIATE_NOTICE' }], undefined, { forAgent: 'scout', durable: true, placement }); + writeFileSync(join(dir, 'receipt.json'), JSON.stringify(placement)); + process.kill(process.pid, 'SIGKILL'); + } else { + const all = JSON.stringify(fw.getAgent('scout').getContextManager().getAllMessages()); + writeFileSync(join(dir, 'count.json'), JSON.stringify(all.split('DURABLE_IMMEDIATE_NOTICE').length - 1)); + await fw.stop(); + process.exit(0); + } + `); + try { + const killed = spawnSync(process.execPath, [script, 'deliver', dir], { encoding: 'utf8' }); + assert.equal(killed.signal, 'SIGKILL', killed.stderr); + const receipt = JSON.parse(readFileSync(join(dir, 'receipt.json'), 'utf8')) as MessagePlacement; + assert.equal(receipt.durable, true); + const reopened = spawnSync(process.execPath, [script, 'check', dir], { encoding: 'utf8' }); + assert.equal(reopened.status, 0, reopened.stderr); + assert.equal(JSON.parse(readFileSync(join(dir, 'count.json'), 'utf8')), 1, 'present after the kill, exactly once'); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); +}); diff --git a/test/operator-change-gate.test.ts b/test/operator-change-gate.test.ts index 6ced82066..8cc9abebb 100644 --- a/test/operator-change-gate.test.ts +++ b/test/operator-change-gate.test.ts @@ -32,15 +32,33 @@ import { MockMembrane, createMockResponse } from './helpers/mock-membrane.js'; class Mover implements Module { readonly name = 'mover'; ctx!: ModuleContext; + /** Resolves `hold` once released. */ + release?: () => void; + entered?: () => void; + /** The delegated change `set_budget_later` makes after its call returned. */ + later?: Promise; async start(ctx: ModuleContext): Promise { this.ctx = ctx; } async stop(): Promise {} getTools(): ToolDefinition[] { - return [{ name: 'set_budget', description: 'Change the caller budget.', inputSchema: { type: 'object', properties: {} } }]; + return ['set_budget', 'hold', 'set_budget_later'].map((name) => ({ name, description: name, inputSchema: { type: 'object', properties: {} } })); } async handleToolCall(call: ToolCall): Promise { - return this.ctx.callTool!({ - id: 'delegated', name: 'agent_settings', input: { action: 'update', context_budget_tokens: 130_000 }, callerAgentName: call.callerAgentName, + const change = (id: string, tokens = 130_000) => this.ctx.callTool!({ + id, name: 'agent_settings', input: { action: 'update', context_budget_tokens: tokens }, callerAgentName: call.callerAgentName, }); + if (call.name === 'hold') { + this.entered?.(); + await new Promise((resolve) => { this.release = resolve; }); + return { success: true, data: 'held' }; + } + if (call.name === 'set_budget_later') { + let fire!: () => void; + const fired = new Promise((resolve) => { fire = resolve; }); + this.later = fired.then(() => change('late', 160_000)); + (this as { fireLater?: () => void }).fireLater = fire; + return { success: true, data: 'scheduled' }; + } + return change('delegated'); } async onProcess(): Promise> { return {}; } } @@ -278,6 +296,38 @@ describe('operator-change gate: self-change tools', () => { assert.equal(budget(), 150_000, 'the lease holder can apply under its own lease'); }); + it("refuses to apply while another call owns the agent's live turn", async () => { + decide = async () => ({ decision: 'apply' }); + const before = budget(); + const entered = new Promise((resolve) => { mover.entered = resolve; }); + const holding = framework.puppetToolCall('scout', 'mover--hold', {}); + await entered; // the puppet now owns scout's turn token + const outside = await mover.ctx.callTool!({ id: 'own', name: 'agent_settings', input: { action: 'update', context_budget_tokens: 150_000 }, callerAgentName: 'scout' }); + assert.match(String(outside.error), /scout has a live turn that isn't this call's/); + assert.equal(budget(), before); + mover.release!(); + await holding; + }); + + it("applies a puppet's own delegated change during its turn, but not from a late callback in a later turn", async () => { + decide = async () => ({ decision: 'apply' }); + const own = await framework.puppetToolCall('scout', 'mover--set_budget', {}); + assert.equal(own.result.success, true, String(own.result.error)); + assert.equal(budget(), 130_000, 'the puppet owns its turn'); + + await framework.puppetToolCall('scout', 'mover--set_budget_later', {}); // returns; the change is still to come + const tokens = (framework as unknown as { activeTurnTokens: Map }).activeTurnTokens; + tokens.set('scout', 987_654); // a later turn is alive when the callback fires + try { + (mover as unknown as { fireLater: () => void }).fireLater(); + const late = await mover.later!; + assert.match(String(late.error), /live turn that isn't this call's/, 'provenance kept, permission not'); + assert.equal(budget(), 130_000, 'unchanged by the late callback'); + } finally { + tokens.delete('scout'); + } + }); + it('admits only the exact call its admission was registered for, and revalidates it as it runs', async () => { await framework.puppetToolCall('scout', 'agent_settings', { action: 'update', context_budget_tokens: 120_000 }); const change = asked[0]!; @@ -558,6 +608,8 @@ describe('operator-change gate: host/command undo by turns', () => { const again = await apply(); assert.equal(again.kind === 'undo-turns' && again.alreadyApplied, true, 'a retry changes nothing'); assert.deepEqual({ branch: branch(), texts: texts() }, after); + assert.equal(host().redoStacks.get('scout')!.length, 1, 'no second redo entry'); + assert.equal(framework.getOperatorLog().filter((e) => e.kind === 'undo-turns').length, 1, 'no second log record'); framework.redo('scout'); // the active branch moves away from the destination await assert.rejects(apply(), (e: Error & { code?: string }) => e.code === 'stale' && /already applied/.test(e.message)); @@ -575,6 +627,66 @@ describe('operator-change gate: host/command undo by turns', () => { ); }); + for (const seam of ['marker', 'checkpoints', 'log'] as const) { + it(`finishes the cut's bookkeeping on retry after a failure at the ${seam} write, redo included`, async () => { + await turn('one'); await turn('two'); await turn('three'); + const source = branch(); + await undo(2); + const change = asked[0]!; + const fw = framework as unknown as { + saveTurnCheckpoints: (agent: string, list: unknown[]) => void; + recordOperatorAction: (entry: { kind: string }) => void; + }; + const store = framework.getStore() as unknown as { setStateJson: (id: string, v: unknown) => void }; + const real = { setState: store.setStateJson.bind(store), save: fw.saveTurnCheckpoints.bind(fw), record: fw.recordOperatorAction.bind(fw) }; + let armed = true; + const fail = () => { armed = false; throw new Error(`injected ${seam} failure`); }; + if (seam === 'marker') store.setStateJson = (id, v) => (armed && (v as { operatorCut?: unknown })?.operatorCut ? fail() : real.setState(id, v)); + if (seam === 'checkpoints') fw.saveTurnCheckpoints = (agent, list) => (armed ? fail() : real.save(agent, list)); + if (seam === 'log') fw.recordOperatorAction = (entry) => (armed && entry.kind === 'undo-turns' ? fail() : real.record(entry)); + const apply = () => framework.runAtSafeBoundary({ verb: 'apply' }, (lease) => + framework.applyResolvedOperatorChange(change, { lease, admission: { id: 'rev-u' } })); + try { + await assert.rejects(apply(), new RegExp(`injected ${seam} failure`)); + } finally { + store.setStateJson = real.setState; + fw.saveTurnCheckpoints = real.save; + fw.recordOperatorAction = real.record; + } + assert.equal(branch(), `undo/scout/op-${change.id}`, 'the cut landed before the failure'); + const retried = await apply(); + assert.equal(retried.kind === 'undo-turns' && retried.alreadyApplied, true, 'not cut again'); + assert.equal(host().getTurnCheckpoints('scout').length, 1); + assert.equal(framework.getOperatorLog().filter((e) => e.kind === 'undo-turns').length, 1, 'logged once'); + assert.equal(host().redoStacks.get('scout')!.length, 1, 'one redo entry, not one per attempt'); + const redone = framework.redo('scout'); + assert.deepEqual([redone.redone, redone.toBranch], [true, source], 'the promised one-step redo exists'); + assert.equal(host().getTurnCheckpoints('scout').length, 3); + }); + } + + it('never awaits awareness delivery, and never reports its failure as the undo\'s', async () => { + decide = async () => ({ decision: 'apply' }); + const fw = framework as unknown as { discordAwarenessOutbox: unknown; syncDiscordAwarenessMarkers: () => Promise }; + fw.discordAwarenessOutbox = {}; // so delivery is attempted + let calls = 0; + fw.syncDiscordAwarenessMarkers = () => { calls++; return calls === 1 ? Promise.reject(new Error('delivery down')) : new Promise(() => {}); }; + try { + await turn('one'); await turn('two'); await turn('three'); + const now = await undo(1); + assert.equal(now.ok, true, 'a delivery failure is not the undo\'s'); + decide = async (change) => ({ decision: 'staged', receipt: { id: 'rev-d', text: `staged ${change.kind}` } }); + await undo(1); + const staged = asked.at(-1)!; + const applied = await framework.runAtSafeBoundary({ verb: 'apply' }, (lease) => + framework.applyResolvedOperatorChange(staged, { lease, admission: { id: 'rev-d' } })); + assert.equal(applied.kind, 'undo-turns', 'returned while delivery is still pending'); + assert.equal(calls, 2, 'delivery started both times'); + } finally { + fw.discordAwarenessOutbox = null; + } + }); + it('refuses a staged undo whose checkpoint is gone though the branch stayed', async () => { await turn('one'); await turn('two'); await undo(1); From e8243795ee604c3714a9ce877e1a68712a75dc13 Mon Sep 17 00:00:00 2001 From: Agnes-1009 Date: Wed, 7 Oct 2026 12:22:11 -0400 Subject: [PATCH 43/73] feat(framework): one leased application route; planned, journaled unstick; durable undo completion and honest store-only receipts (R2) Four strands. A design change agreed in room-222: Kit #45780, my reply #45802, Tessa #45806, recorded in shelf-380 4b830aa6308b. Unstick on it, to Kit's and Tessa's terms (#45177, #45228, #45240, #45270). Two findings from Kit on c0cddac (#45809, #45903). 1. A single application route. - The gate stages and never applies: FrameworkConfig.operatorChangeGate(change) => Promise. The receipt means accepted and queued, not applied. A throw, or a missing receipt, refuses the change. OperatorChangeDecision and its `apply` answer are gone. - A staged change is applied only by applyResolvedOperatorChange, under a held safe-boundary lease, whenever the host finds it eligible. Silent, notify and override revisions are scheduled through that same route. - Removed, with no functional loss: the immediate-apply branches (gatedSelfChange's apply path, the puppet apply path, apply-now undo under reserveStoreForSurgery, the self-driven unstick) and the ownership machinery that made them safe (turnScope, leaseScope, applyBoundaryAgainstCaller). An immediate apply was refused when the agent was busy, which is exactly when a lease waits. - Admissions now exist only while applyResolvedOperatorChange applies their change under the lease. An exactly matching call is revalidated and runs. Any other call is staged as a new change, including a delegated callback that fires after its puppet call returned. - The agent's own model calls and every no-gate path are unchanged. host/command undo and unstick return { code: 'staged', staged: { id } } with the receipt text. 2. Planned, journaled unstick (ResolvedUnstickChange). - Planning at staging walks back from the tail as it stood then (planShedExchange), one complete exchange per step through the cap. Each planned message carries a content fingerprint. The plan is the known range the resident is asked about; nothing that arrives later is ever chosen. - Each step runs under its own lease (applyResolvedOperatorChange with opts.step) and sheds exactly its planned exchange plus the consolidated marker. - Step k > 1 follows only a refused re-run after step k-1, and a step beyond the plan is refused. - A step is stale if a planned message is gone, edited in place (fingerprint), or the branch moved. - The record is a store-bound, branch-independent RecordJournal (operator/unstick; RecordJournal merged in as 2153e3d, as Ada's branch does). - A step's intent, naming its exact ids, is appended durable before anything is removed, and its completion after the removal is committed (afterCommittedState). - A step journaled shed reports alreadyApplied. One journaled only as intent finishes exactly its planned exchange. - Ambiguous writes reload and reduce. - Checkpoints keep the newest 32 operations every 64 entries. - rerunUnstick(change, { step }) runs outside any lease and is bound to the operation and step. - An attempt already journaled is reported, never launched again. - A new one is journaled `launched` durable before its one inference is queued; the trigger carries { operationId, step }. A batched turn and a budget restart carry it too. - In the stream such a re-run never sheds: it records responded, refused (with category) or failed (an error, or a stream that ended without a response, unless a restart carries it on). - The promise resolves only at the end of the stream's teardown, so the turn has fully settled. - A `launched` attempt nobody is running reads `interrupted` (outcome unknown) and is never relaunched. - getUnstickOperation(id) returns the record for reconciliation. 3. Undo completion is durable, and later work is never touched (#45809.1, #45903). - The bookkeeping no longer rewrites checkpoints. The lists are branch-scoped, and the destination inherits exactly the older ones; the re-save erased a later turn's checkpoint on retry. - The destination's marker is written {changeId} after the switch, then {changeId, completed: true} followed by a store sync (completeUndoTurnsCut). Branch state is durable only after sync, so completion is reported only after it. - A retry on a completed cut confirms that sync and touches nothing else, in process or after reopen: no checkpoints, no redo entry, no second log record. - A repair of an interrupted cut pushes its redo entry only if no turn has run on the destination since. A checkpoint recorded on the destination means new work invalidated redo. 4. A store-only recovery queue no longer claims durability (#45809.2). - With no recovery path, the deferred-write queue lives in branch state, which persists only on sync. persistDeferredWrites now returns true only when the queue would survive a killed process: written to the recovery file, or, in branch state, synced by a forced persist. - A failed sync reports durable:false, for immediate and deferred delivery alike. Tests: - operator-change-gate (46): rewritten for the single route. - Self-change: staged, delegated-late staged, applied via the lease. The ownership and immediate-apply cases are gone. - Undo: staged then applied as one cut; for each seam (marker, log, completion, completion sync), with and without later work on the destination before the retry. Without it, one log record, one redo entry, and the one-step redo. With it, the later turn's checkpoint kept and no redo entry restored. - A completed cut retried after a further turn, in process and after reopen, changes nothing and logs nothing new. - Delivery never awaited inside the lease. - Unstick: - the staged plan's exact range; - steps and re-runs never shedding a later arrival; - order and plan bounds; - stale on an in-place edit and on a branch move; - an interrupted step finishing its own exchange; - at-most-once re-runs, and an orphan reading interrupted; - the record surviving a branch move; - the host-driven loop back to back, asserting no live turn and an idle agent each time a re-run resolves. - module-targeted-delivery (12): with the store-only backend, a SIGKILLed child's durable receipt holds the message exactly once, and with an injected sync failure the receipt says false. The early hang in the back-to-back test was the mock: MockMembrane.streamYielding hands one stream every queued response. Queuing three upfront left the second re-run an empty stream. Responses are now queued as each re-run starts. That run did expose an existing AF gap, not changed here: a provider stream that ends with no events leaves the agent `streaming`. The unstick hook records that attempt as failed. Mutation checks, each caught: - the branch-state queue claiming durability unsynced; - a completed cut redoing its bookkeeping; - repair restoring redo after later work; - completion without its sync; - bookkeeping re-saving checkpoints; - a staged self-change running anyway; - a re-run resolving before its turn settled. Twenty-three suites, record-journal included: 342/342. mcpl-coalescing's undo case passes. Co-Authored-By: Agnes (Claude Opus 5.5) --- src/call-provenance.ts | 17 - src/framework.ts | 647 ++++++++++++++++++++------ src/index.ts | 6 +- src/operator-change.ts | 92 +++- src/types/agent.ts | 3 + src/types/framework.ts | 17 +- test/module-targeted-delivery.test.ts | 64 +++ test/operator-change-gate.test.ts | 385 +++++++++++---- 8 files changed, 940 insertions(+), 291 deletions(-) diff --git a/src/call-provenance.ts b/src/call-provenance.ts index 81e78533a..7d208e821 100644 --- a/src/call-provenance.ts +++ b/src/call-provenance.ts @@ -17,20 +17,3 @@ import type { ToolCall } from './types/index.js'; export type CallProvenance = Pick; export const callProvenance = new AsyncLocalStorage(); - -/** - * The lease whose callback (runAtSafeBoundary) the current work runs - * inside, carried across its asynchronous work. A module's own timer or an - * unrelated caller is outside it even while that lease holds the store, so - * a mutation can tell the lease holder's work from everyone else's. - */ -export const leaseScope = new AsyncLocalStorage(); - -/** - * The agent turn the current work runs as, carried across its asynchronous - * work: puppetToolCall sets it around its execution with the token it holds - * (its own, or the lease's). A gated change for that agent counts this turn - * as the caller's own rather than as a busy agent; anyone else's call finds - * the agent busy. - */ -export const turnScope = new AsyncLocalStorage<{ agent: string; token: number }>(); diff --git a/src/framework.ts b/src/framework.ts index baea6660a..239aa8c8e 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -9,7 +9,7 @@ import { MembraneError } from '@animalabs/membrane'; import { ContextManager, PassthroughStrategy, WindowedPassthroughStrategy, OverBudgetError, UncoveredDropError } from '@animalabs/context-manager'; import type { CacheWireReceipt } from './kv-unified-wire.js'; import { SUBCONSCIOUS_TOOLS, SUBCONSCIOUS_TOOL_NAMES, type SubconsciousConfig } from './tune-out/tools.js'; -import { callProvenance, leaseScope, turnScope } from './call-provenance.js'; +import { callProvenance } from './call-provenance.js'; import { selfChangeKind, sameValue, @@ -19,8 +19,12 @@ import { type ResolvedPresentationChange, type ResolvedSettingsChange, type ResolvedUndoTurnsChange, + type ResolvedUnstickChange, type RuntimeSettingsValues, + type UnstickAttemptOutcome, + type UnstickOperationRecord, } from './operator-change.js'; +import { RecordJournal } from './record-journal.js'; import { TuneOutCoordinator, TUNE_OUT_DEFAULTS } from './tune-out/coordinator.js'; import type { MessageId, @@ -585,6 +589,46 @@ function parseAgentSettingsInput(input: Record, extensions: Age } } +/** One entry of the unstick journal (operator/unstick). */ +type UnstickJournalEntry = + | { kind: 'step-intent'; operationId: string; agent: string; step: number; messageIds: string[] } + | { kind: 'step-done'; operationId: string; step: number } + | { kind: 'attempt-launched'; operationId: string; agent: string; step: number } + | { kind: 'attempt-done'; operationId: string; step: number; outcome: 'responded' | 'refused' | 'failed'; category?: string; error?: string }; +type UnstickJournalSnapshot = Record; + +function reduceUnstickEntry(ops: Map, entry: UnstickJournalEntry): void { + const op = ops.get(entry.operationId) + ?? { operationId: entry.operationId, agent: 'agent' in entry ? entry.agent : '', steps: [], attempts: [] }; + if ('agent' in entry && !op.agent) op.agent = entry.agent; + switch (entry.kind) { + case 'step-intent': + if (!op.steps.some((s) => s.step === entry.step)) { + op.steps.push({ step: entry.step, status: 'intent', messageIds: [...entry.messageIds] }); + } + break; + case 'step-done': { + const step = op.steps.find((s) => s.step === entry.step); + if (step) step.status = 'shed'; + break; + } + case 'attempt-launched': + if (!op.attempts.some((a) => a.step === entry.step)) op.attempts.push({ step: entry.step, status: 'launched' }); + break; + case 'attempt-done': { + const attempt = op.attempts.find((a) => a.step === entry.step); + if (attempt && attempt.status !== 'completed') { + attempt.status = 'completed'; + attempt.outcome = entry.outcome; + if (entry.category) attempt.category = entry.category; + if (entry.error) attempt.error = entry.error; + } + break; + } + } + ops.set(entry.operationId, op); +} + function withDeferredWriteId(metadata: MessageMetadata | undefined, id: string): MessageMetadata { return { ...(metadata ?? {}), deferredWriteId: id } as MessageMetadata; } @@ -1580,6 +1624,16 @@ export class AgentFramework { private activeAdmissions: Map = new Map(); /** Admitted undo cuts whose bookkeeping finished in this process. */ private completedOperatorCuts: Set = new Set(); + /** The unstick journal and its reduced operations (unstickLedger). */ + private unstickJournalState: { + journal: RecordJournal; + ops: Map; + } | null = null; + /** Unstick re-runs this process launched and hasn't settled, by operation#step. */ + private unstickAttemptWaiters: Map; resolve: (outcome: UnstickAttemptOutcome) => void }> = new Map(); + /** Outcomes recorded but not yet released to their waiters (the turn is + * still tearing down), by operation#step. */ + private unstickSettled: Map = new Map(); /** Serialize per-server drains so reconnect and an online undo cannot race. */ private discordAwarenessDrains: Map> = new Map(); /** Framework-global inference gate; older generations cannot release it. */ @@ -3765,9 +3819,11 @@ export class AgentFramework { * still memory-only: they flush within the turn, as before. */ /** - * Returns true when the durable queue is on disk as it now stands (written, - * or cleared because nothing is left), false when it is kept in memory - * only: not in persisted mode, over its size cap, or the write failed. + * Returns true when the queue as it now stands would survive a killed + * process: written to (or cleared from) the recovery file, or, with no + * recovery path, kept in branch state and synced by a forced persist. + * False otherwise: kept in memory only (not in persisted mode, over its + * size cap, or the write failed), or kept in branch state not yet synced. * `opts.force` enters persisted mode outside quiesce, for a module message * that asked to be durable (ModuleMessageOptions.durable); the mode ends, * as it does after quiesce, once the queue drains. @@ -3794,13 +3850,24 @@ export class AgentFramework { return false; } if (this.deferredWritesPath) { + // A written-and-renamed file survives a killed process. this.writeRecoveryFile(this.deferredWritesPath, { version: 2, pending: durable, scanFrom }); - } else { - this.warnRecoveryFallbackOnce(); - this.store.setStateJson(DEFERRED_WRITES_ID, { version: 2, pending: durable, scanFrom }); + this.deferredWritesPersisted = true; + return true; } + this.warnRecoveryFallbackOnce(); + this.store.setStateJson(DEFERRED_WRITES_ID, { version: 2, pending: durable, scanFrom }); this.deferredWritesPersisted = true; - return true; + // Branch state persists only on sync(): the queue is durable now only + // when a persist asked for durability (force) and its sync succeeded. + if (!opts?.force) return false; + try { + this.store.sync(); + return true; + } catch (err) { + console.error('[host-mode] deferred-write queue kept in branch state, but its sync failed:', err); + return false; + } } else if (this.deferredWritesPersisted) { if (this.deferredWritesPath) { this.writeRecoveryFile(this.deferredWritesPath, { version: 1, pending: [] }); @@ -5563,6 +5630,9 @@ export class AgentFramework { } const cap = Math.max(1, Math.min(10, Math.floor(params.maxRewinds ?? agent.refusalHandling?.maxRewinds ?? 3))); + if (this.operatorChangeGate) { + return this.gatedUnstick(agentName, cap, hostCommandRequester(serverId, params)); + } this.forcedRewind.set(agentName, { remaining: cap, removed: [], @@ -6515,8 +6585,7 @@ export class AgentFramework { `${opts.requester ? `, for ${opts.requester.name ?? opts.requester.id ?? 'operator'} via ${opts.requester.via}` : ''})`, ); try { - // The callback, and the work it starts, runs inside this lease's scope. - return await leaseScope.run(held.lease, () => fn(held.lease)); + return await fn(held.lease); } finally { held.release(); console.log(`[safe-boundary] ${held.lease.verb}: released after ${Date.now() - held.lease.since}ms`); @@ -6622,44 +6691,38 @@ export class AgentFramework { // Operator-change admission (FrameworkConfig.operatorChangeGate) // --------------------------------------------------------------------------- - /** Ask the host's gate about one resolved change. Without a gate the answer - * is apply, today's behaviour; a gate that throws or answers nonsense - * refuses the change (fails closed). Staging and refusals are logged. */ + /** Hand one resolved change to the host's gate, which stages it and + * answers with its receipt. A gate that throws, or answers without a + * receipt, refuses the change (fails closed). Both are logged. */ private async askOperatorChangeGate(change: ResolvedOperatorChange): Promise< - | { outcome: 'apply' } | { outcome: 'staged'; receipt: OperatorChangeReceipt } | { outcome: 'gate-failed'; error: string } > { - const gate = this.operatorChangeGate; - if (!gate) return { outcome: 'apply' }; - let answer: { outcome: 'apply' } | { outcome: 'staged'; receipt: OperatorChangeReceipt } | { outcome: 'gate-failed'; error: string }; + let answer: { outcome: 'staged'; receipt: OperatorChangeReceipt } | { outcome: 'gate-failed'; error: string }; try { - const decision = await gate(change); - if (decision?.decision === 'apply') answer = { outcome: 'apply' }; - else if (decision?.decision === 'staged' - && typeof decision.receipt?.id === 'string' && typeof decision.receipt?.text === 'string') { - answer = { outcome: 'staged', receipt: { id: decision.receipt.id, text: decision.receipt.text } }; - } else answer = { outcome: 'gate-failed', error: 'the operator-change gate returned no valid decision' }; + const receipt = await this.operatorChangeGate!(change); + answer = typeof receipt?.id === 'string' && typeof receipt?.text === 'string' + ? { outcome: 'staged', receipt: { id: receipt.id, text: receipt.text } } + : { outcome: 'gate-failed', error: 'the operator-change gate returned no receipt' }; } catch (error) { answer = { outcome: 'gate-failed', error: error instanceof Error ? error.message : String(error) }; } - if (answer.outcome !== 'apply') { - this.recordOperatorAction({ - kind: answer.outcome === 'staged' ? 'operator-change-staged' : 'operator-change-refused', - agent: change.agent, - ...(change.requester ? { requester: change.requester } : {}), - params: { changeId: change.id, change: change.kind, surface: change.surface }, - ...(answer.outcome === 'staged' - ? { result: { receipt: answer.receipt.id } } - : { error: `gate failed: ${answer.error}` }), - }); - } + this.recordOperatorAction({ + kind: answer.outcome === 'staged' ? 'operator-change-staged' : 'operator-change-refused', + agent: change.agent, + ...(change.requester ? { requester: change.requester } : {}), + params: { changeId: change.id, change: change.kind, surface: change.surface }, + ...(answer.outcome === 'staged' + ? { result: { receipt: answer.receipt.id } } + : { error: `gate failed: ${answer.error}` }), + }); return answer; } - /** The change `admission` was registered for (applyResolvedOperatorChange, - * or an applied puppet call), when it is exactly this agent, tool and - * input. Any other admission covers nothing. */ + /** The change `admission` was registered for, when it is exactly this + * agent, tool and input. Admissions exist only while + * applyResolvedOperatorChange applies their change under a held lease; + * any other admission covers nothing. */ private admittedChange( admission: OperatorAdmission | undefined, agentName: string, @@ -6668,7 +6731,7 @@ export class AgentFramework { ): ResolvedSettingsChange | ResolvedPresentationChange | null { if (!admission) return null; const change = this.activeAdmissions?.get(admission.id); - if (!change || change.agent !== agentName || change.kind === 'undo-turns') return null; + if (!change || change.agent !== agentName || (change.kind !== 'agent-settings' && change.kind !== 'tool-presentation')) return null; const changeTool = change.kind === 'agent-settings' ? 'agent_settings' : change.tool; return tool === changeTool && sameValue(change.input, input) ? change : null; } @@ -6813,10 +6876,11 @@ export class AgentFramework { /** * Run a self-change tool call (agent_settings, tool presentation) through * the gate when an operator or a module made it. The agent's own model - * calls (no origin), reads, and calls carrying an admission registered for - * exactly this change run as before. Otherwise the call is resolved and - * the gate decides: staged returns its receipt as the call's error; apply - * revalidates and runs exactly that call; a failed gate refuses it. + * calls (no origin) and reads run as before. A call admitted for exactly + * this change (applyResolvedOperatorChange, under its lease) is revalidated + * and runs. Anything else is resolved and staged: the receipt comes back + * as the call's error and nothing runs. A change that can't be resolved, + * or a failed gate, is refused. */ private async gatedSelfChange(agentName: string, call: ToolCall, run: () => ToolResult): Promise { if (!this.operatorChangeGate || call.origin === undefined) return run(); @@ -6824,66 +6888,36 @@ export class AgentFramework { const input = (call.input ?? {}) as Record; const refused = (error: string): ToolResult => ({ success: false, isError: true, error }); const admitted = this.admittedChange(call.admission, agentName, call.name, input); - let change: ResolvedSettingsChange | ResolvedPresentationChange; if (admitted) { - change = admitted; - } else { - // A change that can't be resolved is refused, never run ungated: the - // resolver shares the handler's parser, so whatever it can't resolve - // the gate could not have decided. + // The last check before the change runs, with no await in between. try { - change = this.resolveSelfChange(agentName, call.name, input, call.origin); + this.revalidateSelfChange(admitted); } catch (error) { - return refused(`Operator change refused: it couldn't be resolved (${error instanceof Error ? error.message : String(error)})`); - } - const answer = await this.askOperatorChangeGate(change); - if (answer.outcome === 'staged') { - return { success: false, isError: true, error: answer.receipt.text, data: { staged: answer.receipt } }; + return refused(error instanceof Error ? error.message : String(error)); } - if (answer.outcome === 'gate-failed') return refused(`Operator change refused: ${answer.error}`); - } - // Permission to apply isn't a safe boundary: while the store is held by - // a reservation this call isn't running under, or the agent has a live - // turn that isn't this call's own, refuse rather than wait (the holder - // may be waiting for this very call's turn). - const busy = this.applyBoundaryAgainstCaller(agentName); - if (busy) return refused(`Operator change refused: ${busy}; retry when it completes`); - // The last check before the change runs, with no await in between. + return run(); + } + // A change that can't be resolved is refused, never run ungated: the + // resolver shares the handler's parser, so whatever it can't resolve the + // gate could not have staged. + let change: ResolvedSettingsChange | ResolvedPresentationChange; try { - this.revalidateSelfChange(change); + change = this.resolveSelfChange(agentName, call.name, input, call.origin); } catch (error) { - return refused(error instanceof Error ? error.message : String(error)); + return refused(`Operator change refused: it couldn't be resolved (${error instanceof Error ? error.message : String(error)})`); } - return run(); + const answer = await this.askOperatorChangeGate(change); + if (answer.outcome === 'gate-failed') return refused(`Operator change refused: ${answer.error}`); + return { success: false, isError: true, error: answer.receipt.text, data: { staged: answer.receipt } }; } - /** - * Why a gated change to `agentName` can't apply now from this caller, or - * null when it can. The store must not be held by a reservation this work - * isn't running under (only the lease whose callback this is), and the - * agent must have no live turn other than the caller's own: the turn a - * puppet call runs as (turnScope), or the lease's reservation of it. - */ - private applyBoundaryAgainstCaller(agentName: string): string | null { - const scoped = leaseScope.getStore(); - const inHeldLease = scoped !== undefined && this.heldLease?.lease === scoped; - if (this.surgeryHold && !inHeldLease) { - return `the store is held by ${this.surgeryHold.verb} for ${this.surgeryHold.agentName}`; - } - const token = this.activeTurnTokens.get(agentName); - if (token === undefined) return null; - const own = turnScope.getStore(); - if (own && own.agent === agentName && own.token === token) return null; - if (inHeldLease && this.heldLease!.tokens.get(agentName) === token) return null; - return `${agentName} has a live turn that isn't this call's`; - } /** * host/command undo by turns when the host has an operator-change gate: - * resolve the exact checkpoints, ask, and either return the receipt - * (staged, nothing changed) or apply exactly that change now as one cut, - * under a store reservation. Without a gate, host/command keeps its - * per-turn loop. + * resolve the exact checkpoints and stage them, returning the receipt. + * Nothing changes now; the host applies the change through + * applyResolvedOperatorChange under a lease. Without a gate, host/command + * keeps its per-turn loop. */ private async gatedUndoTurns(agentName: string, requested: number, requester: OperatorRequester): Promise<{ ok: boolean; @@ -6901,36 +6935,294 @@ export class AgentFramework { const change = this.resolveUndoTurns(agentName, requested, requester); if (!change) return { ok: true, undone: 0, requested, lastVisible: null }; const answer = await this.askOperatorChangeGate(change); - if (answer.outcome === 'staged') { - return { ok: false, code: 'staged', error: answer.receipt.text, staged: { id: answer.receipt.id }, requested }; - } if (answer.outcome === 'gate-failed') { return { ok: false, code: 'gate-failed', error: `Operator change refused: ${answer.error}`, requested }; } - let applied: ReturnType; + return { ok: false, code: 'staged', error: answer.receipt.text, staged: { id: answer.receipt.id }, requested }; + } + + // --------------------------------------------------------------------------- + // Admitted unstick (ResolvedUnstickChange): planned steps, re-runs, journal + // --------------------------------------------------------------------------- + + /** The unstick journal, loaded and reduced on first use. Store-bound and + * branch-independent (RecordJournal): an operation's intents and outcomes + * don't rewind with an undo, and they die with this store. */ + private unstickLedger(): { + journal: RecordJournal; + ops: Map; + } { + if (!this.unstickJournalState) { + const journal = new RecordJournal(this.store, { type: 'operator/unstick' }); + this.unstickJournalState = { journal, ops: new Map() }; + this.reloadUnstickJournal(); + } + return this.unstickJournalState; + } + + /** Rebuild the reduced operations from exactly what the journal holds: at + * first use, and to reconcile after an ambiguous write. */ + private reloadUnstickJournal(): void { + const state = this.unstickJournalState!; + const { snapshot, entries } = state.journal.load(); + state.ops = new Map(Object.entries(snapshot ?? {}).map(([id, op]) => [id, structuredClone(op)])); + for (const { entry } of entries) reduceUnstickEntry(state.ops, entry); + } + + /** Append one journal entry and reduce it. A failure after the entry may + * have reached the store reconciles the reduced state before rethrowing. */ + private journalUnstick(entry: UnstickJournalEntry, opts: { durable?: boolean; afterCommittedState?: boolean }): void { + const ledger = this.unstickLedger(); try { - const release = this.reserveStoreForSurgery('undo', agentName); + ledger.journal.append(entry, opts); + } catch (error) { + if (ledger.journal.needsReconcile) this.reloadUnstickJournal(); + throw error; + } + reduceUnstickEntry(ledger.ops, entry); + if (ledger.journal.entriesSinceCheckpoint >= 64) { + // Keep the newest operations; older ones are long finished. + const kept = [...ledger.ops.entries()].slice(-32); try { - applied = this.applyUndoTurns(change); - // Gate installed before release; delivery drains in the background - // and a failure there is never reported as this undo's. - this.deliverDiscordAwarenessInBackground('undo', agentName); - } finally { - release(); + ledger.journal.checkpoint(Object.fromEntries(kept)); + ledger.ops = new Map(kept); + } catch (error) { + console.error('[unstick] journal checkpoint failed (replay stays longer):', error instanceof Error ? error.message : error); + if (ledger.journal.needsReconcile) this.reloadUnstickJournal(); + } + } + } + + /** The durable record of an unstick operation, or null if it never ran a step. */ + getUnstickOperation(operationId: string): UnstickOperationRecord | null { + const op = this.unstickLedger().ops.get(operationId); + return op ? structuredClone(op) : null; + } + + /** Content-only fingerprint of a message: an edit in place changes it. */ + private messageFingerprint(message: { participant?: unknown; content?: unknown }): string { + return createHash('sha256').update(JSON.stringify([message.participant ?? null, message.content ?? null])).digest('hex').slice(0, 32); + } + + /** + * Plan host/command unstick at staging: walk back from the tail as it + * stands now, one complete exchange per step (planShedExchange), through + * the cap or until nothing is left. Null when there is nothing to shed. + */ + private resolveUnstick(agentName: string, cap: number, requester?: OperatorRequester): ResolvedUnstickChange | null { + const agent = this.agents.get(agentName)!; + const cm = agent.getContextManager(); + const byId = new Map(cm.getAllMessages().map((m) => [String(m.id), m])); + const plan: ResolvedUnstickChange['plan'] = []; + let anchor: MessageId | null | undefined; + for (let step = 1; step <= cap; step++) { + if (anchor === null) break; + const next = this.planShedExchange(agent, anchor); + if (!next || next === 'anchor-missing') break; + const messageIds = next.removedIds.map(String); + plan.push({ step, messageIds, fingerprints: messageIds.map((id) => this.messageFingerprint(byId.get(id)!)) }); + anchor = next.anchor; + } + if (plan.length === 0) return null; + return { + id: randomUUID(), + agent: agentName, + surface: 'host-command', + ...(requester ? { requester } : {}), + resolvedAt: Date.now(), + sourceBranch: this.store.currentBranch().name, + kind: 'unstick', + cap, + plan, + }; + } + + /** + * Apply step `step` of a planned unstick. The caller holds the store (the + * host's lease, or the self-driven form's). Step k > 1 follows only a + * refused re-run after step k-1. The step's intent, naming exactly the + * planned exchange, is journaled durably before anything is removed, and + * its completion after the removal is committed. A step journaled as shed + * reports alreadyApplied; one journaled only as an intent (interrupted) + * finishes exactly its planned exchange, never another. + */ + private applyUnstickStep(change: ResolvedUnstickChange, step: number, admission?: OperatorAdmission): { + step: number; + shedIds: string[]; + alreadyApplied?: true; + } { + const stale = (what: string): never => { + throw new OperatorActionError('stale', `Unstick ${change.id} for ${change.agent} no longer holds: ${what}`); + }; + if (!Number.isInteger(step) || step < 1 || step > change.plan.length) { + throw new OperatorActionError('invalid', `Unstick ${change.id} plans ${change.plan.length} step(s); there is no step ${step}`); + } + const agent = this.agents.get(change.agent); + if (!agent) throw new OperatorActionError('unknown-agent', `Unknown agent: ${change.agent}`); + const op = this.unstickLedger().ops.get(change.id); + const journaled = op?.steps.find((s) => s.step === step); + if (journaled?.status === 'shed') return { step, shedIds: journaled.messageIds, alreadyApplied: true }; + if (step > 1) { + const previous = op?.attempts.find((a) => a.step === step - 1); + if (previous?.status !== 'completed' || previous.outcome !== 'refused') { + throw new OperatorActionError( + 'invalid', + `Unstick ${change.id}: step ${step} follows only a refused re-run after step ${step - 1}`, + ); + } + } + const branch = this.store.currentBranch().name; + if (branch !== change.sourceBranch) stale(`the active branch moved from ${change.sourceBranch} to ${branch}`); + const planned = change.plan[step - 1]!; + const cm = agent.getContextManager(); + const byId = new Map(cm.getAllMessages().map((m) => [String(m.id), m])); + planned.messageIds.forEach((id, i) => { + const message = byId.get(id); + // An interrupted step may already have removed some of its exchange. + if (!message) { + if (!journaled) stale(`message ${id}, planned for step ${step}, is gone`); + return; } + if (this.messageFingerprint(message) !== planned.fingerprints[i]) { + stale(`message ${id}, planned for step ${step}, changed since it was approved`); + } + }); + if (!journaled) { + this.journalUnstick( + { kind: 'step-intent', operationId: change.id, agent: change.agent, step, messageIds: planned.messageIds }, + { durable: true }, + ); + } + for (const id of planned.messageIds) { + if (byId.has(id)) cm.removeMessage(id as MessageId); + } + const lastRefusal = op?.attempts.filter((a) => a.outcome === 'refused').at(-1)?.category; + this.updateRewindMarker(agent, lastRefusal ?? 'unknown'); + this.journalUnstick({ kind: 'step-done', operationId: change.id, step }, { afterCommittedState: true }); + this.recordOperatorAction({ + kind: 'unstick-step', + agent: change.agent, + ...(change.requester ? { requester: change.requester } : {}), + params: { changeId: change.id, step, of: change.plan.length, ...(admission ? { admission: admission.id } : {}) }, + result: { shedIds: planned.messageIds, ...(journaled ? { finishedInterrupted: true } : {}) }, + }); + return { step, shedIds: planned.messageIds }; + } + + /** + * Re-run the agent after step `step` of a planned unstick, outside any + * lease. Bound to the operation and step: an attempt already journaled is + * reported, never launched again. A new one is journaled `launched` + * durably before its one inference is queued, and the promise resolves + * with its outcome when the turn ends. An attempt journaled `launched` + * that no process is running reports `interrupted`: its outcome is unknown. + */ + async rerunUnstick(change: ResolvedUnstickChange, opts: { step: number }): Promise { + const step = opts.step; + const op = this.unstickLedger().ops.get(change.id); + if (op?.steps.find((s) => s.step === step)?.status !== 'shed') { + throw new OperatorActionError('invalid', `Unstick ${change.id}: step ${step} hasn't been applied, so there is nothing to re-run`); + } + const key = `${change.id}#${step}`; + const existing = op.attempts.find((a) => a.step === step); + if (existing) { + if (existing.status === 'completed') { + return { + step, + status: 'completed', + outcome: existing.outcome!, + ...(existing.category ? { category: existing.category } : {}), + ...(existing.error ? { error: existing.error } : {}), + }; + } + return this.unstickAttemptWaiters.get(key)?.promise ?? { step, status: 'interrupted' }; + } + this.journalUnstick({ kind: 'attempt-launched', operationId: change.id, agent: change.agent, step }, { durable: true }); + let resolve!: (outcome: UnstickAttemptOutcome) => void; + const promise = new Promise((r) => { resolve = r; }); + this.unstickAttemptWaiters.set(key, { promise, resolve }); + this.pendingRequests.push({ + agentName: change.agent, + reason: 'unstick-attempt', + source: 'framework', + timestamp: Date.now(), + unstick: { operationId: change.id, step }, + }); + this.recordOperatorAction({ + kind: 'unstick-attempt', + agent: change.agent, + ...(change.requester ? { requester: change.requester } : {}), + params: { changeId: change.id, step }, + result: { launched: true }, + }); + return promise; + } + + /** Record an unstick attempt's outcome once, and resolve whoever waits. */ + private settleUnstickAttempt( + binding: { operationId: string; step: number }, + outcome: 'responded' | 'refused' | 'failed', + detail?: { category?: string; error?: string }, + ): void { + const op = this.unstickLedger().ops.get(binding.operationId); + const attempt = op?.attempts.find((a) => a.step === binding.step); + if (!attempt || attempt.status === 'completed') return; + const result: UnstickAttemptOutcome = { + step: binding.step, + status: 'completed', + outcome, + ...(detail?.category ? { category: detail.category } : {}), + ...(detail?.error ? { error: detail.error } : {}), + }; + try { + this.journalUnstick( + { kind: 'attempt-done', operationId: binding.operationId, step: binding.step, outcome, ...detail }, + { afterCommittedState: true }, + ); } catch (error) { - return { - ok: false, - error: error instanceof Error ? error.message : String(error), - ...(error instanceof OperatorActionError ? { code: error.code } : {}), - requested, - }; + // Not journaled: after a restart this attempt reads `interrupted`, an + // honest unknown. This process still reports what it saw. + console.error( + `[unstick] could not journal attempt ${binding.step} of ${binding.operationId}: ` + + `${error instanceof Error ? error.message : String(error)}`, + ); } - console.error( - `[host-command] undo agent=${agentName} requested=${requested} undone=${applied.undone} ` + - `as one cut (${change.id}) by=${requester.name ?? requester.id ?? requester.via}`, - ); - return { ok: true, undone: applied.undone, requested, lastVisible: await this.lastVisiblePreview(agentName) }; + // Whoever waits hears the outcome only once the turn has fully settled + // (releaseUnstickAttempt, at the end of the stream's teardown): a caller + // that takes the next lease then finds the agent's turn truly over. + this.unstickSettled.set(`${binding.operationId}#${binding.step}`, result); + } + + /** Resolve an unstick re-run's waiter with its recorded outcome, at the + * end of its stream's teardown. */ + private releaseUnstickAttempt(binding: { operationId: string; step: number }): void { + const key = `${binding.operationId}#${binding.step}`; + const result = this.unstickSettled.get(key); + if (!result) return; + this.unstickSettled.delete(key); + this.unstickAttemptWaiters.get(key)?.resolve(result); + this.unstickAttemptWaiters.delete(key); + } + + /** + * host/command unstick when the host has an operator-change gate: plan it + * in full and stage it, returning the receipt. Nothing changes now; the + * host drives the plan (applyResolvedOperatorChange per step under a + * lease, rerunUnstick between steps). Without a gate, host/command keeps + * its in-stream loop. + */ + private async gatedUnstick( + agentName: string, + cap: number, + requester: OperatorRequester, + ): Promise<{ ok: boolean; error?: string; code?: string; staged?: { id: string }; cap?: number }> { + const change = this.resolveUnstick(agentName, cap, requester); + if (!change) return { ok: false, error: `Cannot unstick ${agentName}: nothing to shed` }; + const answer = await this.askOperatorChangeGate(change); + if (answer.outcome === 'gate-failed') { + return { ok: false, code: 'gate-failed', error: `Operator change refused: ${answer.error}` }; + } + return { ok: false, code: 'staged', error: answer.receipt.text, staged: { id: answer.receipt.id }, cap: change.plan.length }; } /** Resolve host/command undo by turns to the exact checkpoints it would @@ -7032,15 +7324,29 @@ export class AgentFramework { private finishUndoTurnsCut(change: ResolvedUndoTurnsChange, admission?: OperatorAdmission): void { const destination = this.undoTurnsDestination(change); const oldest = change.checkpoints[change.checkpoints.length - 1]!; - const framework = this.store.getStateJson(FRAMEWORK_STATE_ID); - const state = (framework && typeof framework === 'object' ? framework : {}) as Record; - if ((state.operatorCut as { changeId?: unknown } | undefined)?.changeId !== change.id) { - this.store.setStateJson(FRAMEWORK_STATE_ID, { ...state, operatorCut: { changeId: change.id, at: Date.now() } }); + const marker = (): { changeId?: unknown; completed?: unknown } | undefined => { + const framework = this.store.getStateJson(FRAMEWORK_STATE_ID); + return (framework && typeof framework === 'object' ? framework as Record : {}).operatorCut as + { changeId?: unknown; completed?: unknown } | undefined; + }; + const writeMarker = (completed: boolean) => { + const framework = this.store.getStateJson(FRAMEWORK_STATE_ID); + this.store.setStateJson(FRAMEWORK_STATE_ID, { + ...(framework && typeof framework === 'object' ? framework as Record : {}), + operatorCut: { changeId: change.id, at: Date.now(), ...(completed ? { completed: true } : {}) }, + }); + }; + // A completed cut is left exactly as it is, whatever happened since on + // its branch (later turns and their checkpoints, a redo stack): only + // its durability is confirmed again. + if (marker()?.changeId === change.id && marker()?.completed === true) { + this.store.sync(); + return; } - this.saveTurnCheckpoints( - change.agent, - this.getTurnCheckpoints(change.agent).filter((c) => c.sequenceBefore < oldest.sequenceBefore), - ); + if (marker()?.changeId !== change.id) writeMarker(false); + // Checkpoints need no bookkeeping: the lists are branch-scoped, and the + // destination, cut where the oldest staged turn had yet to record + // itself, inherits exactly the older ones. Later turns add their own. this.materializeConfigMountAfterBranchSwitch(); let redoStack = this.redoStacks.get(change.agent); if (!redoStack) { @@ -7048,10 +7354,17 @@ export class AgentFramework { this.redoStacks.set(change.agent, redoStack); } // The source tip still holds the removed checkpoints: redo re-adds none. - if (!redoStack.some((entry) => entry.changeId === change.id)) { + // A turn that already ran on the destination (it recorded a checkpoint + // there) invalidated redo, as any new work does, so a repair pushes no + // entry then: redoing would discard that work. + const laterWork = this.getTurnCheckpoints(change.agent).some((c) => c.branchName === destination); + if (!laterWork && !redoStack.some((entry) => entry.changeId === change.id)) { redoStack.push({ branchName: change.sourceBranch, checkpoints: [], changeId: change.id }); } - if (this.completedOperatorCuts.has(change.id)) return; + if (this.completedOperatorCuts.has(change.id)) { + this.completeUndoTurnsCut(change); + return; + } this.emitTrace({ type: 'undo:completed', agentName: change.agent, @@ -7074,6 +7387,20 @@ export class AgentFramework { result: { sourceBranch: change.sourceBranch, targetBranch: destination }, }); this.completedOperatorCuts.add(change.id); + this.completeUndoTurnsCut(change); + } + + /** Mark an admitted undo cut complete on its destination and make that + * durable: branch state persists only on sync(), so completion is + * reported only after the sync succeeds (a failure throws, and a retry + * confirms it again). */ + private completeUndoTurnsCut(change: ResolvedUndoTurnsChange): void { + const framework = this.store.getStateJson(FRAMEWORK_STATE_ID); + this.store.setStateJson(FRAMEWORK_STATE_ID, { + ...(framework && typeof framework === 'object' ? framework as Record : {}), + operatorCut: { changeId: change.id, at: Date.now(), completed: true }, + }); + this.store.sync(); } /** @@ -7088,9 +7415,15 @@ export class AgentFramework { */ async applyResolvedOperatorChange( change: ResolvedOperatorChange, - opts: { lease: SafeBoundaryLease; admission: OperatorAdmission }, + opts: { lease: SafeBoundaryLease; admission: OperatorAdmission; step?: number }, ): Promise { this.heldLeaseTokens(opts.lease, `apply ${change.kind} for ${change.agent}`); + if (change.kind === 'unstick') { + if (opts.step === undefined) { + throw new OperatorActionError('invalid', `Applying unstick ${change.id} takes a step (1 to ${change.plan.length})`); + } + return { kind: 'unstick', ...this.applyUnstickStep(change, opts.step, opts.admission) }; + } if (change.kind === 'undo-turns') { const applied = this.applyUndoTurns(change, opts.admission); // Installs the MCPL data-plane gate now, under the lease, and drains in @@ -8461,6 +8794,10 @@ export class AgentFramework { timestamp: Date.now(), suppressProse: this.activeTurnTriggers.get(agent.name)?.suppressProse, ephemeralSystemPrompt: this.activeTurnTriggers.get(agent.name)?.ephemeralSystemPrompt, + // The restart continues an unstick re-run: it keeps the binding. + ...(this.activeTurnTriggers.get(agent.name)?.unstick + ? { unstick: this.activeTurnTriggers.get(agent.name)!.unstick } + : {}), }); } else if (currentState.stream) { // Streaming path: convert results and resume the stream. @@ -10037,8 +10374,12 @@ export class AgentFramework { // A silent tick never suppresses a genuine coalesced wake. If any // ordinary request shares this batch, the ordinary turn semantics win. const silentOnly = requests.every((r) => r.suppressProse === true); + // An unstick re-run in the batch binds the turn to its operation and + // step, whichever request leads the batch. + const unstickReq = requests.find((r) => r.unstick); await this.startAgentStream(agent, { ...trigger, + ...(unstickReq ? { unstick: unstickReq.unstick } : {}), suppressProse: silentOnly ? trigger?.suppressProse : undefined, ephemeralSystemPrompt: silentOnly ? trigger?.ephemeralSystemPrompt : undefined, channelId: channelReq?.channelId, @@ -11676,7 +12017,14 @@ export class AgentFramework { // config; both are bounded. let handledByRewind = false; const rh = agent.refusalHandling; - const forced = this.forcedRewind.get(agent.name); + // An admitted unstick's re-run never sheds in the stream: it + // records the refusal, and the host decides the next step. + const unstickRun = trigger?.unstick; + if (unstickRun) { + this.settleUnstickAttempt(unstickRun, 'refused', { category }); + handledByRewind = true; + } + const forced = unstickRun ? undefined : this.forcedRewind.get(agent.name); // NOTE: plain retries are NOT done here. `refusalHandling.retries` // is handed to membrane (see Agent.startActivation), which @@ -11723,7 +12071,7 @@ export class AgentFramework { }, () => this.finishUnstick(agent.name, false, category), ); - } else if (rh?.autoRewind && !guardRefusal) { + } else if (rh?.autoRewind && !guardRefusal && !unstickRun) { const cap = Math.max(1, rh.maxRewinds ?? 3); const used = this.refusalRewinds.get(agent.name) ?? 0; doRewind( @@ -11760,6 +12108,7 @@ export class AgentFramework { // A turn that completed WITHOUT a refusal ends the rewind episode: // the model responded. Leave the consolidated marker in place as // the durable record; just clear the per-episode counters. + if (trigger?.unstick) this.settleUnstickAttempt(trigger.unstick, 'responded'); if (this.forcedRewind.has(agent.name)) { this.finishUnstick(agent.name, true); } @@ -12302,10 +12651,18 @@ export class AgentFramework { durationMs, }); this.abortAgentScript(agent.name, 'stream threw'); + if (trigger?.unstick) this.settleUnstickAttempt(trigger.unstick, 'failed', { error: err.message }); lifecyclePhase = 'failed'; // §10.5 — terminal emitted in finally agent.reset(); if (this.agents.get(agent.name) === agent && agent.streamId === myStreamId) this.eventGate?.onInferenceEnded(agent.name); } finally { + // An unstick re-run that ended without an outcome (aborted, cancelled) + // failed, unless a budget restart carries it on. + if (trigger?.unstick && !this.pendingRequests.some((r) => + r.agentName === agent.name && r.unstick?.operationId === trigger.unstick!.operationId + && r.unstick.step === trigger.unstick!.step)) { + this.settleUnstickAttempt(trigger.unstick, 'failed', { error: 'the re-run ended without a response' }); + } const unsettledKvSubmissions = drainKvSubmissionIds?.() ?? []; if (unsettledKvSubmissions.length > 0) { try { @@ -12433,6 +12790,8 @@ export class AgentFramework { } this.ackDeferredWrites(); } + // Last: the turn has settled, so an unstick re-run's caller may act. + if (trigger?.unstick) this.releaseUnstickAttempt(trigger.unstick); } } @@ -12509,11 +12868,11 @@ export class AgentFramework { ): Promise<{ toolUseId: string | null; result: ToolResult; staged?: OperatorChangeReceipt }> { if (!this.agents.get(agentName)) throw new Error(`Unknown agent: ${agentName}`); // An operator's mutating self-change (agent_settings update/reset/cancel, - // a tool-presentation edit) goes through the host's gate before anything - // runs. Staged: nothing executes and no pair is stored, since a pair - // would testify the tool ran; the receipt comes back with toolUseId null. - // Apply: the call proceeds as exactly the resolved change, revalidated, - // and admitted for its own duration only. + // a tool-presentation edit) is staged with the host's gate, not run: + // no pair is stored, since a pair would testify the tool ran, and the + // receipt comes back with toolUseId null. The change runs only when the + // host applies it (applyResolvedOperatorChange under its lease), which + // calls back here carrying the admission for exactly this change. if ( this.operatorChangeGate && selfChangeKind(toolName, input) && @@ -12534,26 +12893,14 @@ export class AgentFramework { }; } const answer = await this.askOperatorChangeGate(change); - if (answer.outcome === 'staged') { - return { - toolUseId: null, - result: { success: false, isError: true, error: answer.receipt.text }, - staged: answer.receipt, - }; - } if (answer.outcome === 'gate-failed') { throw new Error(`puppet refused: the operator-change gate failed (${answer.error})`); } - // Revalidated here and again where the change runs (gatedSelfChange's - // admitted path), right before the handler, with no await between. - this.revalidateSelfChange(change); - const admission: OperatorAdmission = { id: change.id }; - this.activeAdmissions.set(admission.id, change); - try { - return await this.puppetToolCallUngated(agentName, toolName, input, { ...opts, admission }); - } finally { - this.activeAdmissions.delete(admission.id); - } + return { + toolUseId: null, + result: { success: false, isError: true, error: answer.receipt.text }, + staged: answer.receipt, + }; } return this.puppetToolCallUngated(agentName, toolName, input, opts); } @@ -12628,16 +12975,14 @@ export class AgentFramework { let ownedToEnd = false; try { const started = Date.now(); - // The call, and anything it delegates, runs as this turn: a gated - // change it makes to this agent is the caller's own, not a busy agent. - const result = await turnScope.run({ agent: agentName, token: turnToken }, () => this.executeToolCall({ + const result = await this.executeToolCall({ id: toolUseId, name: toolName, input, callerAgentName: agentName, origin: 'puppet', ...(opts?.admission ? { admission: opts.admission } : {}), - })); + }); const durationMs = Date.now() - started; // Store the pair through the same shapes the ordinary path uses. Build diff --git a/src/index.ts b/src/index.ts index c924be5b5..2ee714ad0 100644 --- a/src/index.ts +++ b/src/index.ts @@ -5,7 +5,6 @@ export * from './types/index.js'; export { AgentFramework, BudgetPreflightError, ResumeBlockedError } from './framework.js'; export type { RuntimeSettingsPreview, HostModeStatus, SafeBoundaryLease, SafeBoundaryOptions } from './framework.js'; export type { - OperatorChangeDecision, OperatorChangeReceipt, ResolvedOperatorChange, ResolvedSettingsChange, @@ -13,6 +12,11 @@ export type { RuntimeSettingsValues, AppliedOperatorChange, ResolvedUndoTurnsChange, + ResolvedUnstickChange, + UnstickOperationRecord, + UnstickStepRecord, + UnstickAttemptRecord, + UnstickAttemptOutcome, } from './operator-change.js'; export { Agent } from './agent.js'; export type { StartStreamResult } from './agent.js'; diff --git a/src/operator-change.ts b/src/operator-change.ts index db099b563..2abc4bbbc 100644 --- a/src/operator-change.ts +++ b/src/operator-change.ts @@ -1,24 +1,20 @@ /** * Operator-change admission: the framework resolves each of its own operator - * mutations to an absolute change and asks the host's gate before acting - * (FrameworkConfig.operatorChangeGate). The gate answers `apply` (act now, - * on exactly this change) or `staged` (the host will apply it later, at a - * safe boundary, through AgentFramework.applyResolvedOperatorChange). + * mutations to an absolute change and hands it to the host's gate + * (FrameworkConfig.operatorChangeGate), which stages it and answers with a + * receipt. Nothing mutates then. There is one way to apply a staged change: + * AgentFramework.applyResolvedOperatorChange, under a held safe-boundary + * lease, whenever the host finds it eligible. * - * A change is resolved once. Application, immediate or delayed, revalidates - * it against stable identity (the active branch, the settings it was - * resolved from) and applies exactly what was resolved, never the relative - * command again: a change that no longer holds is refused as `stale` and the - * host restages it. + * A change is resolved once. Application revalidates it against stable + * identity (the active branch, the settings it was resolved from) and + * applies exactly what was resolved, never the relative command again: a + * change that no longer holds is refused as `stale` and the host restages it. */ import type { OperatorRequester } from './operator-log.js'; -/** What the host's gate decided for one resolved change. */ -export type OperatorChangeDecision = - | { decision: 'apply' } - | { decision: 'staged'; receipt: OperatorChangeReceipt }; - -/** The host's receipt for a staged change, shown to whoever asked. */ +/** The host's receipt for a staged change, shown to whoever asked: the + * change is accepted and queued, not applied. */ export interface OperatorChangeReceipt { id: string; text: string; @@ -79,10 +75,66 @@ export interface ResolvedUndoTurnsChange extends ResolvedOperatorChangeBase { checkpoints: Array<{ turnIndex: number; sequenceBefore: number; branchName: string }>; } +/** + * host/command unstick, planned in full at staging: exchanges shed + * newest-first, one per step, re-running the agent after each, until it + * stops refusing or the plan is spent. The plan walks back from the tail as + * it stood at staging and names every exchange that may go, so whoever is + * asked approves a known range; nothing that arrives later is ever chosen. + * The host drives it: applyResolvedOperatorChange per step under a lease, + * rerunUnstick between steps outside any lease. AF keeps the durable record + * (getUnstickOperation) in a store-bound, branch-independent journal. + */ +export interface ResolvedUnstickChange extends ResolvedOperatorChangeBase { + kind: 'unstick'; + /** The operator's cap (1 to 10); `plan` is shorter when history runs out. */ + cap: number; + /** Step k sheds exactly plan[k-1]'s messages, newest exchange first. Each + * message carries a fingerprint of its content, so an edit in place + * makes the step stale rather than shedding something not approved. */ + plan: Array<{ step: number; messageIds: string[]; fingerprints: string[] }>; +} + export type ResolvedOperatorChange = | ResolvedSettingsChange | ResolvedPresentationChange - | ResolvedUndoTurnsChange; + | ResolvedUndoTurnsChange + | ResolvedUnstickChange; + +/** One step of an unstick operation, as journaled: `intent` before its shed + * (with exactly what it will remove), `shed` once it is done. */ +export interface UnstickStepRecord { + step: number; + status: 'intent' | 'shed'; + messageIds: string[]; +} + +/** One re-run of an unstick operation, as journaled. `launched` is written + * durably before the inference is queued. An attempt still `launched` when + * no process is running it was interrupted: its outcome is unknown, and it + * is never relaunched. */ +export interface UnstickAttemptRecord { + step: number; + status: 'launched' | 'completed'; + outcome?: 'responded' | 'refused' | 'failed'; + category?: string; + error?: string; +} + +/** The durable record of an unstick operation (getUnstickOperation). */ +export interface UnstickOperationRecord { + operationId: string; + agent: string; + steps: UnstickStepRecord[]; + attempts: UnstickAttemptRecord[]; +} + +/** What rerunUnstick reports for a step's attempt. `interrupted`: launched, + * but no outcome was recorded and no process is running it; it may not have + * started, or may have finished without a receipt. */ +export type UnstickAttemptOutcome = + | { step: number; status: 'completed'; outcome: 'responded' | 'refused' | 'failed'; category?: string; error?: string } + | { step: number; status: 'interrupted' }; /** What applyResolvedOperatorChange did. */ export type AppliedOperatorChange = @@ -97,6 +149,14 @@ export type AppliedOperatorChange = /** The cut had already been applied (the active branch is its * destination): nothing changed this time. */ alreadyApplied?: true; + } + | { + kind: 'unstick'; + step: number; + /** What this step removed (its planned exchange). */ + shedIds: string[]; + /** The step was already journaled as shed: nothing changed this time. */ + alreadyApplied?: true; }; /** The self-change kind a tool call is, when it mutates the agent's own body: diff --git a/src/types/agent.ts b/src/types/agent.ts index b33fa81da..402f86f13 100644 --- a/src/types/agent.ts +++ b/src/types/agent.ts @@ -303,6 +303,9 @@ export interface CompletedToolCall { * Inference request for an agent. */ export interface InferenceRequest { + /** An admitted unstick operation's re-run (rerunUnstick): the turn records + * its outcome against this operation and step instead of shedding. */ + unstick?: { operationId: string; step: number }; /** Host-owned identity for withdrawal of an unconsumed coalesced wake. */ coalescingSubject?: string; coalescingEventId?: string; diff --git a/src/types/framework.ts b/src/types/framework.ts index 00dbb3d3c..d99c53929 100644 --- a/src/types/framework.ts +++ b/src/types/framework.ts @@ -194,17 +194,18 @@ export interface FrameworkConfig { /** * The host's gate for the framework's own operator mutations: host/command - * undo, hide and unstick, and an operator's or module's mutating - * agent_settings / tool-presentation calls. Each is resolved to an absolute - * change first; the gate answers `apply` (the framework applies exactly - * that change now, after revalidating it) or `staged` with a receipt (the - * host applies it later through applyResolvedOperatorChange). The agent's - * own model calls never reach it. A gate that throws refuses the change. - * Absent: every operator path behaves as it always has. + * undo and unstick, and an operator's or module's mutating agent_settings / + * tool-presentation calls. Each is resolved to an absolute change, which + * the gate stages (records as a revision) and answers with its receipt: + * the change is accepted and queued, not applied. Nothing mutates at that + * point. The host applies a staged change, whenever it is eligible, only + * through applyResolvedOperatorChange under a held safe-boundary lease. A + * gate that throws refuses the change. The agent's own model calls never + * reach it. Absent: every operator path behaves as it always has. */ operatorChangeGate?: ( change: import('../operator-change.js').ResolvedOperatorChange, - ) => Promise; + ) => Promise; /** Interval for periodic store sync in milliseconds (default: 1000ms, 0 to disable) */ syncIntervalMs?: number; diff --git a/test/module-targeted-delivery.test.ts b/test/module-targeted-delivery.test.ts index 767f781f6..7ecc93f16 100644 --- a/test/module-targeted-delivery.test.ts +++ b/test/module-targeted-delivery.test.ts @@ -231,3 +231,67 @@ describe('ModuleContext.addMessage durable delivery across a hard kill', () => { } }); }); + +describe('ModuleContext.addMessage durable delivery with no recovery path (store only)', () => { + const anchor = fileURLToPath(import.meta.url); + const index = fileURLToPath(new URL('../src/index.js', import.meta.url)); + const mock = fileURLToPath(new URL('./helpers/mock-membrane.js', import.meta.url)); + const childScript = (dir: string) => { + const script = join(dir, 'child.mjs'); + writeFileSync(script, ` + import { createRequire } from 'node:module'; + import { AgentFramework } from ${JSON.stringify(index)}; + import { MockMembrane } from ${JSON.stringify(mock)}; + import { writeFileSync } from 'node:fs'; + import { join } from 'node:path'; + const { JsStore } = createRequire(${JSON.stringify(anchor)})('@animalabs/chronicle'); + const [mode, dir] = process.argv.slice(2); + const courier = { name: 'courier', async start(c) { this.ctx = c; }, async stop() {}, getTools() { return []; }, + async handleToolCall() { return { success: true }; }, async onProcess() { return {}; } }; + console.log = () => {}; console.error = () => {}; + const store = JsStore.openOrCreate({ path: join(dir, 'store') }); + const fw = await AgentFramework.create({ store, membrane: new MockMembrane().asMembrane(), + agents: [{ name: 'scout', model: 'test-model', systemPrompt: 'test' }], modules: [courier] }); + if (mode === 'check') { + const all = JSON.stringify(fw.getAgent('scout').getContextManager().getAllMessages()); + writeFileSync(join(dir, 'count.json'), JSON.stringify(all.split('STORE_ONLY_NOTICE').length - 1)); + await fw.stop(); + process.exit(0); + } + fw.getStore().sync(); + if (mode === 'failing-sync') fw.getStore().sync = () => { throw new Error('injected sync failure'); }; + const placement = {}; + courier.ctx.addMessage('user', [{ type: 'text', text: 'STORE_ONLY_NOTICE' }], undefined, { forAgent: 'scout', durable: true, placement }); + writeFileSync(join(dir, 'receipt.json'), JSON.stringify(placement)); + process.kill(process.pid, 'SIGKILL'); + `); + return script; + }; + const run = (script: string, mode: string, dir: string) => spawnSync(process.execPath, [script, mode, dir], { encoding: 'utf8' }); + + it('says durable only after its sync succeeded, and keeps the message through a kill', () => { + const dir = mkdtempSync(join(tmpdir(), 'store-only-durable-')); + try { + const script = childScript(dir); + assert.equal(run(script, 'deliver', dir).signal, 'SIGKILL'); + assert.equal((JSON.parse(readFileSync(join(dir, 'receipt.json'), 'utf8')) as MessagePlacement).durable, true); + const check = run(script, 'check', dir); + assert.equal(check.status, 0, check.stderr); + assert.equal(JSON.parse(readFileSync(join(dir, 'count.json'), 'utf8')), 1, 'present after the kill, exactly once'); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it('reports a delivery unconfirmed when its sync fails, rather than claiming a buffered queue durable', () => { + const dir = mkdtempSync(join(tmpdir(), 'store-only-unconfirmed-')); + try { + const script = childScript(dir); + assert.equal(run(script, 'failing-sync', dir).signal, 'SIGKILL'); + assert.equal((JSON.parse(readFileSync(join(dir, 'receipt.json'), 'utf8')) as MessagePlacement).durable, false, + 'neither the queue in branch state nor the message was ever synced'); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); +}); diff --git a/test/operator-change-gate.test.ts b/test/operator-change-gate.test.ts index 8cc9abebb..6754d2635 100644 --- a/test/operator-change-gate.test.ts +++ b/test/operator-change-gate.test.ts @@ -21,7 +21,7 @@ import type { InferenceRequest, Module, ModuleContext, - OperatorChangeDecision, + OperatorChangeReceipt, ResolvedOperatorChange, ToolCall, ToolDefinition, @@ -103,11 +103,11 @@ describe('operator-change gate: self-change tools', () => { let mover: Mover; let knobs: Knobs; let asked: ResolvedOperatorChange[]; - let decide: (change: ResolvedOperatorChange) => Promise; + let decide: (change: ResolvedOperatorChange) => Promise; let quiet: typeof console.log; - const staged = async (change: ResolvedOperatorChange): Promise => - ({ decision: 'staged', receipt: { id: `rev-${asked.length}`, text: `staged ${change.kind} as rev-${asked.length}; waiting for scout` } }); + const staged = async (change: ResolvedOperatorChange): Promise => + ({ id: `rev-${asked.length}`, text: `staged ${change.kind} as rev-${asked.length}; waiting for scout` }); beforeEach(async () => { tempDir = mkdtempSync(join(tmpdir(), 'operator-gate-')); @@ -155,17 +155,6 @@ describe('operator-change gate: self-change tools', () => { assert.ok(framework.getOperatorLog().some((e) => e.kind === 'operator-change-staged')); }); - it('applies at once on apply, asking only once even though the call goes through two checks', async () => { - decide = async () => ({ decision: 'apply' }); - const out = await framework.puppetToolCall('scout', 'agent_settings', { action: 'update', context_budget_tokens: 120_000 }); - assert.equal(out.result.success, true, String(out.result.error)); - assert.equal(typeof out.toolUseId, 'string'); - assert.equal(budget(), 120_000); - assert.equal(pairs(), 1); - assert.equal(asked.length, 1); - assert.equal((framework as unknown as Internals).activeAdmissions.size, 0, 'the admission ended with the call'); - }); - it('refuses when the gate fails, applying nothing', async () => { decide = async () => { throw new Error('journal unavailable'); }; const before = budget(); @@ -274,58 +263,14 @@ describe('operator-change gate: self-change tools', () => { ); }); - it("refuses to apply while the store is held by a lease it isn't running under, and applies inside one", async () => { - decide = async () => ({ decision: 'apply' }); - const before = budget(); - let releaseLease!: () => void; - const holding = new Promise((r) => { releaseLease = r; }); - let granted!: () => void; - const isGranted = new Promise((r) => { granted = r; }); - const leased = framework.runAtSafeBoundary({ verb: 'held elsewhere' }, async () => { granted(); await holding; }); - await isGranted; - const outside = await mover.ctx.callTool!({ id: 'own', name: 'agent_settings', input: { action: 'update', context_budget_tokens: 150_000 }, callerAgentName: 'scout' }); - assert.match(String(outside.error), /the store is held by held elsewhere/); - assert.equal(budget(), before, 'permission to apply is not a safe boundary'); - releaseLease(); - await leased; - - await framework.runAtSafeBoundary({ verb: 'mine' }, async () => { - const inside = await mover.ctx.callTool!({ id: 'own2', name: 'agent_settings', input: { action: 'update', context_budget_tokens: 150_000 }, callerAgentName: 'scout' }); - assert.equal(inside.success, true, String(inside.error)); - }); - assert.equal(budget(), 150_000, 'the lease holder can apply under its own lease'); - }); - - it("refuses to apply while another call owns the agent's live turn", async () => { - decide = async () => ({ decision: 'apply' }); + it('stages a delegated change made after its puppet call returned, never running it', async () => { const before = budget(); - const entered = new Promise((resolve) => { mover.entered = resolve; }); - const holding = framework.puppetToolCall('scout', 'mover--hold', {}); - await entered; // the puppet now owns scout's turn token - const outside = await mover.ctx.callTool!({ id: 'own', name: 'agent_settings', input: { action: 'update', context_budget_tokens: 150_000 }, callerAgentName: 'scout' }); - assert.match(String(outside.error), /scout has a live turn that isn't this call's/); + await framework.puppetToolCall('scout', 'mover--set_budget_later', {}); // a module tool: runs, schedules the change + (mover as unknown as { fireLater: () => void }).fireLater(); + const late = await mover.later!; + assert.match(String(late.error), /staged agent-settings/, 'it keeps its actor, so it is gated, and only staged'); + assert.equal(asked.at(-1)!.surface, 'puppet'); assert.equal(budget(), before); - mover.release!(); - await holding; - }); - - it("applies a puppet's own delegated change during its turn, but not from a late callback in a later turn", async () => { - decide = async () => ({ decision: 'apply' }); - const own = await framework.puppetToolCall('scout', 'mover--set_budget', {}); - assert.equal(own.result.success, true, String(own.result.error)); - assert.equal(budget(), 130_000, 'the puppet owns its turn'); - - await framework.puppetToolCall('scout', 'mover--set_budget_later', {}); // returns; the change is still to come - const tokens = (framework as unknown as { activeTurnTokens: Map }).activeTurnTokens; - tokens.set('scout', 987_654); // a later turn is alive when the callback fires - try { - (mover as unknown as { fireLater: () => void }).fireLater(); - const late = await mover.later!; - assert.match(String(late.error), /live turn that isn't this call's/, 'provenance kept, permission not'); - assert.equal(budget(), 130_000, 'unchanged by the late callback'); - } finally { - tokens.delete('scout'); - } }); it('admits only the exact call its admission was registered for, and revalidates it as it runs', async () => { @@ -444,7 +389,7 @@ describe('operator-change gate: tool presentation', () => { modules: [workspace], operatorChangeGate: async (change) => { asked.push(structuredClone(change)); - return { decision: 'staged', receipt: { id: 'rev-p', text: 'staged' } }; + return { id: 'rev-p', text: 'staged' }; }, }); const quiet = console.log; @@ -485,7 +430,7 @@ describe('operator-change gate: host/command undo by turns', () => { let membrane: MockMembrane; let framework: AgentFramework; let asked: ResolvedOperatorChange[]; - let decide: (change: ResolvedOperatorChange) => Promise; + let decide: (change: ResolvedOperatorChange) => Promise; let quiet: { log: typeof console.log; error: typeof console.error }; type HostInternals = { pendingRequests: InferenceRequest[]; @@ -501,7 +446,7 @@ describe('operator-change gate: host/command undo by turns', () => { tempDir = mkdtempSync(join(tmpdir(), 'operator-gate-undo-')); membrane = new MockMembrane(); asked = []; - decide = async (change) => ({ decision: 'staged', receipt: { id: 'rev-u', text: `staged ${change.kind}` } }); + decide = async (change) => ({ id: 'rev-u', text: `staged ${change.kind}` }); framework = await AgentFramework.create({ storePath: join(tempDir, 'test.chronicle'), membrane: membrane.asMembrane(), @@ -546,14 +491,16 @@ describe('operator-change gate: host/command undo by turns', () => { assert.equal(change.sourceBranch, before.branch); }); - it('applies an admitted multi-turn undo as one cut with one redo entry', async () => { - decide = async () => ({ decision: 'apply' }); + it('applies a staged multi-turn undo as one cut with one redo entry', async () => { await turn('one'); await turn('two'); await turn('three'); const source = branch(); const r = await undo(2); - assert.equal(r.ok, true, String(r.error)); - assert.deepEqual([r.undone, r.requested], [2, 2]); - assert.equal(branch(), `undo/scout/op-${asked[0]!.id}`, 'the destination is named for the change'); + assert.deepEqual([r.ok, r.code], [false, 'staged'], 'host/command only stages'); + const change = asked[0]!; + const applied = await framework.runAtSafeBoundary({ verb: 'apply' }, (lease) => + framework.applyResolvedOperatorChange(change, { lease, admission: { id: 'rev-u' } })); + assert.deepEqual(applied.kind === 'undo-turns' && [applied.undone, applied.requested], [2, 2]); + assert.equal(branch(), `undo/scout/op-${change.id}`, 'the destination is named for the change'); // A turn's checkpoint is taken after its input landed, so undoing turns // keeps the oldest undone turn's input, exactly as the per-turn loop does. assert.deepEqual(texts(), ['one', 'reply to one', 'two'], 'turns two and three are undone'); @@ -627,37 +574,49 @@ describe('operator-change gate: host/command undo by turns', () => { ); }); - for (const seam of ['marker', 'checkpoints', 'log'] as const) { - it(`finishes the cut's bookkeeping on retry after a failure at the ${seam} write, redo included`, async () => { + for (const [seam, later] of (['marker', 'log', 'completion', 'completion sync'] as const).flatMap((x) => [[x, false], [x, true]] as const)) { + it(`finishes the cut's bookkeeping on retry after a failure at the ${seam} write${later ? ', keeping later work' : ', redo included'}`, async () => { await turn('one'); await turn('two'); await turn('three'); const source = branch(); await undo(2); const change = asked[0]!; - const fw = framework as unknown as { - saveTurnCheckpoints: (agent: string, list: unknown[]) => void; - recordOperatorAction: (entry: { kind: string }) => void; - }; - const store = framework.getStore() as unknown as { setStateJson: (id: string, v: unknown) => void }; - const real = { setState: store.setStateJson.bind(store), save: fw.saveTurnCheckpoints.bind(fw), record: fw.recordOperatorAction.bind(fw) }; + const fw = framework as unknown as { recordOperatorAction: (entry: { kind: string }) => void }; + const store = framework.getStore() as unknown as { setStateJson: (id: string, v: unknown) => void; sync: () => void }; + const real = { setState: store.setStateJson.bind(store), record: fw.recordOperatorAction.bind(fw), sync: store.sync.bind(store) }; let armed = true; const fail = () => { armed = false; throw new Error(`injected ${seam} failure`); }; - if (seam === 'marker') store.setStateJson = (id, v) => (armed && (v as { operatorCut?: unknown })?.operatorCut ? fail() : real.setState(id, v)); - if (seam === 'checkpoints') fw.saveTurnCheckpoints = (agent, list) => (armed ? fail() : real.save(agent, list)); + const cut = (v: unknown) => (v as { operatorCut?: { completed?: boolean } })?.operatorCut; + if (seam === 'marker') store.setStateJson = (id, v) => (armed && cut(v) && !cut(v)!.completed ? fail() : real.setState(id, v)); + if (seam === 'completion') store.setStateJson = (id, v) => (armed && cut(v)?.completed ? fail() : real.setState(id, v)); if (seam === 'log') fw.recordOperatorAction = (entry) => (armed && entry.kind === 'undo-turns' ? fail() : real.record(entry)); + if (seam === 'completion sync') { + // The sync right after the completion marker is the barrier that + // makes completion durable: fail it once. + let marked = false; + store.setStateJson = (id, v) => { if (cut(v)?.completed) marked = true; return real.setState(id, v); }; + store.sync = () => (armed && marked ? fail() : real.sync()); + } const apply = () => framework.runAtSafeBoundary({ verb: 'apply' }, (lease) => framework.applyResolvedOperatorChange(change, { lease, admission: { id: 'rev-u' } })); try { await assert.rejects(apply(), new RegExp(`injected ${seam} failure`)); } finally { store.setStateJson = real.setState; - fw.saveTurnCheckpoints = real.save; fw.recordOperatorAction = real.record; + store.sync = real.sync; } assert.equal(branch(), `undo/scout/op-${change.id}`, 'the cut landed before the failure'); + if (later) await turn('after the failure'); // legitimate work on the destination before the retry const retried = await apply(); assert.equal(retried.kind === 'undo-turns' && retried.alreadyApplied, true, 'not cut again'); - assert.equal(host().getTurnCheckpoints('scout').length, 1); assert.equal(framework.getOperatorLog().filter((e) => e.kind === 'undo-turns').length, 1, 'logged once'); + if (later) { + assert.equal(host().getTurnCheckpoints('scout').length, 2, 'the older turn and the later one: repair kept later work'); + assert.equal(texts().at(-1), 'reply to after the failure'); + assert.equal(host().redoStacks.get('scout')?.length ?? 0, 0, 'the later work invalidated redo, and repair did not restore it'); + return; + } + assert.equal(host().getTurnCheckpoints('scout').length, 1); assert.equal(host().redoStacks.get('scout')!.length, 1, 'one redo entry, not one per attempt'); const redone = framework.redo('scout'); assert.deepEqual([redone.redone, redone.toBranch], [true, source], 'the promised one-step redo exists'); @@ -665,22 +624,55 @@ describe('operator-change gate: host/command undo by turns', () => { }); } - it('never awaits awareness delivery, and never reports its failure as the undo\'s', async () => { - decide = async () => ({ decision: 'apply' }); + it('leaves a completed cut alone on retry after further work, in process and after reopen', async () => { + await turn('one'); await turn('two'); await turn('three'); + await undo(1); + const change = asked[0]!; + const apply = () => framework.runAtSafeBoundary({ verb: 'apply' }, (lease) => + framework.applyResolvedOperatorChange(change, { lease, admission: { id: 'rev-u' } })); + await apply(); + await turn('after the cut'); // real work on the destination: a new turn and its checkpoint + const now = () => ({ + checkpoints: host().getTurnCheckpoints('scout').map((c) => c.turnIndex), + texts: texts(), + redo: host().redoStacks.get('scout')?.length ?? 0, + }); + const settled = now(); + assert.equal(settled.checkpoints.length, 3, 'two older turns and the one after the cut'); + const again = await apply(); + assert.equal(again.kind === 'undo-turns' && again.alreadyApplied, true); + assert.deepEqual(now(), settled, 'nothing touched in process'); + await framework.stop(); + framework = await AgentFramework.create({ + storePath: join(tempDir, 'test.chronicle'), + membrane: membrane.asMembrane(), + agents: [{ name: 'scout', model: 'test-model', systemPrompt: 'You are scout.', maxTokens: 1000 }], + modules: [], + operatorChangeGate: async (c) => { asked.push(structuredClone(c)); return decide(c); }, + }); + const logged = () => framework.getOperatorLog().filter((e) => e.kind === 'undo-turns').length; + const before = logged(); + const reopened = await apply(); + assert.equal(reopened.kind === 'undo-turns' && reopened.alreadyApplied, true); + assert.deepEqual(now(), { ...settled, redo: 0 }, 'nor after reopen, and no redo entry conjured'); + assert.equal(logged(), before, 'and no second record of the cut'); + }); + + + it("never awaits awareness delivery inside the lease, and never reports its failure as the undo's", async () => { const fw = framework as unknown as { discordAwarenessOutbox: unknown; syncDiscordAwarenessMarkers: () => Promise }; fw.discordAwarenessOutbox = {}; // so delivery is attempted let calls = 0; fw.syncDiscordAwarenessMarkers = () => { calls++; return calls === 1 ? Promise.reject(new Error('delivery down')) : new Promise(() => {}); }; try { await turn('one'); await turn('two'); await turn('three'); - const now = await undo(1); - assert.equal(now.ok, true, 'a delivery failure is not the undo\'s'); - decide = async (change) => ({ decision: 'staged', receipt: { id: 'rev-d', text: `staged ${change.kind}` } }); - await undo(1); - const staged = asked.at(-1)!; - const applied = await framework.runAtSafeBoundary({ verb: 'apply' }, (lease) => - framework.applyResolvedOperatorChange(staged, { lease, admission: { id: 'rev-d' } })); - assert.equal(applied.kind, 'undo-turns', 'returned while delivery is still pending'); + for (const id of ['rev-a', 'rev-b']) { + await undo(1); + const staged = asked.at(-1)!; + const applied = await framework.runAtSafeBoundary({ verb: 'apply' }, (lease) => + framework.applyResolvedOperatorChange(staged, { lease, admission: { id } })); + assert.equal(applied.kind, 'undo-turns', 'returned whether delivery failed or is still pending'); + } assert.equal(calls, 2, 'delivery started both times'); } finally { fw.discordAwarenessOutbox = null; @@ -700,11 +692,14 @@ describe('operator-change gate: host/command undo by turns', () => { }); it('resolves fewer turns than requested when history is shorter, and reports both', async () => { - decide = async () => ({ decision: 'apply' }); await turn('only'); const r = await undo(3); - assert.deepEqual([r.ok, r.undone, r.requested], [true, 1, 3]); - assert.equal(asked[0]!.kind === 'undo-turns' && asked[0]!.checkpoints.length, 1); + assert.deepEqual([r.code, r.requested], ['staged', 3]); + const change = asked[0]!; + assert.equal(change.kind === 'undo-turns' && change.checkpoints.length, 1); + const applied = await framework.runAtSafeBoundary({ verb: 'apply' }, (lease) => + framework.applyResolvedOperatorChange(change, { lease, admission: { id: 'rev-u' } })); + assert.deepEqual(applied.kind === 'undo-turns' && [applied.undone, applied.requested], [1, 3]); }); it('refuses when the gate fails, and asks nothing when there is nothing to undo', async () => { @@ -718,3 +713,197 @@ describe('operator-change gate: host/command undo by turns', () => { assert.equal(host().getTurnCheckpoints('scout').length, 1, 'nothing undone'); }); }); + +describe('operator-change gate: host/command unstick, planned at staging', () => { + let tempDir: string; + let membrane: MockMembrane; + let framework: AgentFramework; + let asked: ResolvedOperatorChange[]; + let decide: (change: ResolvedOperatorChange) => Promise; + let quiet: { log: typeof console.log; error: typeof console.error }; + type UnstickInternals = { + pendingRequests: InferenceRequest[]; + unstickJournalState: unknown; + unstickAttemptWaiters: Map; + handleHostCommand(serverId: string, params: Record): Promise>; + addMessage(participant: string, content: unknown[]): string; + }; + const host = () => framework as unknown as UnstickInternals; + + beforeEach(async () => { + tempDir = mkdtempSync(join(tmpdir(), 'operator-gate-unstick-')); + membrane = new MockMembrane(); + asked = []; + decide = async (change) => ({ id: 'rev-s', text: `staged ${change.kind}` }); + framework = await AgentFramework.create({ + storePath: join(tempDir, 'test.chronicle'), + membrane: membrane.asMembrane(), + agents: [{ name: 'scout', model: 'test-model', systemPrompt: 'You are scout.', maxTokens: 1000 }], + modules: [], + operatorChangeGate: async (change) => { asked.push(structuredClone(change)); return decide(change); }, + }); + quiet = { log: console.log, error: console.error }; + console.log = () => {}; + console.error = () => {}; + // History: two answered turns, then a message the agent keeps refusing + // (a refusal stores nothing, so the culprit is the newest message). + for (const text of ['one', 'two']) { + host().addMessage('user', [{ type: 'text', text }]); + membrane.pushResponse(createMockResponse([{ type: 'text', text: `reply to ${text}` }])); + host().pendingRequests.push({ agentName: 'scout', reason: 'mcpl:channel-incoming', source: 'test', timestamp: Date.now() } as InferenceRequest); + await framework.runUntilIdle(); + } + host().addMessage('user', [{ type: 'text', text: 'culprit' }]); + }); + afterEach(async () => { + console.log = quiet.log; + console.error = quiet.error; + await framework.stop(); + rmSync(tempDir, { recursive: true, force: true }); + }); + + const cm = () => framework.getAgent('scout')!.getContextManager(); + const texts = () => (cm().getAllMessages() as Array<{ content: Array<{ text?: string }>; metadata?: { system?: unknown } }>) + .filter((m) => !m.metadata?.system).map((m) => m.content[0]?.text ?? ''); + const textOf = (id: string) => (cm().getAllMessages() as Array<{ id: string; content: Array<{ text?: string }> }>) + .find((m) => String(m.id) === id)?.content[0]?.text; + const unstick = (maxRewinds = 3) => host().handleHostCommand('discord', { command: 'unstick', agentName: 'scout', maxRewinds, requesterName: 'nissa' }); + const refusal = () => createMockResponse([{ type: 'thinking', thinking: '', signature: 'sig' } as never], 'refusal'); + const step = (change: ResolvedOperatorChange, n: number) => framework.runAtSafeBoundary({ verb: `step ${n}` }, (lease) => + framework.applyResolvedOperatorChange(change, { lease, admission: { id: 'rev-s' }, step: n })); + const rerun = async (change: ResolvedOperatorChange, n: number) => { + const outcome = framework.rerunUnstick(change as never, { step: n }); + await framework.runUntilIdle(); + return outcome; + }; + + it('stages the whole plan, newest exchange first, with exact ids and changes nothing', async () => { + const before = texts(); + const r = await unstick(3); + assert.deepEqual([r.ok, r.code, r.cap], [false, 'staged', 3]); + const change = asked[0]!; + assert.equal(change.kind, 'unstick'); + const plan = change.kind === 'unstick' ? change.plan : []; + assert.deepEqual(plan.map((p) => p.messageIds.map(textOf)), [['culprit'], ['reply to two'], ['two']], 'the known range at risk'); + assert.ok(plan.every((p) => p.fingerprints.length === p.messageIds.length)); + assert.deepEqual(texts(), before); + }); + + it('applies planned steps under leases and re-runs between them, never shedding a later arrival', async () => { + await unstick(3); + const change = asked[0]!; + host().addMessage('user', [{ type: 'text', text: 'later arrival' }]); // e.g. the notice's answer + assert.deepEqual((await step(change, 1)).kind === 'unstick' && (await step(change, 1)), { kind: 'unstick', step: 1, shedIds: (change as { plan: Array<{ messageIds: string[] }> }).plan[0]!.messageIds, alreadyApplied: true }); + assert.deepEqual(texts(), ['one', 'reply to one', 'two', 'reply to two', 'later arrival']); + membrane.pushResponse(refusal()); + assert.deepEqual(await rerun(change, 1), { step: 1, status: 'completed', outcome: 'refused', category: 'unknown' }); + await step(change, 2); + assert.deepEqual(texts(), ['one', 'reply to one', 'two', 'later arrival'], 'step 2 took its planned exchange, not the later arrival'); + membrane.pushResponse(createMockResponse([{ type: 'text', text: 'back' }])); + const second = await rerun(change, 2); + assert.equal(second.status === 'completed' && second.outcome, 'responded'); + const op = framework.getUnstickOperation(change.id)!; + assert.deepEqual(op.steps.map((s) => [s.step, s.status]), [[1, 'shed'], [2, 'shed']]); + assert.deepEqual(op.attempts.map((a) => [a.step, a.outcome]), [[1, 'refused'], [2, 'responded']]); + }); + + it('takes steps only in order, after a refused re-run, and only within the plan', async () => { + await unstick(2); + const change = asked[0]!; + await assert.rejects(step(change, 2), /follows only a refused re-run after step 1/); + await step(change, 1); + membrane.pushResponse(createMockResponse([{ type: 'text', text: 'fine now' }])); + await rerun(change, 1); + await assert.rejects(step(change, 2), /follows only a refused re-run/, 'it responded: nothing more is authorized'); + await assert.rejects(step(change, 3), /plans 2 step\(s\); there is no step 3/); + }); + + it('refuses a step whose planned message was edited in place, or whose branch moved', async () => { + await unstick(2); + const change = asked[0]!; + await step(change, 1); + membrane.pushResponse(refusal()); + await rerun(change, 1); + const planned = (change as { plan: Array<{ messageIds: string[] }> }).plan[1]!.messageIds[0]!; + cm().editMessage(planned as never, [{ type: 'text', text: 'reply to two, edited' }]); + await assert.rejects(step(change, 2), (e: Error & { code?: string }) => e.code === 'stale' && /changed since it was approved/.test(e.message)); + const store = framework.getStore(); + store.createBranch('elsewhere'); + store.switchBranch('elsewhere'); + await assert.rejects(step(change, 2), (e: Error & { code?: string }) => e.code === 'stale' && /active branch moved/.test(e.message)); + }); + + it('finishes an interrupted step on its planned exchange, never another', async () => { + await unstick(2); + const change = asked[0]!; + const manager = cm() as unknown as { removeMessage: (id: string) => void }; + const realRemove = manager.removeMessage.bind(manager); + manager.removeMessage = () => { throw new Error('injected crash mid-shed'); }; + try { + await assert.rejects(step(change, 1), /injected crash mid-shed/); + } finally { + manager.removeMessage = realRemove; + } + assert.equal(framework.getUnstickOperation(change.id)!.steps[0]!.status, 'intent', 'the intent was journaled first'); + const finished = await step(change, 1); + assert.equal(finished.kind === 'unstick' && finished.alreadyApplied, undefined, 'finished now'); + assert.deepEqual(texts(), ['one', 'reply to one', 'two', 'reply to two'], 'exactly the culprit, nothing older'); + assert.equal(framework.getUnstickOperation(change.id)!.steps[0]!.status, 'shed'); + }); + + it('launches each re-run at most once, and reports an orphaned one as interrupted', async () => { + await unstick(2); + const change = asked[0]!; + await step(change, 1); + const first = framework.rerunUnstick(change as never, { step: 1 }); + const again = framework.rerunUnstick(change as never, { step: 1 }); + assert.equal(host().pendingRequests.filter((r) => r.reason === 'unstick-attempt').length, 1, 'one inference queued'); + // As if the process died before the queued re-run ran. + host().pendingRequests.length = 0; + host().unstickAttemptWaiters.clear(); + host().unstickJournalState = null; + assert.deepEqual(await framework.rerunUnstick(change as never, { step: 1 }), { step: 1, status: 'interrupted' }); + assert.equal(host().pendingRequests.length, 0, 'never relaunched'); + void first; void again; + }); + + it('keeps its record when the branch moves, since the journal is the store\'s, not the branch\'s', async () => { + await unstick(2); + const change = asked[0]!; + await step(change, 1); + const store = framework.getStore(); + store.createBranch('moved'); + store.switchBranch('moved'); + host().unstickJournalState = null; // reload from the store + assert.equal(framework.getUnstickOperation(change.id)!.steps[0]!.status, 'shed'); + }); + + it('runs back to back as a host would: each step after the previous re-run has fully settled', async () => { + await unstick(3); + const change = asked[0]!; + // One response per re-run, queued as it starts: the mock hands a stream + // every queued response (later ones resume tool rounds). + const responses = [refusal(), refusal(), createMockResponse([{ type: 'text', text: 'back' }])]; + let done = false; + const pump = (async () => { while (!done) { await framework.runUntilIdle(); await new Promise((r) => setTimeout(r, 5)); } })(); + const outcomes: string[] = []; + try { + for (const { step: n } of (change as { plan: Array<{ step: number }> }).plan) { + await step(change, n); // takes the lease as soon as the previous re-run has settled + membrane.pushResponse(responses[n - 1]!); + const attempt = await framework.rerunUnstick(change as never, { step: n }); + // Resolved only once the turn has fully settled: no live turn left. + assert.equal((framework as unknown as { activeTurnTokens: Map }).activeTurnTokens.has('scout'), false); + assert.equal(framework.getAgent('scout')!.state.status, 'idle'); + outcomes.push(attempt.status === 'completed' ? attempt.outcome : attempt.status); + if (attempt.status !== 'completed' || attempt.outcome !== 'refused') break; + } + } finally { + done = true; + await pump; + } + assert.deepEqual(outcomes, ['refused', 'refused', 'responded']); + assert.deepEqual(framework.getUnstickOperation(change.id)!.steps.map((x) => x.status), ['shed', 'shed', 'shed']); + assert.deepEqual(texts(), ['one', 'reply to one', 'back'], 'exactly the planned range went, then the agent answered'); + }); +}); From 6c25f839740122855bc5efa6855a10ffed35a9d7 Mon Sep 17 00:00:00 2001 From: Agnes-1009 Date: Wed, 7 Oct 2026 13:43:45 -0400 Subject: [PATCH 44/73] fix+feat(framework): store-bound resolved changes, five unstick review fixes, source-bound re-runs, and frozen-ref marks for staged undo (R2) On Ada's approved base dc2928c (AF #250). This answers Kit's review of 019d4c8 (room-222 #46571, and #46733 on batched wakes) and adds the frozen-ref marks agreed with Ada and June (#45297, #45349, #46840). Store identity (Kit #46571.1) - AgentFramework.storeIdentity() returns a UUID recorded once per store in a typed record ('framework/store-identity'). Typed records are store-bound and don't follow branch switches, which is the scope in which branch names and message ids mean anything. - Every resolved operator change carries storeId (ResolvedOperatorChangeBase), and every application path refuses another store's descriptor as stale ('resolved against another store'): self-change revalidation, undo-turns and unstick steps. A replacement or fresh store with the same agent, branch name, tail id and text can no longer receive a plan staged elsewhere. Unstick - Journal checkpoints keep every operation (#46571.2). The record is the authority for at-most-once re-runs and for the outcomes getUnstickOperation reports, finished or not. Records are small and operations rare; the checkpoint only bounds replay. The old slice(-32) erased operations still waiting on re-runs. - One re-run per agent at a time (#46571.3). A batched turn carries one binding, so a second queued re-run was stranded `launched` with an unresolved waiter. A second rerunUnstick for an agent with an unsettled one is refused 'agent-busy' before anything is recorded. - Duplicate callers wait for settlement (#46571.4). While an attempt's waiter exists, any caller for that attempt gets the same promise, even after the outcome is journaled, so nobody sees `completed` while the turn is still live. - One idempotent marker per operation (#46571.5). setUnstickMarker finds the marker by metadata {kind: 'unstick-marker', operationId} and sets its text from the step number, so a repaired or restarted step never counts twice. The text describes the planned removals ("set aside N earlier exchange(s) ... approved to take up to M, newest first") and says later messages were kept. The marker is system metadata, which planning skips. - Source-bound re-runs (Kit's question in #46571, and #46733). The request binding carries the operation's storeId and sourceBranch. At dispatch, a re-run whose store or branch moved is settled `failed` ('its source branch or store moved before the re-run started') and released without inference. Ordinary wakes batched with it keep their turn and don't inherit its binding. Frozen-ref marks for staged undo-turns (Ada, June) - host/command undo passes the operator's marks choice and its serverId to the gated path. resolveUndoTurns freezes the choice at staging: 'none', or {scope, refs}, where refs are exactly the addressable refs among messages stored after the oldest staged turn began (StoredMessage.sequence > sequenceBefore, the same store-sequence space the cut uses). ResolvedUndoTurnsChange gains marks and serverId. - applyUndoTurns takes Ada's addressingSnapshot before and after the switch, inside the held lease, and passes what was actually removed to scheduleAppliedMarks('undo', ..., change.marks). Only frozen refs the cut removed are marked. A later arrival that the cut also removed is counted `unmarked`, and the set is never widened. - Marks are scheduled once. The destination's marker state records {marked, markers: receipt} before the remaining bookkeeping. A retry after a failure, or a completed cut's re-apply, reports the recorded receipt and never schedules again. A repair with no snapshot left treats a frozen ref as removed when its message is gone from the destination. - Delivery stays Ada's: scheduleAppliedMarks starts the background drain when queued. The interim deliverDiscordAwarenessInBackground call in applyResolvedOperatorChange is gone. - AppliedOperatorChange 'undo-turns' gains markers. Tests (operator-change-gate, 56) - Marks: - staging freezes exactly the refs the cut would remove (a2; the oldest undone turn keeps its input); - application marks the frozen ref and counts a later cut arrival unmarked ({scope all, queued 1, unmarked 1, notRemoved 0}), and a re-apply reports the same receipt; - a failure after scheduling still schedules once, and the retry reports the receipt; - a 'none' choice stays local while counting removals; - the delivery test is rewritten on the real outbox: queued marks, delivery started both times, never awaited. - Unstick: - a replacement store refuses the plan and removes nothing; - an operation survives 70 journal entries across a checkpoint; - a second re-run is refused and records nothing; - a duplicate caller is still waiting at the outcome-recorded-but-unsettled point, then resolves with the first; - a failed completion record plus repair leaves one marker saying 1 exchange; - a re-run whose source moved fails without inference, while a batched ordinary wake answers. Mutation checks, each caught: - unstick ignoring the store; - checkpoint pruning; - two re-runs per agent; - a duplicate skipping settlement; - the old counting marker; - no dispatch source check; - marks not frozen; - marks scheduled per attempt (this one first survived; the failure-after-scheduling test now catches it). 24 suites, Ada's awareness-delivery and offline-recovery included: 375/375. mcpl-coalescing's undo case passes. Co-Authored-By: Agnes (Claude Opus 5.5) --- src/framework.ts | 214 +++++++++++++++++++++++----- src/operator-change.ts | 18 ++- src/types/agent.ts | 2 +- test/operator-change-gate.test.ts | 223 ++++++++++++++++++++++++++++-- 4 files changed, 407 insertions(+), 50 deletions(-) diff --git a/src/framework.ts b/src/framework.ts index 239aa8c8e..b8a483e3e 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -1624,13 +1624,15 @@ export class AgentFramework { private activeAdmissions: Map = new Map(); /** Admitted undo cuts whose bookkeeping finished in this process. */ private completedOperatorCuts: Set = new Set(); + /** This store's identity (storeIdentity), once read or minted. */ + private storeIdentityValue: string | null = null; /** The unstick journal and its reduced operations (unstickLedger). */ private unstickJournalState: { journal: RecordJournal; ops: Map; } | null = null; /** Unstick re-runs this process launched and hasn't settled, by operation#step. */ - private unstickAttemptWaiters: Map; resolve: (outcome: UnstickAttemptOutcome) => void }> = new Map(); + private unstickAttemptWaiters: Map; resolve: (outcome: UnstickAttemptOutcome) => void }> = new Map(); /** Outcomes recorded but not yet released to their waiters (the turn is * still tearing down), by operation#step. */ private unstickSettled: Map = new Map(); @@ -5832,7 +5834,10 @@ export class AgentFramework { const requested = Math.max(1, Math.min(20, Math.floor(params.turns ?? 1))); if (this.operatorChangeGate) { - return this.gatedUndoTurns(agentName, requested, hostCommandRequester(serverId, params)); + return this.gatedUndoTurns(agentName, requested, hostCommandRequester(serverId, params), { + marks: params.marks === 'addressed' || params.marks === 'all' ? { scope: params.marks } : 'none', + serverId, + }); } const turnMarks: DiscordAwarenessMarks = params.marks === 'addressed' || params.marks === 'all' ? { scope: params.marks } @@ -6818,6 +6823,7 @@ export class AgentFramework { ...(requester ? { requester } : {}), resolvedAt: Date.now(), sourceBranch: this.store.currentBranch().name, + storeId: this.storeIdentity(), }; if (tool === 'agent_settings') { return { ...base, kind: 'agent-settings', input: structuredClone(input), ...this.resolveSettingsEffect(agentName, input) }; @@ -6842,6 +6848,7 @@ export class AgentFramework { throw new OperatorActionError('stale', `Change ${change.id} for ${change.agent} no longer holds: ${what}`); }; if (!this.agents.has(change.agent)) throw new OperatorActionError('unknown-agent', `Unknown agent: ${change.agent}`); + if (change.storeId !== this.storeIdentity()) stale('it was resolved against another store'); const branch = this.store.currentBranch().name; if (branch !== change.sourceBranch) stale(`the active branch moved from ${change.sourceBranch} to ${branch}`); if (change.kind === 'agent-settings') { @@ -6919,7 +6926,12 @@ export class AgentFramework { * applyResolvedOperatorChange under a lease. Without a gate, host/command * keeps its per-turn loop. */ - private async gatedUndoTurns(agentName: string, requested: number, requester: OperatorRequester): Promise<{ + private async gatedUndoTurns( + agentName: string, + requested: number, + requester: OperatorRequester, + publication: { marks: 'none' | { scope: 'addressed' | 'all' }; serverId: string }, + ): Promise<{ ok: boolean; error?: string; code?: string; @@ -6932,7 +6944,7 @@ export class AgentFramework { if (agent.state.status !== 'idle') { return { ok: false, error: `Cannot undo while agent is ${agent.state.status}` }; } - const change = this.resolveUndoTurns(agentName, requested, requester); + const change = this.resolveUndoTurns(agentName, requested, requester, publication); if (!change) return { ok: true, undone: 0, requested, lastVisible: null }; const answer = await this.askOperatorChangeGate(change); if (answer.outcome === 'gate-failed') { @@ -6945,6 +6957,50 @@ export class AgentFramework { // Admitted unstick (ResolvedUnstickChange): planned steps, re-runs, journal // --------------------------------------------------------------------------- + /** + * This store's identity: a UUID recorded once, in a typed record that + * doesn't follow branch switches, so it names the store whichever branch + * is active and is never shared by a replacement or fresh store. + */ + private storeIdentity(): string { + if (this.storeIdentityValue) return this.storeIdentityValue; + const ids = this.store.getRecordIdsByType('framework/store-identity') + .map((id) => ({ id, value: BigInt(id) })) + .sort((a, b) => (a.value < b.value ? -1 : a.value > b.value ? 1 : 0)); + if (ids.length > 0) { + const record = this.store.getRecord(ids[0]!.id); + const parsed = record ? JSON.parse(Buffer.from(record.payload).toString('utf-8')) as { id?: unknown } : null; + if (typeof parsed?.id === 'string') return (this.storeIdentityValue = parsed.id); + } + const minted = randomUUID(); + this.store.appendJson('framework/store-identity', { id: minted, at: Date.now() }); + this.store.sync(); + return (this.storeIdentityValue = minted); + } + + /** + * The one marker an admitted unstick leaves, set to what its steps have + * shed so far. Found by its operation id, it is written idempotently: a + * repaired or restarted step never counts twice. It describes the planned + * removals, not "the most recent turns": later messages were kept. + */ + private setUnstickMarker(agent: Agent, change: ResolvedUnstickChange, shedSteps: number, category: string): void { + const cm = agent.getContextManager(); + const text = + `[unstick] An operator set aside ${shedSteps} earlier exchange(s) from this conversation ` + + `(approved to take up to ${change.plan.length}, newest first) because the model kept refusing on them ` + + `(content filter: ${category}). Their content is not reproduced here, so this note can't re-trigger ` + + `the filter; the originals remain in the raw record. Messages that arrived after them were kept. ` + + `You are clear to continue: if you were mid-task, take a different approach; otherwise carry on with ` + + `whatever is in front of you, or briefly acknowledge the gap and ask what's next.`; + const existing = cm.getAllMessages().find((m) => { + const meta = (m.metadata ?? {}) as { kind?: unknown; operationId?: unknown }; + return meta.kind === 'unstick-marker' && meta.operationId === change.id; + }); + if (existing) cm.editMessage(existing.id, [{ type: 'text', text }]); + else cm.addMessage('user', [{ type: 'text', text }], { system: true, kind: 'unstick-marker', operationId: change.id, category }); + } + /** The unstick journal, loaded and reduced on first use. Store-bound and * branch-independent (RecordJournal): an operation's intents and outcomes * don't rewind with an undo, and they die with this store. */ @@ -6981,11 +7037,12 @@ export class AgentFramework { } reduceUnstickEntry(ledger.ops, entry); if (ledger.journal.entriesSinceCheckpoint >= 64) { - // Keep the newest operations; older ones are long finished. - const kept = [...ledger.ops.entries()].slice(-32); + // Every operation stays: its record is the authority for at-most-once + // re-runs and the outcomes getUnstickOperation reports, finished or + // not. Records are small and operations rare; the checkpoint only + // bounds the replay. try { - ledger.journal.checkpoint(Object.fromEntries(kept)); - ledger.ops = new Map(kept); + ledger.journal.checkpoint(Object.fromEntries(ledger.ops)); } catch (error) { console.error('[unstick] journal checkpoint failed (replay stays longer):', error instanceof Error ? error.message : error); if (ledger.journal.needsReconcile) this.reloadUnstickJournal(); @@ -7031,6 +7088,7 @@ export class AgentFramework { ...(requester ? { requester } : {}), resolvedAt: Date.now(), sourceBranch: this.store.currentBranch().name, + storeId: this.storeIdentity(), kind: 'unstick', cap, plan, @@ -7059,6 +7117,7 @@ export class AgentFramework { } const agent = this.agents.get(change.agent); if (!agent) throw new OperatorActionError('unknown-agent', `Unknown agent: ${change.agent}`); + if (change.storeId !== this.storeIdentity()) stale('it was resolved against another store'); const op = this.unstickLedger().ops.get(change.id); const journaled = op?.steps.find((s) => s.step === step); if (journaled?.status === 'shed') return { step, shedIds: journaled.messageIds, alreadyApplied: true }; @@ -7097,7 +7156,7 @@ export class AgentFramework { if (byId.has(id)) cm.removeMessage(id as MessageId); } const lastRefusal = op?.attempts.filter((a) => a.outcome === 'refused').at(-1)?.category; - this.updateRewindMarker(agent, lastRefusal ?? 'unknown'); + this.setUnstickMarker(agent, change, step, lastRefusal ?? 'unknown'); this.journalUnstick({ kind: 'step-done', operationId: change.id, step }, { afterCommittedState: true }); this.recordOperatorAction({ kind: 'unstick-step', @@ -7124,6 +7183,10 @@ export class AgentFramework { throw new OperatorActionError('invalid', `Unstick ${change.id}: step ${step} hasn't been applied, so there is nothing to re-run`); } const key = `${change.id}#${step}`; + // Whoever else is waiting for this attempt is waiting for its turn to + // settle: a duplicate caller gets the same promise, outcome recorded or not. + const waiting = this.unstickAttemptWaiters.get(key); + if (waiting) return waiting.promise; const existing = op.attempts.find((a) => a.step === step); if (existing) { if (existing.status === 'completed') { @@ -7135,18 +7198,28 @@ export class AgentFramework { ...(existing.error ? { error: existing.error } : {}), }; } - return this.unstickAttemptWaiters.get(key)?.promise ?? { step, status: 'interrupted' }; + return { step, status: 'interrupted' }; + } + // One re-run per agent at a time: a batched turn carries one binding, so + // a second would be stranded. Refused before anything is recorded. + for (const other of this.unstickAttemptWaiters.values()) { + if (other.agent === change.agent) { + throw new OperatorActionError( + 'agent-busy', + `An unstick re-run for ${change.agent} is already in flight; re-run step ${step} of ${change.id} once it settles`, + ); + } } this.journalUnstick({ kind: 'attempt-launched', operationId: change.id, agent: change.agent, step }, { durable: true }); let resolve!: (outcome: UnstickAttemptOutcome) => void; const promise = new Promise((r) => { resolve = r; }); - this.unstickAttemptWaiters.set(key, { promise, resolve }); + this.unstickAttemptWaiters.set(key, { agent: change.agent, promise, resolve }); this.pendingRequests.push({ agentName: change.agent, reason: 'unstick-attempt', source: 'framework', timestamp: Date.now(), - unstick: { operationId: change.id, step }, + unstick: { operationId: change.id, step, storeId: change.storeId, sourceBranch: change.sourceBranch }, }); this.recordOperatorAction({ kind: 'unstick-attempt', @@ -7227,10 +7300,31 @@ export class AgentFramework { /** Resolve host/command undo by turns to the exact checkpoints it would * undo, newest first; null when there is nothing to undo. */ - private resolveUndoTurns(agentName: string, requested: number, requester?: OperatorRequester): ResolvedUndoTurnsChange | null { + private resolveUndoTurns( + agentName: string, + requested: number, + requester?: OperatorRequester, + publication?: { marks: 'none' | { scope: 'addressed' | 'all' }; serverId: string }, + ): ResolvedUndoTurnsChange | null { const checkpoints = this.getTurnCheckpoints(agentName); if (checkpoints.length === 0) return null; const taken = checkpoints.slice(-requested).reverse(); + // Freeze the publication choice to the refs the cut would remove now: + // every message stored after the oldest staged turn began. + let marks: ResolvedUndoTurnsChange['marks'] = 'none'; + const choice = publication?.marks ?? 'none'; + if (choice !== 'none') { + const cutAt = taken[taken.length - 1]!.sequenceBefore; + const wouldRemove = this.agents.get(agentName)!.getContextManager().getAllMessages() + .filter((m) => Number(m.sequence) > cutAt) + .map((m) => ({ + metadata: { + ...((m.metadata ?? {}) as Record), + serverId: (m.metadata as { serverId?: unknown } | undefined)?.serverId ?? publication!.serverId, + }, + })); + marks = { scope: choice.scope, refs: selectDiscordAwarenessRefs(wouldRemove, choice).refs }; + } return { id: randomUUID(), agent: agentName, @@ -7238,9 +7332,12 @@ export class AgentFramework { ...(requester ? { requester } : {}), resolvedAt: Date.now(), sourceBranch: this.store.currentBranch().name, + storeId: this.storeIdentity(), kind: 'undo-turns', requestedTurns: requested, checkpoints: taken.map((c) => ({ turnIndex: c.turnIndex, sequenceBefore: c.sequenceBefore, branchName: c.branchName })), + marks, + ...(publication?.serverId ? { serverId: publication.serverId } : {}), }; } @@ -7268,19 +7365,24 @@ export class AgentFramework { fromBranch: string; toBranch: string; alreadyApplied?: true; + markers?: SurgeryMarkerReceipt; } { const stale = (what: string): never => { throw new OperatorActionError('stale', `Undo ${change.id} for ${change.agent} no longer holds: ${what}`); }; if (!this.agents.has(change.agent)) throw new OperatorActionError('unknown-agent', `Unknown agent: ${change.agent}`); + if (change.storeId !== this.storeIdentity()) stale('it was resolved against another store'); const destination = this.undoTurnsDestination(change); const undone = change.checkpoints.length; const current = this.store.currentBranch().name; if (current === destination) { // The cut happened. Finish whatever bookkeeping an interrupted attempt // left undone, without cutting again, and only then report it applied. - this.finishUndoTurnsCut(change, admission); - return { requested: change.requestedTurns, undone, fromBranch: change.sourceBranch, toBranch: destination, alreadyApplied: true }; + const markers = this.finishUndoTurnsCut(change, admission); + return { + requested: change.requestedTurns, undone, fromBranch: change.sourceBranch, toBranch: destination, alreadyApplied: true, + ...(markers ? { markers } : {}), + }; } // The destination exists if an earlier attempt got as far as creating // it. Never written to (its head still at its branch point), that @@ -7305,10 +7407,19 @@ export class AgentFramework { // source: the destination, cut before the oldest staged turn recorded // itself, keeps only the older checkpoints, and the source tip keeps // every checkpoint for redo to return to. + // What the cut removes from this agent's context is whatever is in it + // now and not after: addressing metadata only, read within the same + // reservation that holds the switch. + const before = this.addressingSnapshot(change.agent); if (!created) this.store.createBranchAt(destination, change.sourceBranch, oldest.sequenceBefore); this.store.switchBranch(destination); - this.finishUndoTurnsCut(change, admission); - return { requested: change.requestedTurns, undone, fromBranch: change.sourceBranch, toBranch: destination }; + const after = new Set(this.addressingSnapshot(change.agent).keys()); + const removed = [...before].filter(([id]) => !after.has(id)).map(([, carrier]) => carrier); + const markers = this.finishUndoTurnsCut(change, admission, removed); + return { + requested: change.requestedTurns, undone, fromBranch: change.sourceBranch, toBranch: destination, + ...(markers ? { markers } : {}), + }; } /** @@ -7321,29 +7432,52 @@ export class AgentFramework { * - the one redo entry back to the source tip; * - the trace and the operator-log record, once per process. */ - private finishUndoTurnsCut(change: ResolvedUndoTurnsChange, admission?: OperatorAdmission): void { + private finishUndoTurnsCut( + change: ResolvedUndoTurnsChange, + admission?: OperatorAdmission, + removed?: Array<{ metadata: Record }>, + ): SurgeryMarkerReceipt | undefined { const destination = this.undoTurnsDestination(change); const oldest = change.checkpoints[change.checkpoints.length - 1]!; - const marker = (): { changeId?: unknown; completed?: unknown } | undefined => { + const marker = (): { changeId?: unknown; completed?: unknown; marked?: unknown; markers?: SurgeryMarkerReceipt } | undefined => { const framework = this.store.getStateJson(FRAMEWORK_STATE_ID); return (framework && typeof framework === 'object' ? framework as Record : {}).operatorCut as - { changeId?: unknown; completed?: unknown } | undefined; + { changeId?: unknown; completed?: unknown; marked?: unknown; markers?: SurgeryMarkerReceipt } | undefined; }; - const writeMarker = (completed: boolean) => { + const writeMarker = (state: { markers?: SurgeryMarkerReceipt }) => { const framework = this.store.getStateJson(FRAMEWORK_STATE_ID); this.store.setStateJson(FRAMEWORK_STATE_ID, { ...(framework && typeof framework === 'object' ? framework as Record : {}), - operatorCut: { changeId: change.id, at: Date.now(), ...(completed ? { completed: true } : {}) }, + operatorCut: { changeId: change.id, at: Date.now(), ...(state.markers ? { marked: true, markers: state.markers } : {}) }, }); }; // A completed cut is left exactly as it is, whatever happened since on // its branch (later turns and their checkpoints, a redo stack): only - // its durability is confirmed again. + // its durability is confirmed again, and its marks receipt reported. if (marker()?.changeId === change.id && marker()?.completed === true) { this.store.sync(); - return; + return marker()?.markers; + } + if (marker()?.changeId !== change.id) writeMarker({}); + // Marks, once: what the cut actually removed, limited to the refs frozen + // at staging. A repair that has no snapshot left treats a frozen ref as + // removed when its message is gone from the destination (later + // arrivals' counts are then unknown, and reported as none). + // Scheduled once; every later answer reports the receipt recorded then. + let markers = marker()?.markers; + if (marker()?.marked !== true) { + let carriers = removed; + if (!carriers) { + const present = new Set( + [...this.addressingSnapshot(change.agent).values()].map((c) => `${c.metadata.channelId}|${c.metadata.messageId}`), + ); + carriers = (change.marks === 'none' ? [] : change.marks.refs) + .filter((ref) => !present.has(`${ref.channelId}|${ref.messageId}`)) + .map((ref) => ({ metadata: { ...ref } })); + } + markers = this.scheduleAppliedMarks('undo', change.agent, change.serverId ?? '', carriers, change.marks); + writeMarker({ markers }); } - if (marker()?.changeId !== change.id) writeMarker(false); // Checkpoints need no bookkeeping: the lists are branch-scoped, and the // destination, cut where the oldest staged turn had yet to record // itself, inherits exactly the older ones. Later turns add their own. @@ -7362,8 +7496,8 @@ export class AgentFramework { redoStack.push({ branchName: change.sourceBranch, checkpoints: [], changeId: change.id }); } if (this.completedOperatorCuts.has(change.id)) { - this.completeUndoTurnsCut(change); - return; + this.completeUndoTurnsCut(change, markers); + return markers; } this.emitTrace({ type: 'undo:completed', @@ -7387,18 +7521,19 @@ export class AgentFramework { result: { sourceBranch: change.sourceBranch, targetBranch: destination }, }); this.completedOperatorCuts.add(change.id); - this.completeUndoTurnsCut(change); + this.completeUndoTurnsCut(change, markers); + return markers; } /** Mark an admitted undo cut complete on its destination and make that * durable: branch state persists only on sync(), so completion is * reported only after the sync succeeds (a failure throws, and a retry * confirms it again). */ - private completeUndoTurnsCut(change: ResolvedUndoTurnsChange): void { + private completeUndoTurnsCut(change: ResolvedUndoTurnsChange, markers?: SurgeryMarkerReceipt): void { const framework = this.store.getStateJson(FRAMEWORK_STATE_ID); this.store.setStateJson(FRAMEWORK_STATE_ID, { ...(framework && typeof framework === 'object' ? framework as Record : {}), - operatorCut: { changeId: change.id, at: Date.now(), completed: true }, + operatorCut: { changeId: change.id, at: Date.now(), marked: true, ...(markers ? { markers } : {}), completed: true }, }); this.store.sync(); } @@ -7425,11 +7560,9 @@ export class AgentFramework { return { kind: 'unstick', ...this.applyUnstickStep(change, opts.step, opts.admission) }; } if (change.kind === 'undo-turns') { + // Marks are scheduled within the cut's bookkeeping, under the lease; + // their delivery runs in the background and is never this undo's. const applied = this.applyUndoTurns(change, opts.admission); - // Installs the MCPL data-plane gate now, under the lease, and drains in - // the background: remote delivery is never awaited inside the lease, - // and its failure is never this undo's. - if (!applied.alreadyApplied) this.deliverDiscordAwarenessInBackground('undo', change.agent); return { kind: 'undo-turns', ...applied }; } this.revalidateSelfChange(change); @@ -10374,6 +10507,19 @@ export class AgentFramework { // A silent tick never suppresses a genuine coalesced wake. If any // ordinary request shares this batch, the ordinary turn semantics win. const silentOnly = requests.every((r) => r.suppressProse === true); + // An unstick re-run is bound to the source it was approved on. If the + // store or branch moved before it could start, it fails without + // running; ordinary wakes batched with it keep their turn, unbound. + const movedRuns = requests.filter((r) => r.unstick + && (r.unstick.storeId !== this.storeIdentity() || r.unstick.sourceBranch !== this.store.currentBranch().name)); + if (movedRuns.length > 0) { + for (const r of movedRuns) { + this.settleUnstickAttempt(r.unstick!, 'failed', { error: 'its source branch or store moved before the re-run started' }); + this.releaseUnstickAttempt(r.unstick!); + } + requests = requests.filter((r) => !movedRuns.includes(r)); + if (requests.length === 0) continue; + } // An unstick re-run in the batch binds the turn to its operation and // step, whichever request leads the batch. const unstickReq = requests.find((r) => r.unstick); diff --git a/src/operator-change.ts b/src/operator-change.ts index 2abc4bbbc..5adb52860 100644 --- a/src/operator-change.ts +++ b/src/operator-change.ts @@ -11,7 +11,7 @@ * applies exactly what was resolved, never the relative command again: a * change that no longer holds is refused as `stale` and the host restages it. */ -import type { OperatorRequester } from './operator-log.js'; +import type { OperatorRequester, SurgeryMarkerReceipt } from './operator-log.js'; /** The host's receipt for a staged change, shown to whoever asked: the * change is accepted and queued, not applied. */ @@ -38,6 +38,10 @@ interface ResolvedOperatorChangeBase { /** The active branch when resolved: the stable identity application * checks. Appending messages (a notice, an answer) doesn't change it. */ sourceBranch: string; + /** The store it was resolved in (AgentFramework's store identity). Branch + * names and message ids mean something only within one store, so a + * change is refused anywhere else, a replacement store included. */ + storeId: string; } /** agent_settings update / reset / cancel, from an operator or a module. */ @@ -73,6 +77,15 @@ export interface ResolvedUndoTurnsChange extends ResolvedOperatorChangeBase { /** The checkpoints it undoes, newest first. Application makes one cut at * the oldest one's sequenceBefore, onto a branch named for this change. */ checkpoints: Array<{ turnIndex: number; sequenceBefore: number; branchName: string }>; + /** The operator's awareness-marks choice, frozen at staging: 'none', or a + * scope with exactly the refs the cut would have removed then. The cut + * can also reach messages that arrive later; application marks only the + * frozen refs it actually removed, reports later removals as unmarked, + * and never widens the set. */ + marks: 'none' | { scope: 'addressed' | 'all'; refs: Array<{ serverId: string; channelId: string; messageId: string }> }; + /** The MCPL server the command came from: marks whose message names no + * server are routed through it. */ + serverId?: string; } /** @@ -149,6 +162,9 @@ export type AppliedOperatorChange = /** The cut had already been applied (the active branch is its * destination): nothing changed this time. */ alreadyApplied?: true; + /** The awareness marks receipt: scheduled once, by the first + * application to get that far, and reported again by any retry. */ + markers?: SurgeryMarkerReceipt; } | { kind: 'unstick'; diff --git a/src/types/agent.ts b/src/types/agent.ts index 402f86f13..384610040 100644 --- a/src/types/agent.ts +++ b/src/types/agent.ts @@ -305,7 +305,7 @@ export interface CompletedToolCall { export interface InferenceRequest { /** An admitted unstick operation's re-run (rerunUnstick): the turn records * its outcome against this operation and step instead of shedding. */ - unstick?: { operationId: string; step: number }; + unstick?: { operationId: string; step: number; storeId: string; sourceBranch: string }; /** Host-owned identity for withdrawal of an unconsumed coalesced wake. */ coalescingSubject?: string; coalescingEventId?: string; diff --git a/test/operator-change-gate.test.ts b/test/operator-change-gate.test.ts index 6754d2635..edfe62ec6 100644 --- a/test/operator-change-gate.test.ts +++ b/test/operator-change-gate.test.ts @@ -659,24 +659,108 @@ describe('operator-change gate: host/command undo by turns', () => { }); + const say = async (messageId: string) => { + framework.getAgent('scout')!.getContextManager().addMessage('Member', [{ type: 'text', text: messageId }], { + serverId: 'discord', channelId: 'discord:g1:c1', messageId, tags: ['chat:addressed'], + }); + membrane.pushResponse(createMockResponse([{ type: 'text', text: `reply to ${messageId}` }])); + host().pendingRequests.push({ agentName: 'scout', reason: 'mcpl:channel-incoming', source: 'test', timestamp: Date.now() } as InferenceRequest); + await framework.runUntilIdle(); + }; + const undoWithMarks = (turns: number, marks: string) => + host().handleHostCommand('discord', { command: 'undo', agentName: 'scout', turns, marks, requesterName: 'nissa' }); + + it('freezes the marks choice to the refs the cut would remove at staging', async () => { + await say('a0'); await say('a1'); await say('a2'); + const r = await undoWithMarks(2, 'all'); + assert.equal(r.code, 'staged'); + const change = asked[0]!; + assert.deepEqual(change.kind === 'undo-turns' && change.marks, { + scope: 'all', + refs: [{ serverId: 'discord', channelId: 'discord:g1:c1', messageId: 'a2' }], + }, "the oldest undone turn keeps its input (a1); a2's turn is what the cut removes"); + assert.equal(change.kind === 'undo-turns' && change.serverId, 'discord'); + }); + + it('marks only the frozen refs the cut removed, counting a later cut arrival as unmarked', async () => { + await say('a0'); await say('a1'); await say('a2'); + await undoWithMarks(2, 'all'); + const change = asked[0]!; + // Arrives after staging: the cut removes it too, but nobody authorized it. + framework.getAgent('scout')!.getContextManager().addMessage('Member', [{ type: 'text', text: 'late' }], { + serverId: 'discord', channelId: 'discord:g1:c1', messageId: 'late', tags: ['chat:addressed'], + }); + const applied = await framework.runAtSafeBoundary({ verb: 'apply' }, (lease) => + framework.applyResolvedOperatorChange(change, { lease, admission: { id: 'rev-u' } })); + assert.equal(applied.kind, 'undo-turns'); + const m = applied.kind === 'undo-turns' ? applied.markers! : null; + assert.deepEqual( + m && { scope: m.scope, status: m.status, queued: m.queued, unmarked: m.unmarked, notRemoved: m.notRemoved }, + { scope: 'all', status: 'queued', queued: 1, unmarked: 1, notRemoved: 0 }, + 'the reserved before/after snapshot established what was removed; a2 marked, late unmarked', + ); + const again = await framework.runAtSafeBoundary({ verb: 'apply' }, (lease) => + framework.applyResolvedOperatorChange(change, { lease, admission: { id: 'rev-u' } })); + assert.deepEqual(again.kind === 'undo-turns' && again.markers, m, 'a retry reports the same receipt'); + }); + + it('schedules marks once even when the first attempt fails after scheduling them', async () => { + await say('a0'); await say('a1'); await say('a2'); + await undoWithMarks(2, 'all'); + const change = asked[0]!; + const fw = framework as unknown as { + scheduleAppliedMarks: (...args: unknown[]) => unknown; + recordOperatorAction: (entry: { kind: string }) => void; + }; + const realSchedule = fw.scheduleAppliedMarks.bind(fw); + const realRecord = fw.recordOperatorAction.bind(fw); + let scheduled = 0; + fw.scheduleAppliedMarks = (...args) => { scheduled++; return realSchedule(...args); }; + let armed = true; + fw.recordOperatorAction = (entry) => { + if (armed && entry.kind === 'undo-turns') { armed = false; throw new Error('injected failure after marks'); } + return realRecord(entry); + }; + const apply = () => framework.runAtSafeBoundary({ verb: 'apply' }, (lease) => + framework.applyResolvedOperatorChange(change, { lease, admission: { id: 'rev-u' } })); + try { + await assert.rejects(apply(), /injected failure after marks/); + const retried = await apply(); + const m = retried.kind === 'undo-turns' ? retried.markers : undefined; + assert.deepEqual(m && [m.status, m.queued], ['queued', 1], 'the retry reports the receipt the failed attempt recorded'); + assert.equal(scheduled, 1, 'scheduled once'); + } finally { + fw.scheduleAppliedMarks = realSchedule; + fw.recordOperatorAction = realRecord; + } + }); + + it("keeps a staged undo without a marks choice local, still counting what it removed", async () => { + await say('a0'); await say('a1'); await say('a2'); + await undoWithMarks(2, 'none'); + const change = asked[0]!; + assert.equal(change.kind === 'undo-turns' && change.marks, 'none'); + const applied = await framework.runAtSafeBoundary({ verb: 'apply' }, (lease) => + framework.applyResolvedOperatorChange(change, { lease, admission: { id: 'rev-u' } })); + const m = applied.kind === 'undo-turns' ? applied.markers! : null; + assert.deepEqual(m && { scope: m.scope, queued: m.queued, unmarked: m.unmarked }, { scope: 'none', queued: 0, unmarked: 1 }); + }); + it("never awaits awareness delivery inside the lease, and never reports its failure as the undo's", async () => { - const fw = framework as unknown as { discordAwarenessOutbox: unknown; syncDiscordAwarenessMarkers: () => Promise }; - fw.discordAwarenessOutbox = {}; // so delivery is attempted + const fw = framework as unknown as { syncDiscordAwarenessMarkers: () => Promise }; let calls = 0; + // The real outbox queues the marks; only delivery is stubbed: it fails + // the first time and never finishes the second. fw.syncDiscordAwarenessMarkers = () => { calls++; return calls === 1 ? Promise.reject(new Error('delivery down')) : new Promise(() => {}); }; - try { - await turn('one'); await turn('two'); await turn('three'); - for (const id of ['rev-a', 'rev-b']) { - await undo(1); - const staged = asked.at(-1)!; - const applied = await framework.runAtSafeBoundary({ verb: 'apply' }, (lease) => - framework.applyResolvedOperatorChange(staged, { lease, admission: { id } })); - assert.equal(applied.kind, 'undo-turns', 'returned whether delivery failed or is still pending'); - } - assert.equal(calls, 2, 'delivery started both times'); - } finally { - fw.discordAwarenessOutbox = null; + await say('a0'); await say('a1'); await say('a2'); await say('a3'); + for (const id of ['rev-a', 'rev-b']) { + await undoWithMarks(2, 'all'); // two turns, so the cut removes addressed inputs to mark + const staged = asked.at(-1)!; + const applied = await framework.runAtSafeBoundary({ verb: 'apply' }, (lease) => + framework.applyResolvedOperatorChange(staged, { lease, admission: { id } })); + assert.equal(applied.kind === 'undo-turns' && applied.markers?.status, 'queued', 'marks queued; the undo stands either way'); } + assert.equal(calls, 2, 'delivery started both times, and neither apply waited for it'); }); it('refuses a staged undo whose checkpoint is gone though the branch stayed', async () => { @@ -878,6 +962,117 @@ describe('operator-change gate: host/command unstick, planned at staging', () => assert.equal(framework.getUnstickOperation(change.id)!.steps[0]!.status, 'shed'); }); + it("refuses a plan in any store but the one it was staged in, a replacement store included", async () => { + await unstick(1); + const change = asked[0]!; + // A fresh store with the same agent, branch name and tail id. + const other = await AgentFramework.create({ + storePath: join(tempDir, 'replacement.chronicle'), + membrane: new MockMembrane().asMembrane(), + agents: [{ name: 'scout', model: 'test-model', systemPrompt: 'You are scout.', maxTokens: 1000 }], + modules: [], + }); + try { + const ocm = other.getAgent('scout')!.getContextManager(); + ocm.addMessage('user', [{ type: 'text', text: 'culprit' }]); + await assert.rejects( + other.runAtSafeBoundary({ verb: 'apply elsewhere' }, (lease) => + other.applyResolvedOperatorChange(change, { lease, admission: { id: 'rev-s' }, step: 1 })), + (e: Error & { code?: string }) => e.code === 'stale' && /another store/.test(e.message), + ); + assert.equal(ocm.getAllMessages().length, 1, 'nothing removed there'); + } finally { + await other.stop(); + } + }); + + it('keeps every operation through journal checkpoints, finished or not', async () => { + await unstick(1); + const change = asked[0]!; + await step(change, 1); + const journal = framework as unknown as { journalUnstick: (entry: unknown, opts: object) => void }; + for (let i = 0; i < 70; i++) { + journal.journalUnstick({ kind: 'step-intent', operationId: `filler-${i}`, agent: 'scout', step: 1, messageIds: [] }, {}); + } + host().unstickJournalState = null; // reload from the store, through the checkpoint + assert.equal(framework.getUnstickOperation(change.id)!.steps[0]!.status, 'shed', 'its re-run still pending, it is still on record'); + }); + + it("refuses a second agent's re-run while one is in flight, recording nothing for it", async () => { + await unstick(1); + const first = asked[0]!; + await step(first, 1); + host().addMessage('user', [{ type: 'text', text: 'second culprit' }]); + await unstick(1); + const second = asked[1]!; + await step(second, 1); + void framework.rerunUnstick(first as never, { step: 1 }); + await assert.rejects(framework.rerunUnstick(second as never, { step: 1 }), (e: Error & { code?: string }) => e.code === 'agent-busy'); + assert.equal(framework.getUnstickOperation(second.id)!.attempts.length, 0); + assert.equal(host().pendingRequests.filter((r) => r.reason === 'unstick-attempt').length, 1); + }); + + it('makes a duplicate caller wait for the same settlement as the first', async () => { + await unstick(1); + const change = asked[0]!; + await step(change, 1); + const fw = framework as unknown as { releaseUnstickAttempt: (b: unknown) => void }; + const realRelease = fw.releaseUnstickAttempt.bind(fw); + let held: unknown = null; + fw.releaseUnstickAttempt = (b) => { held = b; }; // the turn hasn't settled yet + membrane.pushResponse(createMockResponse([{ type: 'text', text: 'back' }])); + const first = framework.rerunUnstick(change as never, { step: 1 }); + await framework.runUntilIdle(); + assert.equal(framework.getUnstickOperation(change.id)!.attempts[0]!.status, 'completed', 'the outcome is recorded'); + const duplicate = framework.rerunUnstick(change as never, { step: 1 }); + const early = await Promise.race([duplicate.then(() => 'resolved'), new Promise((r) => setTimeout(() => r('waiting'), 50))]); + assert.equal(early, 'waiting', 'not before settlement'); + fw.releaseUnstickAttempt = realRelease; + realRelease(held); + assert.deepEqual(await duplicate, await first); + }); + + it('keeps one marker per operation, counting a repaired step once', async () => { + await unstick(2); + const change = asked[0]!; + const journal = framework as unknown as { journalUnstick: (entry: { kind: string }, opts: object) => void }; + const realJournal = journal.journalUnstick.bind(journal); + let armed = true; + journal.journalUnstick = (entry, opts) => { + if (armed && entry.kind === 'step-done') { armed = false; throw new Error('injected completion failure'); } + return realJournal(entry, opts); + }; + try { + await assert.rejects(step(change, 1), /injected completion failure/); + } finally { + journal.journalUnstick = realJournal; + } + await step(change, 1); // the repair + const markers = (cm().getAllMessages() as Array<{ content: Array<{ text?: string }>; metadata?: { kind?: string } }>) + .filter((m) => m.metadata?.kind === 'unstick-marker'); + assert.equal(markers.length, 1, 'one marker for the operation'); + assert.match(String(markers[0]!.content[0]!.text), /set aside 1 earlier exchange\(s\)/); + assert.match(String(markers[0]!.content[0]!.text), /Messages that arrived after them were kept/); + }); + + it("fails a queued re-run whose source moved before it started, and keeps an ordinary wake batched with it", async () => { + await unstick(1); + const change = asked[0]!; + await step(change, 1); + const attempt = framework.rerunUnstick(change as never, { step: 1 }); // queued, not yet run + host().pendingRequests.push({ agentName: 'scout', reason: 'mcpl:channel-incoming', source: 'test', timestamp: Date.now() } as InferenceRequest); + const store = framework.getStore(); + store.createBranch('moved-before-dispatch'); + store.switchBranch('moved-before-dispatch'); + membrane.pushResponse(createMockResponse([{ type: 'text', text: 'ordinary answer' }])); + await framework.runUntilIdle(); + const outcome = await attempt; + assert.equal(outcome.status === 'completed' && outcome.outcome, 'failed'); + assert.match(String(outcome.status === 'completed' && outcome.error), /moved before the re-run started/); + assert.equal(texts().at(-1), 'ordinary answer', 'the ordinary wake still had its turn'); + assert.equal(framework.getUnstickOperation(change.id)!.attempts[0]!.outcome, 'failed', 'and it was not reported as the re-run'); + }); + it('runs back to back as a host would: each step after the previous re-run has fully settled', async () => { await unstick(3); const change = asked[0]!; From 679ee66548b85b6d3cb2ed58743599091f90b300 Mon Sep 17 00:00:00 2001 From: Agnes-1009 Date: Wed, 7 Oct 2026 15:15:40 -0400 Subject: [PATCH 45/73] fix(framework): every queue exit settles its unstick re-run; batch semantics come from surviving requests (review) Kit's two dispatch findings on 0019652 (room-222 #47291). 1. A rejected re-run's binding no longer reaches the ordinary wake batched with it. The moved-source filter ran after `trigger`, the routing winners and `silentOnly` had been taken from the whole batch, so with the stale re-run first the ordinary stream was dispatched with reason 'unstick-attempt' and its binding. The filter now runs before the policy check, so policy and every batch semantic see only the surviving requests. 2. Every path that takes a request off the queue without a turn now gives its re-run a terminal outcome: failQueuedUnstick settles it 'failed' with the reason and releases its waiter. Before, a policy skip (inferencePolicy.shouldInfer false) drained the request but left the attempt `launched` with an unresolved waiter, and the one-per-agent guard then refused every later re-run. The paths: - a moved source; - a policy skip; - an orphaned agent (agent_not_found). The other exits only re-queue or park: the busy requeue, quiesce, the safe-boundary hold, and provider-cooldown holding. stop() releases waiting callers as 'interrupted', matching what their journal entry ('launched', with no runner) reads. Tests: - the moved-source case asserts the dispatched trigger itself: one turn, with no unstick binding and no unstick reason; - a declined re-run settles failed, and the next operation's re-run runs, not refused as busy; - stop() resolves a waiting re-run as interrupted, and after reopen the journal reads launched. Mutants caught: - the trigger taken from the unfiltered batch (Kit's case); - a policy skip that doesn't settle; - stop() that leaves waiters. Gate suite and scheduler neighbours: 93/93. Co-Authored-By: Agnes (Claude Opus 5.5) --- src/framework.ts | 47 ++++++++++++++++++------- test/operator-change-gate.test.ts | 58 +++++++++++++++++++++++++++++-- 2 files changed, 90 insertions(+), 15 deletions(-) diff --git a/src/framework.ts b/src/framework.ts index b8a483e3e..89696d4c1 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -2203,6 +2203,12 @@ export class AgentFramework { this.pushCoalescer?.suspend(); this.flushCoalescingSnapshot(); this.running = false; + // A re-run still waiting has no turn coming: its journal says launched, + // which reads interrupted, and so does what its callers are told. + for (const [key, waiter] of this.unstickAttemptWaiters) { + this.unstickAttemptWaiters.delete(key); + waiter.resolve({ step: Number(key.slice(key.lastIndexOf('#') + 1)), status: 'interrupted' }); + } // Flushed-but-unsynced deferred writes: sync and ack now, while the // store is still open, rather than leaving them to a reboot replay. this.ackDeferredWrites(); @@ -7268,6 +7274,19 @@ export class AgentFramework { /** Resolve an unstick re-run's waiter with its recorded outcome, at the * end of its stream's teardown. */ + /** + * Settle and release the unstick re-runs among requests that leave the + * queue without a turn. Every exit needs a terminal outcome: otherwise the + * caller waits forever and the agent's next re-run is refused as busy. + */ + private failQueuedUnstick(requests: InferenceRequest[], error: string): void { + for (const r of requests) { + if (!r.unstick) continue; + this.settleUnstickAttempt(r.unstick, 'failed', { error }); + this.releaseUnstickAttempt(r.unstick); + } + } + private releaseUnstickAttempt(binding: { operationId: string; step: number }): void { const key = `${binding.operationId}#${binding.step}`; const result = this.unstickSettled.get(key); @@ -10281,6 +10300,7 @@ export class AgentFramework { oldestRequestAge: now - oldest, }); console.error(`[inference-dropped] agent=${agentName} reason=agent_not_found requests=${requests.length}`); + this.failQueuedUnstick(requests, `its agent ${agentName} no longer exists`); continue; } @@ -10430,6 +10450,19 @@ export class AgentFramework { continue; } + // An unstick re-run is bound to the source it was approved on. If the + // store or branch moved before it could start, it fails without + // running. It leaves the batch before policy and turn semantics are + // taken from it, so ordinary wakes batched with it keep their turn and + // inherit nothing of it. + const movedRuns = requests.filter((r) => r.unstick + && (r.unstick.storeId !== this.storeIdentity() || r.unstick.sourceBranch !== this.store.currentBranch().name)); + if (movedRuns.length > 0) { + this.failQueuedUnstick(movedRuns, 'its source branch or store moved before the re-run started'); + requests = requests.filter((r) => !movedRuns.includes(r)); + if (requests.length === 0) continue; + } + // Check policy if (!this.inferencePolicy.shouldInfer(agentName, requests, state)) { // Loud drop: a queued request that dies here is otherwise invisible — @@ -10439,6 +10472,7 @@ export class AgentFramework { `[inference-dropped] agent=${agentName} reason=policy-skip ` + `requests=${requests.length} triggers=${requests.map((r) => r.reason).join(',')}`, ); + this.failQueuedUnstick(requests, 'the inference policy declined the re-run'); // A context-budget restart inherits the predecessor's EventGate liveness. // If policy drops the queued successor, no driveStream remains to release it. if (budgetRestart) this.eventGate?.onInferenceEnded(agentName); @@ -10507,19 +10541,6 @@ export class AgentFramework { // A silent tick never suppresses a genuine coalesced wake. If any // ordinary request shares this batch, the ordinary turn semantics win. const silentOnly = requests.every((r) => r.suppressProse === true); - // An unstick re-run is bound to the source it was approved on. If the - // store or branch moved before it could start, it fails without - // running; ordinary wakes batched with it keep their turn, unbound. - const movedRuns = requests.filter((r) => r.unstick - && (r.unstick.storeId !== this.storeIdentity() || r.unstick.sourceBranch !== this.store.currentBranch().name)); - if (movedRuns.length > 0) { - for (const r of movedRuns) { - this.settleUnstickAttempt(r.unstick!, 'failed', { error: 'its source branch or store moved before the re-run started' }); - this.releaseUnstickAttempt(r.unstick!); - } - requests = requests.filter((r) => !movedRuns.includes(r)); - if (requests.length === 0) continue; - } // An unstick re-run in the batch binds the turn to its operation and // step, whichever request leads the batch. const unstickReq = requests.find((r) => r.unstick); diff --git a/test/operator-change-gate.test.ts b/test/operator-change-gate.test.ts index edfe62ec6..7860eaa60 100644 --- a/test/operator-change-gate.test.ts +++ b/test/operator-change-gate.test.ts @@ -1059,13 +1059,24 @@ describe('operator-change gate: host/command unstick, planned at staging', () => await unstick(1); const change = asked[0]!; await step(change, 1); - const attempt = framework.rerunUnstick(change as never, { step: 1 }); // queued, not yet run + const attempt = framework.rerunUnstick(change as never, { step: 1 }); // queued first, not yet run host().pendingRequests.push({ agentName: 'scout', reason: 'mcpl:channel-incoming', source: 'test', timestamp: Date.now() } as InferenceRequest); const store = framework.getStore(); store.createBranch('moved-before-dispatch'); store.switchBranch('moved-before-dispatch'); membrane.pushResponse(createMockResponse([{ type: 'text', text: 'ordinary answer' }])); - await framework.runUntilIdle(); + const fw = framework as unknown as { startAgentStream: (agent: unknown, trigger: InferenceRequest) => Promise }; + const realStart = fw.startAgentStream.bind(fw); + const dispatched: InferenceRequest[] = []; + fw.startAgentStream = (agent, trigger) => { dispatched.push(trigger); return realStart(agent, trigger); }; + try { + await framework.runUntilIdle(); + } finally { + fw.startAgentStream = realStart; + } + assert.equal(dispatched.length, 1, 'one turn: the ordinary wake'); + assert.equal(dispatched[0]!.unstick, undefined, 'the turn carries no unstick binding'); + assert.notEqual(dispatched[0]!.reason, 'unstick-attempt', 'nor its reason'); const outcome = await attempt; assert.equal(outcome.status === 'completed' && outcome.outcome, 'failed'); assert.match(String(outcome.status === 'completed' && outcome.error), /moved before the re-run started/); @@ -1073,6 +1084,49 @@ describe('operator-change gate: host/command unstick, planned at staging', () => assert.equal(framework.getUnstickOperation(change.id)!.attempts[0]!.outcome, 'failed', 'and it was not reported as the re-run'); }); + it("settles a re-run the inference policy declines, leaving the agent's next re-run free", async () => { + await unstick(1); + const first = asked[0]!; + await step(first, 1); + const fw = framework as unknown as { inferencePolicy: { shouldInfer: (...args: unknown[]) => boolean } }; + const realPolicy = fw.inferencePolicy; + fw.inferencePolicy = { ...realPolicy, shouldInfer: () => false }; + let outcome; + try { + outcome = await rerun(first, 1); + } finally { + fw.inferencePolicy = realPolicy; + } + assert.deepEqual(outcome.status === 'completed' && [outcome.outcome, /policy declined/.test(String(outcome.error))], ['failed', true]); + assert.equal(framework.getUnstickOperation(first.id)!.attempts[0]!.outcome, 'failed'); + host().addMessage('user', [{ type: 'text', text: 'second culprit' }]); + await unstick(1); + const second = asked[1]!; + await step(second, 1); + membrane.pushResponse(createMockResponse([{ type: 'text', text: 'back' }])); + const next = await rerun(second, 1); + assert.equal(next.status === 'completed' && next.outcome, 'responded', 'not refused as busy'); + }); + + it('releases a waiting re-run as interrupted when the framework stops', async () => { + await unstick(1); + const change = asked[0]!; + await step(change, 1); + const fw = framework as unknown as { inferencePolicy: { shouldInfer: (...args: unknown[]) => boolean } }; + const attempt = framework.rerunUnstick(change as never, { step: 1 }); // queued; nothing dispatches it + await framework.stop(); + assert.deepEqual(await attempt, { step: 1, status: 'interrupted' }); + void fw; + framework = await AgentFramework.create({ + storePath: join(tempDir, 'test.chronicle'), + membrane: membrane.asMembrane(), + agents: [{ name: 'scout', model: 'test-model', systemPrompt: 'You are scout.', maxTokens: 1000 }], + modules: [], + operatorChangeGate: async (c) => { asked.push(structuredClone(c)); return decide(c); }, + }); + assert.equal(framework.getUnstickOperation(change.id)!.attempts[0]!.status, 'launched', 'the journal agrees: launched, which reads interrupted'); + }); + it('runs back to back as a host would: each step after the previous re-run has fully settled', async () => { await unstick(3); const change = asked[0]!; From 1ae93fc2ab582f49198d34b910d0f385bfb08c73 Mon Sep 17 00:00:00 2001 From: Ada-1017 Date: Wed, 7 Oct 2026 18:13:10 -0400 Subject: [PATCH 46/73] feat(awareness): staged publication choices, activated by their change after the body; activations record what they queued The staged-intent outbox part of R2 (Agnes-1009's operator-change admission), rebased onto #250's head 485a3c7 for her to merge (room-225 #49421). Same contract as 528001a/c6e6cbf on 0019652 (room-222 #47572/#47575, June's design go #47681): the outbox owns publication authorization; the change's own journal (operator/changes) owns body commitment and recovery. - DiscordAwarenessPrepareInput takes `id` and `staged`. prepare({ id, staged: true, ... }) records a staged batch whose record's journal position is the choice's authorization for life. A staged batch is never dispatched, crash-completed, held at startup, or offered for the outbox's suppression body resume (preparedSuppressionsForBranch skips it: its change owns its body). It is listed, cancellable (counted as stopped in the receipt) and retractable by id, and retract('all') covers it. - prepare({ id }) validates the id (non-empty, <= 200, not 'all', not a retract request id); an existing id returns the batch as recorded, in any status, and writes nothing. - activateStaged(id, { targetBranch, refs, unmarked, notRemoved }), called once the change has established its body committed: one durable record, written after the body's state is synced (afterCommittedState), holds the facts (refs narrowed to the staged choice's; the audience only shrinks), the activation and the adds it requested, under the staging-time authorization (a key retracted since staging is born superseded). Returns queued (now or already: the recorded facts and count replayed, never a recount) | not-scheduled (cancelled while staged, or discarded; nothing written) | unresolved (the record could not be written, or the journal could not be read back after a failed write; the batch stays staged and a later call activates it). - discard accepts a staged batch (a dropped change). - Every activation records `queued` in its activated record: live activate, startup crash completion, staged activation, and so settleApplied (hide, turn undo) through settleActivation. settleActivation replays the recorded count for an already-active batch and refuses a staged one (unresolved, nothing written). Adapted to the new base: the settlement type (DiscordAwarenessSettlement) and settledMarkerReceipt are unchanged and carry the replayed count; settleApplied's input excludes id and staged; there are no checkpoints (538b136), so nothing here snapshots state. Tests (outbox 44, +1 here): the five staged/replay tests from 528001a, plus an applied change replaying its recorded count after a cancel, plus a staged suppression never offered for body resume (a mutant dropping that exclusion is caught). All six new behaviour tests fail on 485a3c7's outbox. Awareness/surgery files (delivery, outbox, offline recovery, operator surgery, recover CLI log, surgery marks receipt): 95/95. --- src/recovery/discord-awareness-outbox.ts | 182 ++++++++++++++++++++--- test/discord-awareness-outbox.test.ts | 132 ++++++++++++++++ 2 files changed, 290 insertions(+), 24 deletions(-) diff --git a/src/recovery/discord-awareness-outbox.ts b/src/recovery/discord-awareness-outbox.ts index 8850c3df4..5bfd873cd 100644 --- a/src/recovery/discord-awareness-outbox.ts +++ b/src/recovery/discord-awareness-outbox.ts @@ -138,10 +138,23 @@ export interface DiscordAwarenessBatchRecord { activationPolicy?: 'target-branch' | 'explicit'; /** Idempotent interval operations used to resume an interrupted suppression. */ suppressionIntervals?: DiscordSuppressionInterval[]; + /** + * A publication choice recorded when an operator change was staged, before + * its body change: its record's position is the choice's authorization for + * life. A staged batch is never dispatched, crash-completed or held; the + * change's own journal activates it (activateStaged) once its body change + * is established, or retires it (discard) when the change is dropped. + */ + staged?: boolean; } /** A surgery's request, as it prepares a batch. */ export interface DiscordAwarenessPrepareInput { + /** A caller-chosen identity: an existing batch of this id is returned as + * recorded, in whatever status, and nothing is written. */ + id?: string; + /** Record a staged publication choice (see DiscordAwarenessBatchRecord.staged). */ + staged?: boolean; agentName: string; sourceBranch: string; targetBranch: string; @@ -167,7 +180,7 @@ export type DiscordAwarenessSettlement = | { status: 'not-scheduled'; error: string } | { status: 'unresolved'; error: string }; -export type DiscordAwarenessBatchStatus = 'prepared' | 'active' | 'held' | 'discarded'; +export type DiscordAwarenessBatchStatus = 'staged' | 'prepared' | 'active' | 'held' | 'discarded'; export interface DiscordAwarenessBatch extends DiscordAwarenessBatchRecord { status: DiscordAwarenessBatchStatus; @@ -180,6 +193,9 @@ export interface DiscordAwarenessBatch extends DiscordAwarenessBatchRecord { /** Journal position of the operator's release: the authorization of the * requests the release queues. */ releaseAuthorization?: number; + /** What the activation recorded: when, and how many adds it requested (the + * original scheduling fact a retry replays, never a recount). */ + activation?: { at: number; queued: number }; held?: { reason: string; at: number; releaseActions: DiscordAwarenessReleaseAction[] }; cancelled?: { at: number; by?: string }; released?: { at: number; by?: string }; @@ -455,7 +471,15 @@ function isAddressed(message: DiscordMessageMetadataCarrier): boolean { type JournalRecord = | { t: 'batch'; at: number; batch: DiscordAwarenessBatchRecord } - | { t: 'activated'; at: number; batchId: string } + | { + t: 'activated'; + at: number; + batchId: string; + /** Adds this activation requested (not born superseded). */ + queued?: number; + /** A staged batch's facts, from the body change that committed. */ + facts?: { targetBranch: string; refs: DiscordAwarenessRef[]; unmarked?: number; notRemoved?: number }; + } /** An imported ledger's `active`: a historical fact about its marks, never * a certificate that a suppression's body completed. */ | { t: 'legacy-activated'; at: number; batchId: string } @@ -569,11 +593,21 @@ export class DiscordAwarenessOutbox { * delivers: activate() (or startup's crash completion) does. */ prepare(input: DiscordAwarenessPrepareInput): DiscordAwarenessBatch | null { + if (input.id !== undefined) { + const state = this.load(); + if (typeof input.id !== 'string' || input.id.length === 0 || input.id.length > 200 + || input.id === 'all' || state.retracts.has(input.id)) { + throw new Error(`Invalid Discord awareness batch id: ${JSON.stringify(boundDiscordAwarenessText(String(input.id)))}`); + } + const existing = state.batches.get(input.id); + if (existing) return structuredClone(existing); + } const batch = batchRecord(input); if (!batch) return null; - // Durable before the surgery's switch, so startup can crash-complete it. + // Durable before the surgery's switch, so startup can crash-complete it; + // a staged choice, before anything can act on it. this.append([{ t: 'batch', at: batch.createdAt, batch }], { durable: true }); - return { ...structuredClone(batch), status: 'prepared' }; + return structuredClone(this.load().batches.get(batch.id)!); } /** @@ -592,7 +626,7 @@ export class DiscordAwarenessOutbox { * Never throws. */ settleApplied( - input: Omit + input: Omit & { branch: string }, ): { batchId: string; marks: number; settled: DiscordAwarenessSettlement } | null { const { branch, ...rest } = input; @@ -637,6 +671,7 @@ export class DiscordAwarenessOutbox { if (!batch) throw new Error(`Discord awareness batch not found: ${batchId}`); if (batch.status === 'active') return 0; if (batch.status !== 'prepared') { + // A staged batch is activated by its change (activateStaged), never here. throw new Error(`Discord awareness batch ${batchId} is ${batch.status}, not prepared`); } if (batch.cancelled) { @@ -644,12 +679,11 @@ export class DiscordAwarenessOutbox { throw new Error(`Discord awareness batch ${batchId} was cancelled before activation`); } const at = Date.now(); - const records: JournalRecord[] = [{ t: 'activated', at, batchId }]; const requested = this.requestRecords(state, batch.refs.map((ref) => ({ ...ref, action: 'add' as const })), batch.emoji, { kind: 'batch', batchId, }, at, batch.authorization); - records.push(...requested); + const records: JournalRecord[] = [{ t: 'activated', at, batchId, queued: liveRequests(requested) }, ...requested]; // The activation asserts the body change just committed (the switch, or // the last redaction): sync that first, so a crash can never leave marks // active for a change the store lost. @@ -671,18 +705,21 @@ export class DiscordAwarenessOutbox { settleActivation(batchId: string): DiscordAwarenessSettlement { let detail: string; try { + const before = this.load().batches.get(batchId); + // Already active (a retry): replay what its activation recorded. + if (before?.status === 'active') return { status: 'queued', queued: this.recordedQueued(batchId) }; + if (before?.status === 'staged') { + return { status: 'unresolved', error: `Discord awareness batch ${batchId} is staged: its change activates it (activateStaged)` }; + } return { status: 'queued', queued: this.activate(batchId) }; } catch (error) { detail = boundDiscordAwarenessText(error instanceof Error ? error.message : String(error)); } try { - const state = this.load(); - const current = state.batches.get(batchId); + const current = this.load().batches.get(batchId); if (current?.status === 'active') { // The activation landed but its barrier failed: report what it queued. - const queued = [...state.ops.values()] - .filter((op) => op.cause.kind === 'batch' && op.cause.batchId === batchId && !op.cancelled).length; - return { status: 'queued', queued }; + return { status: 'queued', queued: this.recordedQueued(batchId) }; } if (this.discard(batchId)) return { status: 'not-scheduled', error: detail }; } catch (error) { @@ -691,13 +728,96 @@ export class DiscordAwarenessOutbox { return { status: 'unresolved', error: detail }; } + /** The adds an active batch's activation recorded; for an activation + * recorded without the count, its own requests still standing. */ + private recordedQueued(batchId: string): number { + const state = this.load(); + const batch = state.batches.get(batchId); + if (batch?.activation) return batch.activation.queued; + return [...state.ops.values()] + .filter((op) => op.cause.kind === 'batch' && op.cause.batchId === batchId && !op.cancelled).length; + } + /** - * Retire a batch whose surgery did not complete. Only a prepared batch can - * be discarded; returns false when nothing was retired. + * Activate a staged batch once its change has established that the body + * change committed. One durable record, written after the body's state is + * synced, holds the change's facts (refs narrowed to the staged choice's; + * the publication audience only shrinks), the activation and the adds it + * requested, under the staging-time authorization: a key retracted since + * staging is born superseded. Never throws for the batch's own state: + * - `queued`: activated now, or already (a retry), with the facts and + * count its activation recorded, never a recount; + * - `not-scheduled`: cancelled while staged, or discarded; nothing written; + * - `unresolved`: the record could not be written; the batch stays staged + * and a later call activates it. + */ + activateStaged( + batchId: string, + facts: { targetBranch: string; refs: DiscordAwarenessRef[]; unmarked?: number; notRemoved?: number }, + ): + | { status: 'queued'; queued: number; batch: DiscordAwarenessBatch } + | { status: 'not-scheduled'; error: string } + | { status: 'unresolved'; error: string } { + const state = this.load(); + const batch = state.batches.get(batchId); + if (!batch) throw new Error(`Discord awareness batch not found: ${batchId}`); + if (batch.status === 'active') { + return { status: 'queued', queued: this.recordedQueued(batchId), batch: structuredClone(batch) }; + } + if (batch.status === 'discarded') { + return { status: 'not-scheduled', error: `Discord awareness batch ${batchId} was discarded` }; + } + if (batch.status !== 'staged') { + throw new Error(`Discord awareness batch ${batchId} is ${batch.status}, not staged`); + } + if (batch.cancelled) { + return { status: 'not-scheduled', error: `Discord awareness batch ${batchId} was cancelled while staged` }; + } + const staged = new Set(batch.refs.map(refKey)); + const refs = dedupeRefs(facts.refs).filter((ref) => staged.has(refKey(ref))); + const at = Date.now(); + const requested = this.requestRecords(state, refs.map((ref) => ({ ...ref, action: 'add' as const })), batch.emoji, { + kind: 'batch', + batchId, + }, at, batch.authorization); + const records: JournalRecord[] = [{ + t: 'activated', + at, + batchId, + queued: liveRequests(requested), + facts: { + targetBranch: facts.targetBranch, + refs, + ...(facts.unmarked !== undefined ? { unmarked: facts.unmarked } : {}), + ...(facts.notRemoved !== undefined ? { notRemoved: facts.notRemoved } : {}), + }, + }, ...requested]; + try { + this.append(records, { afterCommittedState: true, durable: true }); + } catch (error) { + let current: DiscordAwarenessBatch | undefined; + try { + current = this.load().batches.get(batchId); + } catch (readError) { + return { status: 'unresolved', error: `${errorText(error)}; reading the journal back also failed: ${errorText(readError)}` }; + } + if (current?.status === 'active') { + return { status: 'queued', queued: this.recordedQueued(batchId), batch: structuredClone(current) }; + } + return { status: 'unresolved', error: errorText(error) }; + } + const after = this.load().batches.get(batchId)!; + return { status: 'queued', queued: this.recordedQueued(batchId), batch: structuredClone(after) }; + } + + /** + * Retire a batch whose surgery did not complete, or a staged batch whose + * change was dropped. Only a prepared or staged batch can be discarded; + * returns false when nothing was retired. */ discard(batchId: string): boolean { const batch = this.load().batches.get(batchId); - if (!batch || batch.status !== 'prepared') return false; + if (!batch || (batch.status !== 'prepared' && batch.status !== 'staged')) return false; // Retiring a batch retires its obligations (marks and any body resume): // the branch state its surgery left (a restored source, or a completed // body) is synced first, and the record is durable, so a crash can never @@ -712,12 +832,14 @@ export class DiscordAwarenessOutbox { * whose body was never recorded complete. Body recovery is independent of * the marks: a prepared batch is crash-completed (resume, then activate); * any other, held, released or cancelled, has only its body resumed. - * A batch its own surgery retired (discarded) is not resumed. Branch + * A batch its own surgery retired (discarded) is not resumed, and neither + * is a staged one: its change's own journal owns its body. Branch * ancestry is never consulted. */ preparedSuppressionsForBranch(branchName: string): DiscordAwarenessBatch[] { return [...this.load().batches.values()] - .filter((batch) => batch.status !== 'discarded' + // A staged batch's body belongs to its change's own journal. + .filter((batch) => batch.status !== 'discarded' && batch.status !== 'staged' && !batch.suppressionComplete && batch.activationPolicy === 'explicit' && !!batch.suppressionIntervals?.length @@ -771,15 +893,15 @@ export class DiscordAwarenessOutbox { if (batch.targetBranch === activeBranchName) { if (suppression) continue; // resumePreparedDiscordSuppressions completes it - records.push({ t: 'activated', at, batchId: batch.id }); - records.push(...this.requestRecords( + const requested = this.requestRecords( state, batch.refs.map((ref) => ({ ...ref, action: 'add' as const })), batch.emoji, { kind: 'batch', batchId: batch.id }, at, batch.authorization, - )); + ); + records.push({ t: 'activated', at, batchId: batch.id, queued: liveRequests(requested) }, ...requested); activated.push(batch.id); } else { records.push({ @@ -822,7 +944,7 @@ export class DiscordAwarenessOutbox { kind: 'batch', // A prepared batch (its surgery still running, or its bookkeeping // unresolved) has no requests yet: every one of its marks is stopped. - cancelled: batch.status === 'prepared' && !batch.cancelled ? batch.refs.length : 0, + cancelled: (batch.status === 'prepared' || batch.status === 'staged') && !batch.cancelled ? batch.refs.length : 0, heldDropped: batch.held?.releaseActions.length ?? 0, inFlight: 0, unknown: 0, @@ -1390,13 +1512,24 @@ function applyRecords(state: JournalState, records: JournalRecord[]): void { switch (record.t) { case 'batch': if (!state.batches.has(record.batch.id)) { - state.batches.set(record.batch.id, { ...structuredClone(record.batch), status: 'prepared', authorization: position }); + state.batches.set(record.batch.id, { + ...structuredClone(record.batch), + status: record.batch.staged ? 'staged' : 'prepared', + authorization: position, + }); } break; case 'activated': { const batch = state.batches.get(record.batchId); if (batch && batch.status !== 'discarded') { batch.status = 'active'; + if (record.queued !== undefined) batch.activation = { at: record.at, queued: record.queued }; + if (record.facts) { + batch.targetBranch = record.facts.targetBranch; + batch.refs = record.facts.refs.map((ref) => ({ ...ref })); + if (record.facts.unmarked !== undefined) batch.unmarked = record.facts.unmarked; + if (record.facts.notRemoved !== undefined) batch.notRemoved = record.facts.notRemoved; + } // Only a surgery (or its resume) activates a suppression, after its // last redaction synced: activation records the body complete. // Release does not, and neither does an imported `active`. @@ -1411,7 +1544,7 @@ function applyRecords(state: JournalState, records: JournalRecord[]): void { } case 'discarded': { const batch = state.batches.get(record.batchId); - if (batch && batch.status === 'prepared') batch.status = 'discarded'; + if (batch && (batch.status === 'prepared' || batch.status === 'staged')) batch.status = 'discarded'; break; } case 'held': { @@ -1674,7 +1807,7 @@ function batchRecord(input: DiscordAwarenessPrepareInput): DiscordAwarenessBatch const refs = dedupeRefs(input.refs); if (refs.length === 0 && !input.suppressionIntervals?.length) return null; return { - id: randomUUID(), + id: input.id ?? randomUUID(), agentName: input.agentName, sourceBranch: input.sourceBranch, targetBranch: input.targetBranch, @@ -1688,6 +1821,7 @@ function batchRecord(input: DiscordAwarenessPrepareInput): DiscordAwarenessBatch ...(input.suppressionIntervals?.length ? { suppressionIntervals: input.suppressionIntervals.map((interval) => ({ ...interval })) } : {}), + ...(input.staged ? { staged: true } : {}), }; } diff --git a/test/discord-awareness-outbox.test.ts b/test/discord-awareness-outbox.test.ts index 7e47bb3f0..6b0697bbf 100644 --- a/test/discord-awareness-outbox.test.ts +++ b/test/discord-awareness-outbox.test.ts @@ -443,6 +443,138 @@ test('startup records a crash-completion activation only after syncing the store assert.deepEqual(h.reopen().recoverAtStartup('rollback/cairn/1').activated, [batch.id]); })); +test('a staged choice: authorized at staging, never dispatched or touched at startup, activated by its change after the body', withJournal((outbox, h) => { + const due = (o: DiscordAwarenessOutbox) => o.pendingDispatches('discord').map((d) => `${d.action}:${d.key.messageId}`).sort(); + const stage = (id: string, refs: DiscordAwarenessRef[]) => outbox.prepare({ + id, staged: true, agentName: 'cairn', sourceBranch: 'main', targetBranch: `undo/cairn/${id}`, refs, scope: 'addressed', + })!; + + const a = stage('op-a', [ref('m1'), ref('m2'), ref('m3')]); + assert.equal(a.status, 'staged'); + assert.equal(outbox.pendingDispatches('discord').length, 0); + assert.deepEqual(h.reopen().recoverAtStartup('undo/cairn/op-a'), { activated: [], held: [], unknownAttempts: 0 }); + assert.equal(outbox.view().find((v) => v.id === 'op-a')!.kind, 'batch'); + // Its body is its change's: even a staged suppression is never offered + // for the outbox's body resume. + outbox.prepare({ + id: 'op-s', staged: true, agentName: 'cairn', sourceBranch: 'main', targetBranch: 'main', refs: [ref('s1')], scope: 'addressed', + activationPolicy: 'explicit', suppressionIntervals: [{ fromId: 's1', toId: 's1' }], + }); + assert.deepEqual(outbox.preparedSuppressionsForBranch('main'), []); + outbox.discard('op-s'); + + // The change commits its body: the facts narrow the audience (m4 was + // never chosen, m3 was not cut) and count the late arrival unmarked. + const settled = outbox.activateStaged('op-a', { targetBranch: 'undo/cairn/op-a', refs: [ref('m1'), ref('m2'), ref('m4')], unmarked: 1, notRemoved: 1 }); + assert.equal(settled.status, 'queued'); + assert.equal(settled.status === 'queued' && settled.queued, 2); + const recorded = settled.status === 'queued' ? settled.batch : null; + assert.deepEqual(recorded!.refs.map((r) => r.messageId), ['m1', 'm2']); + assert.deepEqual({ unmarked: recorded!.unmarked, notRemoved: recorded!.notRemoved, status: recorded!.status }, { unmarked: 1, notRemoved: 1, status: 'active' }); + assert.deepEqual(due(outbox), ['add:m1', 'add:m2']); + + // A retry replays the recorded facts and count, even after the operator + // cancelled in between; it writes nothing and recounts nothing. + outbox.cancel('op-a'); + const records = h.store.getRecordIdsByType(DISCORD_AWARENESS_RECORD_TYPE).length; + const retry = outbox.activateStaged('op-a', { targetBranch: 'elsewhere', refs: [ref('m9')], unmarked: 7 }); + assert.equal(retry.status === 'queued' && retry.queued, 2); + assert.deepEqual(retry.status === 'queued' && retry.batch.refs.map((r) => r.messageId), ['m1', 'm2']); + assert.deepEqual(outbox.settleActivation('op-a'), { status: 'queued', queued: 2 }); + assert.equal(h.store.getRecordIdsByType(DISCORD_AWARENESS_RECORD_TYPE).length, records); +})); + +test('a staged choice keeps its staging-time authority: a retract while it waits wins, and cancelling it never revives', withJournal((outbox, h) => { + const due = (o: DiscordAwarenessOutbox) => o.pendingDispatches('discord').map((d) => `${d.action}:${d.key.messageId}`).sort(); + const stage = (id: string, refs: DiscordAwarenessRef[]) => outbox.prepare({ + id, staged: true, agentName: 'cairn', sourceBranch: 'main', targetBranch: `undo/cairn/${id}`, refs, scope: 'all', + })!; + // retract('all') covers staged choices; retracting by the staged id works too. + stage('op-b', [ref('m1'), ref('m2')]); + stage('op-c', [ref('m3')]); + const all = outbox.retract('all'); + assert.equal(all.removalsQueued, 3); + // A choice staged after the retract is a later authorization. + stage('op-d', [ref('m4')]); + const restarted = h.reopen(); // the authority survives a restart + const { requestId } = restarted.retract('op-d'); + restarted.cancel(requestId); // stopping the retract revives nothing + const b = restarted.activateStaged('op-b', { targetBranch: 'undo/cairn/op-b', refs: [ref('m1'), ref('m2')] }); + const c = restarted.activateStaged('op-c', { targetBranch: 'undo/cairn/op-c', refs: [ref('m3')] }); + const d = restarted.activateStaged('op-d', { targetBranch: 'undo/cairn/op-d', refs: [ref('m4')] }); + assert.deepEqual([b, c, d].map((r) => r.status === 'queued' && r.queued), [0, 0, 0]); + assert.deepEqual(due(restarted), ['remove:m1', 'remove:m2', 'remove:m3']); +})); + +test('a staged choice cancelled or dropped before its body is never scheduled; a failed write leaves it staged for the retry', withJournal((outbox, h) => { + const stage = (id: string) => outbox.prepare({ + id, staged: true, agentName: 'cairn', sourceBranch: 'main', targetBranch: `undo/cairn/${id}`, refs: [ref(`m-${id}`)], scope: 'all', + })!; + stage('op-e'); + assert.equal(outbox.cancel('op-e').cancelled, 1, 'a staged choice counts as stopped'); + const records = h.store.getRecordIdsByType(DISCORD_AWARENESS_RECORD_TYPE).length; + const cancelled = outbox.activateStaged('op-e', { targetBranch: 'undo/cairn/op-e', refs: [ref('m-op-e')] }); + assert.equal(cancelled.status, 'not-scheduled'); + assert.equal(h.store.getRecordIdsByType(DISCORD_AWARENESS_RECORD_TYPE).length, records, 'nothing written'); + + stage('op-f'); + assert.equal(outbox.discard('op-f'), true); + assert.equal(outbox.batches().find((b) => b.id === 'op-f')!.status, 'discarded'); + assert.equal(outbox.activateStaged('op-f', { targetBranch: 'x', refs: [] }).status, 'not-scheduled'); + + stage('op-g'); + const sync = h.store.sync.bind(h.store); + (h.store as any).sync = () => { throw new Error('injected sync failure'); }; + let failed; + try { + failed = outbox.activateStaged('op-g', { targetBranch: 'undo/cairn/op-g', refs: [ref('m-op-g')] }); + } finally { + (h.store as any).sync = sync; + } + assert.equal(failed.status, 'unresolved'); + assert.equal(outbox.batches().find((b) => b.id === 'op-g')!.status, 'staged'); + const again = outbox.activateStaged('op-g', { targetBranch: 'undo/cairn/op-g', refs: [ref('m-op-g')] }); + assert.equal(again.status === 'queued' && again.queued, 1); + // A staged batch is not the live surgeries' to settle. + stage('op-h'); + assert.equal(outbox.settleActivation('op-h').status, 'unresolved'); + assert.equal(outbox.batches().find((b) => b.id === 'op-h')!.status, 'staged'); +})); + +test('a batch id is validated, and an existing id is returned as recorded with nothing written', withJournal((outbox, h) => { + const input = { agentName: 'cairn', sourceBranch: 'main', targetBranch: 't', refs: [ref('m1')], scope: 'all' as const }; + const first = outbox.prepare({ ...input, id: 'op-1', staged: true })!; + const records = h.store.getRecordIdsByType(DISCORD_AWARENESS_RECORD_TYPE).length; + const again = outbox.prepare({ ...input, id: 'op-1', refs: [ref('m2'), ref('m3')] })!; + assert.deepEqual({ status: again.status, refs: again.refs.map((r) => r.messageId) }, { status: 'staged', refs: ['m1'] }); + assert.equal(again.authorization, first.authorization); + assert.equal(h.store.getRecordIdsByType(DISCORD_AWARENESS_RECORD_TYPE).length, records); + // Whatever its status: a discarded id stays discarded. + outbox.discard('op-1'); + assert.equal(outbox.prepare({ ...input, id: 'op-1' })!.status, 'discarded'); + const { requestId } = outbox.retract('all'); + for (const bad of ['', 'all', 'x'.repeat(201), requestId]) { + assert.throws(() => outbox.prepare({ ...input, id: bad }), /Invalid Discord awareness batch id/); + } +})); + +test('live activations record what they queued, and settleActivation replays it after a cancel', withJournal((outbox) => { + const batch = outbox.prepare({ agentName: 'cairn', sourceBranch: 'main', targetBranch: 't', refs: [ref('m1'), ref('m2')], scope: 'all' })!; + assert.deepEqual(outbox.settleActivation(batch.id), { status: 'queued', queued: 2 }); + outbox.cancel(batch.id); + assert.deepEqual(outbox.settleActivation(batch.id), { status: 'queued', queued: 2 }); + assert.equal(outbox.batches()[0].activation!.queued, 2); +})); + +test('an applied change records what its activation queued, and settling it again replays that count', withJournal((outbox) => { + const applied = outbox.settleApplied({ agentName: 'cairn', branch: 'main', refs: [ref('m1'), ref('m2')], scope: 'addressed' })!; + assert.deepEqual(applied.settled, { status: 'queued', queued: 2 }); + outbox.cancel(applied.batchId); + // The original scheduling fact, not a recount of what still stands. + assert.deepEqual(outbox.settleActivation(applied.batchId), { status: 'queued', queued: 2 }); + assert.equal(outbox.batches()[0].activation!.queued, 2); +})); + test('a retract request can be cancelled by the id its receipt returned, even after a restart', withJournal((outbox, h) => { const batch = activeBatch(outbox, [ref('m1'), ref('m2'), ref('m3')]); for (const id of ['m1', 'm2', 'm3']) answer(outbox, id, 'confirmed'); From 76d0e9ef619ddf1e127e4382066d881f1de6f5f9 Mon Sep 17 00:00:00 2001 From: Agnes-1009 Date: Wed, 7 Oct 2026 16:20:16 -0400 Subject: [PATCH 47/73] feat(framework): operator/changes owns gated body recovery; gated hide and undo by messages; staged-choice marks; held-lease and preview-bound surgeries (R2) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The marks-intent lifecycle agreed in room-222: Kit #47551/#47630, Ada #47575/#47609, June's design go #47681. Also the AF side of the preview-context contract from room-225: Rowan #48068, Ada #48190, my #48201, June #48216, Rowan #48242. It builds on Ada's outbox commit (c6e6cbf: staged batches, activateStaged, discard of staged, recorded activation counts). Three facts, each with one owner (June #47507): 1. **Choice.** The operator's publication choice: a `staged` awareness batch `op-`, recorded at staging when frozen refs exist. Its journal position is the authorization for life, so a retract or cancel made while the change waits outranks it. 2. **Prediction.** Each application attempt's evidence, recorded under the lease before its body change: `attempt {n, evidence}` in operator/changes. Evidence is the target, the source head, the exact hide ids with fingerprints, the removal count and the marks facts. 3. **Outcome.** The established outcome: `outcome {n, removed, markers}`, recorded once after the body change and the activation, and immutable. Zero-ref accounting lives here too. operator/changes - A store-bound, branch-independent RecordJournal holding attempt, switched, outcome, completed and dropped entries. Every record survives checkpoints. - `getOperatorChangeRecord(changeId)` lets the host reconcile a revision after a restart. - `dropResolvedOperatorChange(change)` discards a staged choice and refuses later application. A change that began to apply can't be dropped. - The gate refusing at staging discards the choice too. Commitment and recovery (Kit #47465, #47630; June #47681). A branch cut has committed if any of these holds: - its `switched` record (written right after the switch); - its destination is active; - its destination was written past its branch point (today's used-destination evidence, kept). Being back on the source proves nothing. The outcome of a committed cut is established from that attempt's evidence and it is never cut again. A cut still on its source with none of that evidence is a fresh attempt n+1, under the same staging authorization and with fresh evidence, so a second late arrival is counted. A hide's interrupted attempt is finished, never re-planned: exactly its own ids, each checked unchanged, so what it removed earlier still counts. If the branch left the source meanwhile it is refused as stale (outcome unknown). Establishing an outcome: - `activateStaged(op-, {targetBranch, refs ⊆ frozen, unmarked, notRemoved})` from the attempt's evidence. An `unresolved` result leaves the outcome unrecorded and throws, so a retry settles it. - Then the outcome record, then the bookkeeping, then `completed`. The operator-log record is written once per change, matched by kind and change id, so a failure at `completed` doesn't log twice. Startup reconciliation runs the same logic before modules or traffic, so a committed body never waits on its host to settle its marks. On the switch record: a RecordJournal append is a typed record, and it advances the active branch's head (probed). So the `switched` append itself makes the destination "used", and the two proofs coincide; a mutant dropping `switched` as proof is not separable by any test. It stays as the explicit, host-readable proof, independent of that Chronicle behaviour. Gated host/command hide and undo by messages (they previously applied immediately even with a gate) - **Hide:** ResolvedHideChange carries the exact store ids and fingerprints of the named range, and refs frozen from it. - **Undo by messages:** ResolvedUndoMessagesChange carries the tail (snapped to its body group) with its fingerprint, and refs frozen from the messages after it. Its destination is `undo-msgs//op-`, and an existing one an interrupted attempt created is reused. - **Undo-turns:** now runs through the same journal. The branch-state operatorCut markers and their sync are gone, and so is the completedOperatorCuts set. - **Staging:** no gated path requires an idle agent to stage; application waits for the safe boundary. Held-lease and preview-bound surgeries - `rollbackToMessage` and `suppressMessages` take `lease` and run under a host's held lease instead of reserving: `surgeryReservation` checks the lease is the one held. - `getStoreIdentity()` is public. - `previewSurgeryMarks` returns `context: {storeId, branch}`. - Both surgeries take `expected: {storeId?, branch?}`, checked by `checkExpectedContext` immediately once the reservation or lease is held and before anything is prepared or changed. A mismatch is `stale`: "chosen in another store", or "the active branch moved from X to Y". Arrivals on the same branch are unaffected. Tests (operator-change-gate, 78) - **Journal seams** (attempt, cut, switched, outcome, log, completed), with and without later work: a committed cut is recovered and never cut again, an uncommitted one becomes a fresh attempt, it's logged once, and redo holds without later work while later work is kept. - **Kit's failure point with a late arrival:** the activation's recorded receipt {queued 1, unmarked 1}, and one batch. - **June's case:** the outcome record fails, the destination is used and left, the source is restored; the established outcome is recovered with no new branch and the source stays active. - **Used destination as proof** when the switch record was lost. - **Left at once:** never cut again. - **Retract while staged** ('all' and by id): queued 0, no adds sent. - **Cancel while staged:** not-scheduled, with the cut applied. - **Cut failure, then a second arrival:** a new attempt counting both, the first prediction kept as history. - **Startup** establishes the outcome. - **Gate refusal and drop** discard the choice, and a dropped change is stale. - **Hide:** staged exactly and applied; an interrupted hide finished from its own evidence; an edited message is stale, and so is an interrupted hide after the branch moved. - **Undo by messages:** staged at its tail and cut to its own branch with a late arrival unmarked; an edited tail is stale. - **Rollback under a held lease;** refused there without the lease. - **Rowan's rebind case at AF level:** two stores with the same ids, a preview on A, the call on B refused before any change, and the same for a suppression. - **Branch moved since the preview** is refused, and a same-branch arrival stays within the rollback. Mutants caught: - the used destination not counting as proof; - refusal keeping the staged choice; - a dropped change still applying; - a log that isn't idempotent; - no startup reconciliation; - no expected-store check; - no expected-branch check. 24 suites, Ada's outbox, awareness and offline-recovery included: 402/402. Co-Authored-By: Agnes (Claude Opus 5.5) --- src/framework.ts | 1009 +++++++++++++++++++++++------ src/operator-change.ts | 126 +++- src/operator-log.ts | 5 + test/operator-change-gate.test.ts | 454 +++++++++++-- 4 files changed, 1334 insertions(+), 260 deletions(-) diff --git a/src/framework.ts b/src/framework.ts index 89696d4c1..230f70ed0 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -18,6 +18,12 @@ import { type ResolvedOperatorChange, type ResolvedPresentationChange, type ResolvedSettingsChange, + type FrozenMarks, + type OperatorChangeEvidence, + type OperatorChangeOutcome, + type OperatorChangeRecord, + type ResolvedHideChange, + type ResolvedUndoMessagesChange, type ResolvedUndoTurnsChange, type ResolvedUnstickChange, type RuntimeSettingsValues, @@ -597,6 +603,35 @@ type UnstickJournalEntry = | { kind: 'attempt-done'; operationId: string; step: number; outcome: 'responded' | 'refused' | 'failed'; category?: string; error?: string }; type UnstickJournalSnapshot = Record; +type OperatorChangesEntry = + | { kind: 'attempt'; changeId: string; changeKind: OperatorChangeRecord['kind']; agent: string; n: number; at: number; evidence: OperatorChangeEvidence } + | { kind: 'switched'; changeId: string; n: number } + | { kind: 'outcome'; changeId: string; outcome: OperatorChangeOutcome } + | { kind: 'completed'; changeId: string } + | { kind: 'dropped'; changeId: string; changeKind: OperatorChangeRecord['kind']; agent: string; at: number }; +type OperatorChangesSnapshot = Record; + +/** Fold one operator/changes entry in. An outcome is immutable: the first + * recorded stands. */ +function reduceOperatorChangesEntry(records: Map, entry: OperatorChangesEntry): void { + if (entry.kind === 'attempt' || entry.kind === 'dropped') { + const record = records.get(entry.changeId) + ?? { changeId: entry.changeId, kind: entry.changeKind, agent: entry.agent, attempts: [] }; + if (entry.kind === 'attempt') { + if (!record.attempts.some((a) => a.n === entry.n)) record.attempts.push({ n: entry.n, at: entry.at, evidence: entry.evidence }); + } else { + record.dropped ??= { at: entry.at }; + } + records.set(entry.changeId, record); + return; + } + const record = records.get(entry.changeId); + if (!record) return; + if (entry.kind === 'switched') record.switched = entry.n; + else if (entry.kind === 'outcome') record.outcome ??= entry.outcome; + else record.completed = true; +} + function reduceUnstickEntry(ops: Map, entry: UnstickJournalEntry): void { const op = ops.get(entry.operationId) ?? { operationId: entry.operationId, agent: 'agent' in entry ? entry.agent : '', steps: [], attempts: [] }; @@ -1622,10 +1657,13 @@ export class AgentFramework { /** Admissions applyResolvedOperatorChange is carrying out, by admission id: * each lets exactly its change's tool call through the gate. */ private activeAdmissions: Map = new Map(); - /** Admitted undo cuts whose bookkeeping finished in this process. */ - private completedOperatorCuts: Set = new Set(); /** This store's identity (storeIdentity), once read or minted. */ private storeIdentityValue: string | null = null; + /** The operator/changes journal, loaded on first use (changesLedger). */ + private changesJournalState: { + journal: RecordJournal; + records: Map; + } | null = null; /** The unstick journal and its reduced operations (unstickLedger). */ private unstickJournalState: { journal: RecordJournal; @@ -1876,6 +1914,9 @@ export class AgentFramework { // Finish any branch-local suppression interrupted after Chronicle switched // branches. This runs before modules, MCPL connections, or inbound traffic. await framework.resumePreparedDiscordSuppressions(); + // Establish the outcome of any gated body change that committed before a + // crash recorded it, before modules, connections or traffic. + await framework.reconcileOperatorChanges(); // Add modules for (const module of config.modules) { @@ -5672,6 +5713,14 @@ export class AgentFramework { // branch, addressed by Discord message id. Unlike undo this is a // removal-in-place (chronicle redact), not a branch rewind. if (params.command === 'hide') { + if (this.operatorChangeGate) { + if (!params.fromMessageId) return { ok: false, error: 'hide: fromMessageId is required' }; + const fromMessageId = params.fromMessageId; + return this.stageOperatorCommand(() => this.resolveHide(agentName, fromMessageId, params.toMessageId, hostCommandRequester(serverId, params), { + marks: params.marks === 'addressed' || params.marks === 'all' ? { scope: params.marks } : 'none', + serverId, + })); + } const agent = this.agents.get(agentName)!; if (agent.state.status !== 'idle') { return { ok: false, error: `Cannot hide while agent is ${agent.state.status}` }; @@ -5797,6 +5846,13 @@ export class AgentFramework { // time-travel branching; see chronicle's rfc-state-item-origins addendum). // No turn checkpoints involved, so this reaches past turn boundaries. if (typeof params.messages === 'number' && params.messages > 0) { + if (this.operatorChangeGate) { + const count = Math.max(1, Math.min(50, Math.floor(params.messages))); + return this.stageOperatorCommand(() => this.resolveUndoMessages(agentName, count, hostCommandRequester(serverId, params), { + marks: params.marks === 'addressed' || params.marks === 'all' ? { scope: params.marks } : 'none', + serverId, + })); + } const agent = this.agents.get(agentName)!; if (agent.state.status !== 'idle') { return { ok: false, error: `Cannot undo while agent is ${agent.state.status}` }; @@ -6445,6 +6501,35 @@ export class AgentFramework { * flushes messages that deferred behind the reservation, as the puppet * does. Always release in `finally`: a token nobody clears wedges the fleet. */ + /** + * Refuse a surgery chosen in another context than the one now under its + * reservation: another store (a session switch since its preview) or + * another active branch. Checked once the reservation is held and before + * anything is prepared or changed, so no switch can come between the check + * and the change. Arrivals on the same branch are fine. + */ + private checkExpectedContext(expected: { storeId?: string; branch?: string } | undefined, what: string): void { + if (!expected) return; + if (expected.storeId !== undefined && expected.storeId !== this.storeIdentity()) { + throw new OperatorActionError('stale', `${what} no longer applies: it was chosen in another store`); + } + const branch = this.store.currentBranch().name; + if (expected.branch !== undefined && expected.branch !== branch) { + throw new OperatorActionError('stale', `${what} no longer applies: the active branch moved from ${expected.branch} to ${branch}`); + } + } + + /** The reservation a surgery runs under: the caller's held lease (checked, + * and released by its holder, so this release does nothing), or one + * taken now. */ + private surgeryReservation(verb: string, agentName: string, lease?: SafeBoundaryLease): () => void { + if (lease) { + this.heldLeaseTokens(lease, verb); + return () => {}; + } + return this.reserveStoreForSurgery(verb, agentName); + } + private reserveStoreForSurgery(verb: string, agentName: string): () => void { if (this.surgeryHold) { throw new OperatorActionError( @@ -6946,23 +7031,24 @@ export class AgentFramework { requested?: number; lastVisible?: { participant?: string; role?: string; preview?: string } | null; }> { - const agent = this.agents.get(agentName)!; - if (agent.state.status !== 'idle') { - return { ok: false, error: `Cannot undo while agent is ${agent.state.status}` }; - } + // Staging is a proposal: it needs no idle agent. Application waits for + // the safe boundary under the host's lease. const change = this.resolveUndoTurns(agentName, requested, requester, publication); if (!change) return { ok: true, undone: 0, requested, lastVisible: null }; - const answer = await this.askOperatorChangeGate(change); - if (answer.outcome === 'gate-failed') { - return { ok: false, code: 'gate-failed', error: `Operator change refused: ${answer.error}`, requested }; - } - return { ok: false, code: 'staged', error: answer.receipt.text, staged: { id: answer.receipt.id }, requested }; + return { ...(await this.stageChange(change)), requested }; } // --------------------------------------------------------------------------- // Admitted unstick (ResolvedUnstickChange): planned steps, re-runs, journal // --------------------------------------------------------------------------- + /** This store's identity (see storeIdentity): what a preview's `context` + * and every resolved operator change carry, so a surgery or change chosen + * against one store is refused against any other. */ + getStoreIdentity(): string { + return this.storeIdentity(); + } + /** * This store's identity: a UUID recorded once, in a typed record that * doesn't follow branch switches, so it names the store whichever branch @@ -7056,6 +7142,75 @@ export class AgentFramework { } } + /** The operator/changes journal, loaded and reduced on first use: every + * gated body change's attempts with their evidence, the switch that + * proves a cut, and the established outcome. Store-bound and + * branch-independent (RecordJournal), like the unstick journal. */ + private changesLedger(): { + journal: RecordJournal; + records: Map; + } { + if (!this.changesJournalState) { + const journal = new RecordJournal(this.store, { type: 'operator/changes' }); + this.changesJournalState = { journal, records: new Map() }; + this.reloadChangesJournal(); + } + return this.changesJournalState; + } + + /** Rebuild the reduced records from exactly what the journal holds. */ + private reloadChangesJournal(): void { + const state = this.changesJournalState!; + const { snapshot, entries } = state.journal.load(); + state.records = new Map(Object.entries(snapshot ?? {}).map(([id, record]) => [id, structuredClone(record)])); + for (const { entry } of entries) reduceOperatorChangesEntry(state.records, entry); + } + + /** Append one entry, durable, and reduce it. A failure after the entry may + * have reached the store reconciles before rethrowing. Every record stays + * through checkpoints: each is the authority for its change's outcome. */ + private journalChange(entry: OperatorChangesEntry): void { + const ledger = this.changesLedger(); + try { + ledger.journal.append(entry, { durable: true }); + } catch (error) { + if (ledger.journal.needsReconcile) this.reloadChangesJournal(); + throw error; + } + reduceOperatorChangesEntry(ledger.records, entry); + if (ledger.journal.entriesSinceCheckpoint >= 64) { + try { + ledger.journal.checkpoint(Object.fromEntries(ledger.records)); + } catch (error) { + console.error('[operator-changes] journal checkpoint failed (replay stays longer):', error instanceof Error ? error.message : error); + if (ledger.journal.needsReconcile) this.reloadChangesJournal(); + } + } + } + + /** The durable body record of a gated operator change, or null if it never + * began to apply and wasn't dropped. A host reconciles its revision with + * this after a restart. */ + getOperatorChangeRecord(changeId: string): OperatorChangeRecord | null { + const record = this.changesLedger().records.get(changeId); + return record ? structuredClone(record) : null; + } + + /** + * Retire a staged change the host won't apply (cancelled, revised or + * restaged): its staged publication choice is discarded and applying it + * is refused from now on. A change whose body began committing can't be + * dropped. + */ + dropResolvedOperatorChange(change: ResolvedOperatorChange): void { + const record = this.changesLedger().records.get(change.id); + if (record?.outcome || record?.switched !== undefined || (record?.attempts.length ?? 0) > 0) { + throw new OperatorActionError('invalid', `Operator change ${change.id} has begun to apply; it can't be dropped`); + } + if (this.hasStagedMarks(change)) this.discordAwarenessOutbox!.discard(this.changeBatchId(change.id)); + this.journalChange({ kind: 'dropped', changeId: change.id, changeKind: change.kind, agent: change.agent, at: Date.now() }); + } + /** The durable record of an unstick operation, or null if it never ran a step. */ getUnstickOperation(operationId: string): UnstickOperationRecord | null { const op = this.unstickLedger().ops.get(operationId); @@ -7330,20 +7485,12 @@ export class AgentFramework { const taken = checkpoints.slice(-requested).reverse(); // Freeze the publication choice to the refs the cut would remove now: // every message stored after the oldest staged turn began. - let marks: ResolvedUndoTurnsChange['marks'] = 'none'; - const choice = publication?.marks ?? 'none'; - if (choice !== 'none') { - const cutAt = taken[taken.length - 1]!.sequenceBefore; - const wouldRemove = this.agents.get(agentName)!.getContextManager().getAllMessages() - .filter((m) => Number(m.sequence) > cutAt) - .map((m) => ({ - metadata: { - ...((m.metadata ?? {}) as Record), - serverId: (m.metadata as { serverId?: unknown } | undefined)?.serverId ?? publication!.serverId, - }, - })); - marks = { scope: choice.scope, refs: selectDiscordAwarenessRefs(wouldRemove, choice).refs }; - } + const cutAt = taken[taken.length - 1]!.sequenceBefore; + const marks = this.freezeMarks( + this.agents.get(agentName)!.getContextManager().getAllMessages().filter((m) => Number(m.sequence) > cutAt), + publication?.marks ?? 'none', + publication?.serverId ?? '', + ); return { id: randomUUID(), agent: agentName, @@ -7366,197 +7513,651 @@ export class AgentFramework { return `undo/${change.agent}/op-${change.id}`; } + // --------------------------------------------------------------------------- + // Gated body changes: staging, application and recovery (operator/changes) + // --------------------------------------------------------------------------- + + /** The awareness batch a change's staged publication choice lives in. */ + private changeBatchId(changeId: string): string { + return `op-${changeId}`; + } + + /** Whether a change staged a publication choice: frozen refs to mark. */ + private hasStagedMarks(change: ResolvedOperatorChange): boolean { + return (change.kind === 'undo-turns' || change.kind === 'undo-messages' || change.kind === 'hide') + && change.marks !== 'none' && change.marks.refs.length > 0 && !!this.discordAwarenessOutbox; + } + + private undoMessagesDestination(change: ResolvedUndoMessagesChange): string { + return `undo-msgs/${change.agent}/op-${change.id}`; + } + /** - * Apply a resolved undo-turns change as one absolute cut. The caller holds - * the store (a lease, or a surgery reservation). Revalidates against - * stable identity: the active branch must still be the source branch, and - * every staged checkpoint must still be in the agent's list (appending a - * notice or an answer changes neither). Then it branches at the oldest - * staged checkpoint's sequenceBefore, which drops the staged turns and - * everything after them, and removes those checkpoints and any newer - * ones. One redo entry restores the source tip and all of them. If the - * active branch is already this change's destination, the cut happened - * before (a host retrying after a crash) and nothing changes. + * Stage a resolved change. A publication choice is recorded first, in the + * awareness journal: its position there is the choice's authorization for + * life, so a retract or cancel made while the change waits outranks it. + * Then the host's gate stages the change; a refusal discards the choice. */ - private applyUndoTurns(change: ResolvedUndoTurnsChange, admission?: OperatorAdmission): { - requested: number; - undone: number; - fromBranch: string; - toBranch: string; - alreadyApplied?: true; - markers?: SurgeryMarkerReceipt; - } { - const stale = (what: string): never => { - throw new OperatorActionError('stale', `Undo ${change.id} for ${change.agent} no longer holds: ${what}`); - }; - if (!this.agents.has(change.agent)) throw new OperatorActionError('unknown-agent', `Unknown agent: ${change.agent}`); - if (change.storeId !== this.storeIdentity()) stale('it was resolved against another store'); - const destination = this.undoTurnsDestination(change); - const undone = change.checkpoints.length; - const current = this.store.currentBranch().name; - if (current === destination) { - // The cut happened. Finish whatever bookkeeping an interrupted attempt - // left undone, without cutting again, and only then report it applied. - const markers = this.finishUndoTurnsCut(change, admission); + private async stageChange(change: ResolvedOperatorChange): Promise<{ + ok: boolean; + code?: string; + error?: string; + staged?: { id: string }; + }> { + if (this.hasStagedMarks(change)) { + const body = change as ResolvedUndoTurnsChange | ResolvedUndoMessagesChange | ResolvedHideChange; + const marks = body.marks as Exclude; + try { + this.discordAwarenessOutbox!.prepare({ + id: this.changeBatchId(change.id), + staged: true, + agentName: change.agent, + sourceBranch: change.sourceBranch, + targetBranch: body.kind === 'undo-turns' ? this.undoTurnsDestination(body) + : body.kind === 'undo-messages' ? this.undoMessagesDestination(body) + : body.sourceBranch, + refs: marks.refs, + scope: marks.scope, + emoji: this.discordAwarenessEmoji, + }); + } catch (error) { + return { + ok: false, + error: `Operator change not staged: its marks choice could not be recorded: ${error instanceof Error ? error.message : String(error)}`, + }; + } + } + const answer = await this.askOperatorChangeGate(change); + if (answer.outcome === 'gate-failed') { + if (this.hasStagedMarks(change)) { + try { + this.discordAwarenessOutbox!.discard(this.changeBatchId(change.id)); + } catch (error) { + console.error(`[operator-change] refused ${change.kind} ${change.id}, but its staged marks could not be discarded:`, + error instanceof Error ? error.message : error); + } + } + return { ok: false, code: 'gate-failed', error: `Operator change refused: ${answer.error}` }; + } + return { ok: false, code: 'staged', error: answer.receipt.text, staged: { id: answer.receipt.id } }; + } + + /** Resolve and stage a host/command change, reporting a resolution error. */ + private async stageOperatorCommand(resolve: () => ResolvedOperatorChange): Promise<{ + ok: boolean; + code?: string; + error?: string; + staged?: { id: string }; + }> { + let change: ResolvedOperatorChange; + try { + change = resolve(); + } catch (error) { return { - requested: change.requestedTurns, undone, fromBranch: change.sourceBranch, toBranch: destination, alreadyApplied: true, - ...(markers ? { markers } : {}), + ok: false, + error: error instanceof Error ? error.message : String(error), + ...(error instanceof OperatorActionError ? { code: error.code } : {}), }; } - // The destination exists if an earlier attempt got as far as creating - // it. Never written to (its head still at its branch point), that - // attempt was interrupted before switching, and this one finishes it. - // Written to, the cut was applied and the active branch has left it - // since (a redo, a later undo): refused, never re-applied. - const dest = this.store.listBranches().find((b) => b.name === destination); - const created = dest !== undefined; - if (dest && dest.head !== (dest.branchPoint ?? dest.head)) { - stale(`its cut was already applied as ${destination}, and the active branch has moved since to ${current}`); + return this.stageChange(change); + } + + /** A message's addressing metadata, with the command's server where the + * message names none (as the ungated surgeries route them). */ + private markCarrier(message: { metadata?: unknown }, serverId: string): { metadata: Record } { + const metadata = (message.metadata ?? {}) as Record; + return { metadata: { ...metadata, serverId: metadata.serverId ?? serverId } }; + } + + /** Freeze a publication choice to the refs among `messages`. */ + private freezeMarks( + messages: Array<{ metadata?: unknown }>, + choice: 'none' | { scope: 'addressed' | 'all' }, + serverId: string, + ): FrozenMarks { + if (choice === 'none') return 'none'; + return { scope: choice.scope, refs: selectDiscordAwarenessRefs(messages.map((m) => this.markCarrier(m, serverId)), choice).refs }; + } + + /** The marks facts of a removal: the frozen refs it removes, and what it + * leaves unmarked or doesn't remove. */ + private removalMarks( + change: { marks: FrozenMarks; serverId?: string }, + removed: Array<{ metadata?: unknown }>, + ): OperatorChangeEvidence['marks'] { + const selection = selectDiscordAwarenessRefs(removed.map((m) => this.markCarrier(m, change.serverId ?? '')), change.marks); + return { scope: marksScope(change.marks), refs: selection.refs, unmarked: selection.unmarked, notRemoved: selection.notRemoved }; + } + + /** Resolve a host/command hide: exactly the named range as it stands. */ + private resolveHide( + agentName: string, + fromMessageId: string, + toMessageId: string | undefined, + requester: OperatorRequester, + publication: { marks: 'none' | { scope: 'addressed' | 'all' }; serverId: string }, + ): ResolvedHideChange { + const agent = this.agents.get(agentName); + if (!agent) throw new OperatorActionError('unknown-agent', `Unknown agent: ${agentName}`); + const all = agent.getContextManager().getAllMessages(); + const byDiscordId = (did: string) => + all.findIndex((m) => String((m.metadata as { messageId?: unknown } | undefined)?.messageId) === String(did)); + const fromIdx = byDiscordId(fromMessageId); + if (fromIdx < 0) { + throw new OperatorActionError( + 'unknown-message', + `Message ${fromMessageId} is not an addressable message in context (it may only exist inside backscroll text, or predates this session).`, + ); } - if (current !== change.sourceBranch) stale(`the active branch moved from ${change.sourceBranch} to ${current}`); - const list = this.getTurnCheckpoints(change.agent); - const at = (c: ResolvedUndoTurnsChange['checkpoints'][number]) => list.findIndex((k) => - k.turnIndex === c.turnIndex && k.sequenceBefore === c.sequenceBefore && k.branchName === c.branchName); - for (const c of change.checkpoints) { - if (at(c) < 0) stale(`turn ${c.turnIndex} is no longer an undoable checkpoint`); + let [lo, hi] = [fromIdx, fromIdx]; + if (toMessageId) { + const toIdx = byDiscordId(toMessageId); + if (toIdx < 0) throw new OperatorActionError('unknown-message', `Message ${toMessageId} is not an addressable message in context.`); + [lo, hi] = fromIdx <= toIdx ? [fromIdx, toIdx] : [toIdx, fromIdx]; } - const oldest = change.checkpoints[change.checkpoints.length - 1]!; + const range = all.slice(lo, hi + 1); + return { + id: randomUUID(), + agent: agentName, + surface: 'host-command', + requester, + resolvedAt: Date.now(), + sourceBranch: this.store.currentBranch().name, + storeId: this.storeIdentity(), + kind: 'hide', + fromMessageId, + ...(toMessageId ? { toMessageId } : {}), + messages: range.map((m) => ({ id: String(m.id), fingerprint: this.messageFingerprint(m) })), + marks: this.freezeMarks(range, publication.marks, publication.serverId), + serverId: publication.serverId, + }; + } - // Checkpoint lists are branch-scoped state, and nothing is written on the - // source: the destination, cut before the oldest staged turn recorded - // itself, keeps only the older checkpoints, and the source tip keeps - // every checkpoint for redo to return to. - // What the cut removes from this agent's context is whatever is in it - // now and not after: addressing metadata only, read within the same - // reservation that holds the switch. - const before = this.addressingSnapshot(change.agent); - if (!created) this.store.createBranchAt(destination, change.sourceBranch, oldest.sequenceBefore); - this.store.switchBranch(destination); - const after = new Set(this.addressingSnapshot(change.agent).keys()); - const removed = [...before].filter(([id]) => !after.has(id)).map(([, carrier]) => carrier); - const markers = this.finishUndoTurnsCut(change, admission, removed); + /** Resolve a host/command undo by messages: the tail it lands on, as it stands. */ + private resolveUndoMessages( + agentName: string, + count: number, + requester: OperatorRequester, + publication: { marks: 'none' | { scope: 'addressed' | 'all' }; serverId: string }, + ): ResolvedUndoMessagesChange { + const agent = this.agents.get(agentName); + if (!agent) throw new OperatorActionError('unknown-agent', `Unknown agent: ${agentName}`); + const cm = agent.getContextManager(); + const total = cm.getMessageCount(); + if (count >= total) { + throw new OperatorActionError('invalid', `Cannot remove ${count} message(s) — history has ${total}; at least one must remain.`); + } + const requestedTail = cm.getMessageWindow(total - 1 - count, 1, { resolveBlobs: false }).messages[0]!; + const { tailMessageId, messagesRemoved, discarded } = this.planRollback(agentName, String(requestedTail.id)); + const tail = cm.getAllMessages().find((m) => String(m.id) === String(tailMessageId))!; return { - requested: change.requestedTurns, undone, fromBranch: change.sourceBranch, toBranch: destination, - ...(markers ? { markers } : {}), + id: randomUUID(), + agent: agentName, + surface: 'host-command', + requester, + resolvedAt: Date.now(), + sourceBranch: this.store.currentBranch().name, + storeId: this.storeIdentity(), + kind: 'undo-messages', + requestedMessages: count, + tail: { id: String(tailMessageId), fingerprint: this.messageFingerprint(tail) }, + messagesAfter: messagesRemoved, + marks: this.freezeMarks(discarded, publication.marks, publication.serverId), + serverId: publication.serverId, }; } + /** Record a gated change's operator-log entry once: a retry after a + * failure past the log (its `completed` entry) finds it and adds none. + * Matched by kind too: staging records carry the change id as well. */ + private recordChangeAction(entry: OperatorLogInput & { params: { changeId: string } }): void { + const changeId = entry.params.changeId; + if (this.getOperatorLog().some((e) => + e.kind === entry.kind && (e.params as { changeId?: unknown } | undefined)?.changeId === changeId)) return; + this.recordOperatorAction(entry); + } + + /** A cut's destination exists and was written past its branch point: the + * cut was applied there, whatever is active now. */ + private destinationUsed(destination: string): boolean { + const dest = this.store.listBranches().find((b) => b.name === destination); + return !!dest && dest.head !== (dest.branchPoint ?? dest.head); + } + /** - * The bookkeeping after an admitted undo cut, once the destination is - * active. Every step is idempotent, so a retry after any failure finishes - * exactly what is missing and none twice: - * - the marker naming this change on the destination, which moves its head - * off its branch point (how applyUndoTurns tells a used cut); - * - the destination's checkpoints: only those older than the cut; - * - the one redo entry back to the source tip; - * - the trace and the operator-log record, once per process. + * Establish a committed body change's outcome from its attempt's evidence: + * activate its staged publication choice with exactly what the body + * removed (under the staging-time authorization, so a key retracted since + * is born superseded), or report none; then record the outcome, durable + * and immutable. An activation the journal couldn't record leaves the + * outcome unrecorded and throws, so a retry settles it; the body stands. */ - private finishUndoTurnsCut( - change: ResolvedUndoTurnsChange, - admission?: OperatorAdmission, - removed?: Array<{ metadata: Record }>, - ): SurgeryMarkerReceipt | undefined { - const destination = this.undoTurnsDestination(change); - const oldest = change.checkpoints[change.checkpoints.length - 1]!; - const marker = (): { changeId?: unknown; completed?: unknown; marked?: unknown; markers?: SurgeryMarkerReceipt } | undefined => { - const framework = this.store.getStateJson(FRAMEWORK_STATE_ID); - return (framework && typeof framework === 'object' ? framework as Record : {}).operatorCut as - { changeId?: unknown; completed?: unknown; marked?: unknown; markers?: SurgeryMarkerReceipt } | undefined; - }; - const writeMarker = (state: { markers?: SurgeryMarkerReceipt }) => { - const framework = this.store.getStateJson(FRAMEWORK_STATE_ID); - this.store.setStateJson(FRAMEWORK_STATE_ID, { - ...(framework && typeof framework === 'object' ? framework as Record : {}), - operatorCut: { changeId: change.id, at: Date.now(), ...(state.markers ? { marked: true, markers: state.markers } : {}) }, + private establishOutcome( + change: { id: string; agent: string; kind: string }, + attempt: { n: number; evidence: OperatorChangeEvidence }, + verb: 'undo' | 'hide', + ): OperatorChangeOutcome { + const { marks } = attempt.evidence; + const facts = { scope: marks.scope as SurgeryMarkerReceipt['scope'], unmarked: marks.unmarked, notRemoved: marks.notRemoved }; + let markers: SurgeryMarkerReceipt = { ...facts, status: 'none', queued: 0 }; + if (attempt.evidence.staged && this.discordAwarenessOutbox) { + const batchId = this.changeBatchId(change.id); + const settled = this.discordAwarenessOutbox.activateStaged(batchId, { + targetBranch: attempt.evidence.target, + refs: marks.refs, + unmarked: marks.unmarked, + notRemoved: marks.notRemoved, }); - }; - // A completed cut is left exactly as it is, whatever happened since on - // its branch (later turns and their checkpoints, a redo stack): only - // its durability is confirmed again, and its marks receipt reported. - if (marker()?.changeId === change.id && marker()?.completed === true) { - this.store.sync(); - return marker()?.markers; - } - if (marker()?.changeId !== change.id) writeMarker({}); - // Marks, once: what the cut actually removed, limited to the refs frozen - // at staging. A repair that has no snapshot left treats a frozen ref as - // removed when its message is gone from the destination (later - // arrivals' counts are then unknown, and reported as none). - // Scheduled once; every later answer reports the receipt recorded then. - let markers = marker()?.markers; - if (marker()?.marked !== true) { - let carriers = removed; - if (!carriers) { - const present = new Set( - [...this.addressingSnapshot(change.agent).values()].map((c) => `${c.metadata.channelId}|${c.metadata.messageId}`), + if (settled.status === 'unresolved') { + throw new OperatorActionError( + 'failed', + `${change.kind} ${change.id} applied, but its marks activation could not be recorded (${settled.error}); retry to settle it`, ); - carriers = (change.marks === 'none' ? [] : change.marks.refs) - .filter((ref) => !present.has(`${ref.channelId}|${ref.messageId}`)) - .map((ref) => ({ metadata: { ...ref } })); - } - markers = this.scheduleAppliedMarks('undo', change.agent, change.serverId ?? '', carriers, change.marks); - writeMarker({ markers }); - } - // Checkpoints need no bookkeeping: the lists are branch-scoped, and the - // destination, cut where the oldest staged turn had yet to record - // itself, inherits exactly the older ones. Later turns add their own. - this.materializeConfigMountAfterBranchSwitch(); - let redoStack = this.redoStacks.get(change.agent); - if (!redoStack) { - redoStack = []; - this.redoStacks.set(change.agent, redoStack); - } - // The source tip still holds the removed checkpoints: redo re-adds none. - // A turn that already ran on the destination (it recorded a checkpoint - // there) invalidated redo, as any new work does, so a repair pushes no - // entry then: redoing would discard that work. - const laterWork = this.getTurnCheckpoints(change.agent).some((c) => c.branchName === destination); - if (!laterWork && !redoStack.some((entry) => entry.changeId === change.id)) { - redoStack.push({ branchName: change.sourceBranch, checkpoints: [], changeId: change.id }); + } + if (settled.status === 'not-scheduled') { + markers = { ...facts, status: 'not-scheduled', queued: 0, error: settled.error }; + } else if (marks.refs.length > 0) { + markers = { ...facts, status: 'queued', queued: settled.queued, batchId }; + if (settled.queued > 0) this.deliverDiscordAwarenessInBackground(verb, change.agent); + } } - if (this.completedOperatorCuts.has(change.id)) { - this.completeUndoTurnsCut(change, markers); - return markers; + this.journalChange({ + kind: 'outcome', + changeId: change.id, + outcome: { n: attempt.n, at: Date.now(), removed: attempt.evidence.removed, markers }, + }); + return this.changesLedger().records.get(change.id)!.outcome!; + } + + /** + * Apply a staged branch cut under the held lease, or recover it. + * - An established outcome is reported as it was recorded. + * - The latest attempt committed if its switch was recorded, its + * destination is active, or its destination was written to since (being + * back on the source proves nothing): its outcome is established from + * its own evidence, and it is never cut again. + * - Otherwise this is a fresh attempt: revalidated on the source, its + * evidence recorded before the cut, its switch right after. + */ + private async applyCut( + change: ResolvedUndoTurnsChange | ResolvedUndoMessagesChange, + destination: string, + steps: { + revalidate: (stale: (why: string) => never) => void; + removal: () => Array<{ metadata?: unknown }>; + cut: () => Promise; + finish: (outcome: OperatorChangeOutcome) => void; + }, + ): Promise<{ outcome: OperatorChangeOutcome; alreadyApplied: boolean }> { + const label = change.kind === 'undo-turns' + ? `Undo of ${change.requestedTurns} turn(s)` + : `Undo of ${change.requestedMessages} message(s)`; + const stale = (why: string): never => { + throw new OperatorActionError('stale', `${label} ${change.id} for ${change.agent} no longer applies: ${why}`); + }; + if (!this.agents.has(change.agent)) throw new OperatorActionError('unknown-agent', `Unknown agent: ${change.agent}`); + if (change.storeId !== this.storeIdentity()) stale('it was resolved against another store'); + const record = () => this.changesLedger().records.get(change.id); + const settle = (outcome: OperatorChangeOutcome, alreadyApplied: boolean) => { + if (!record()?.completed) { + steps.finish(outcome); + this.journalChange({ kind: 'completed', changeId: change.id }); + } + return { outcome, alreadyApplied }; + }; + const known = record(); + if (known?.outcome) return settle(known.outcome, true); + const current = this.store.currentBranch().name; + const last = known?.attempts[known.attempts.length - 1]; + if (last) { + if (known!.switched === last.n || current === destination || this.destinationUsed(destination)) { + if (known!.switched !== last.n) this.journalChange({ kind: 'switched', changeId: change.id, n: last.n }); + return settle(this.establishOutcome(change, last, 'undo'), true); + } + } else if (current === destination || this.destinationUsed(destination)) { + throw new OperatorActionError( + 'failed', + `${label} ${change.id}: its destination ${destination} exists but no attempt was recorded, so its outcome is unknown`, + ); } - this.emitTrace({ - type: 'undo:completed', - agentName: change.agent, - turnIndex: oldest.turnIndex, + if (current !== change.sourceBranch) stale(`the active branch moved from ${change.sourceBranch} to ${current}`); + steps.revalidate(stale); + const removed = steps.removal(); + const n = (last?.n ?? 0) + 1; + this.journalChange({ + kind: 'attempt', + changeId: change.id, + changeKind: change.kind, + agent: change.agent, + n, + at: Date.now(), + evidence: { + target: destination, + sourceHead: this.store.currentBranch().head, + removed: removed.length, + marks: this.removalMarks(change, removed), + staged: this.hasStagedMarks(change), + ...(change.requester ? { requester: change.requester } : {}), + }, + }); + await steps.cut(); + this.journalChange({ kind: 'switched', changeId: change.id, n }); + return settle(this.establishOutcome(change, record()!.attempts.find((a) => a.n === n)!, 'undo'), false); + } + + /** + * Apply an admitted multi-turn undo as one cut: a branch at the oldest + * staged turn's sequenceBefore, named for the change, switched to under + * the held lease. The source tip keeps every checkpoint for the one redo + * entry back to it; the destination inherits only the older ones. + */ + private async applyUndoTurns(change: ResolvedUndoTurnsChange, admission?: OperatorAdmission): Promise<{ + requested: number; + undone: number; + fromBranch: string; + toBranch: string; + alreadyApplied?: true; + markers: SurgeryMarkerReceipt; + }> { + const destination = this.undoTurnsDestination(change); + const oldest = change.checkpoints[change.checkpoints.length - 1]!; + const { outcome, alreadyApplied } = await this.applyCut(change, destination, { + revalidate: (stale) => { + const list = this.getTurnCheckpoints(change.agent); + for (const c of change.checkpoints) { + if (!list.some((k) => k.turnIndex === c.turnIndex && k.sequenceBefore === c.sequenceBefore && k.branchName === c.branchName)) { + stale(`turn ${c.turnIndex} is no longer an undoable checkpoint`); + } + } + }, + // Everything stored after the oldest staged turn began: exactly what + // the cut removes, read within the lease that holds the switch. + removal: () => this.agents.get(change.agent)!.getContextManager().getAllMessages() + .filter((m) => Number(m.sequence) > oldest.sequenceBefore), + cut: async () => { + if (!this.store.listBranches().some((b) => b.name === destination)) { + this.store.createBranchAt(destination, change.sourceBranch, oldest.sequenceBefore); + } + this.store.switchBranch(destination); + this.materializeConfigMountAfterBranchSwitch(); + }, + finish: (established) => { + // The one redo entry back to the source tip, while the destination is + // active and no turn has run on it (new work invalidates redo). + if (this.store.currentBranch().name === destination + && !this.getTurnCheckpoints(change.agent).some((c) => c.branchName === destination)) { + const redoStack = this.redoStacks.get(change.agent) ?? []; + this.redoStacks.set(change.agent, redoStack); + if (!redoStack.some((entry) => entry.changeId === change.id)) { + redoStack.push({ branchName: change.sourceBranch, checkpoints: [], changeId: change.id }); + } + } + this.emitTrace({ + type: 'undo:completed', + agentName: change.agent, + turnIndex: oldest.turnIndex, + fromBranch: change.sourceBranch, + toBranch: destination, + turns: change.checkpoints.length, + }); + this.recordChangeAction({ + kind: 'undo-turns', + agent: change.agent, + ...(change.requester ? { requester: change.requester } : {}), + params: { + changeId: change.id, + requested: change.requestedTurns, + resolved: change.checkpoints.length, + turnIndexes: change.checkpoints.map((c) => c.turnIndex), + marks: describeMarksChoice(change.marks), + ...(admission ? { admission: admission.id } : {}), + }, + result: { sourceBranch: change.sourceBranch, targetBranch: destination, removed: established.removed, markers: established.markers }, + }); + }, + }); + return { + requested: change.requestedTurns, + undone: change.checkpoints.length, fromBranch: change.sourceBranch, toBranch: destination, - turns: change.checkpoints.length, + ...(alreadyApplied ? { alreadyApplied: true as const } : {}), + markers: outcome.markers, + }; + } + + /** + * Apply a staged undo by messages under the held lease: the rollback cut + * at its tail, onto the branch named for the change (an existing one an + * interrupted attempt created is reused), through the same branching the + * ungated rollback uses. + */ + private async applyUndoMessages(change: ResolvedUndoMessagesChange, admission?: OperatorAdmission): Promise<{ + requested: number; + messagesRemoved: number; + fromBranch: string; + toBranch: string; + alreadyApplied?: true; + markers: SurgeryMarkerReceipt; + }> { + const destination = this.undoMessagesDestination(change); + const agent = this.agents.get(change.agent); + if (!agent) throw new OperatorActionError('unknown-agent', `Unknown agent: ${change.agent}`); + const cm = agent.getContextManager(); + const { outcome, alreadyApplied } = await this.applyCut(change, destination, { + revalidate: (stale) => { + const tail = cm.getAllMessages().find((m) => String(m.id) === change.tail.id); + if (!tail) stale(`its tail message ${change.tail.id} is gone`); + if (this.messageFingerprint(tail!) !== change.tail.fingerprint) stale(`its tail message ${change.tail.id} changed since it was staged`); + }, + removal: () => this.planRollback(change.agent, change.tail.id).discarded, + cut: async () => { + try { + const exists = this.store.listBranches().some((b) => b.name === destination); + await cm.switchBranch(exists ? destination : cm.branchAt(change.tail.id as MessageId, destination)); + } catch (error) { + const restored = await this.restoreSourceBranch(cm, change.sourceBranch, destination, change.agent); + throw new OperatorActionError( + 'failed', + `Undo of ${change.requestedMessages} message(s) ${change.id} failed; ${restored}: ${error instanceof Error ? error.message : String(error)}`, + { cause: error }, + ); + } + this.materializeConfigMountAfterBranchSwitch(); + }, + finish: (established) => { + this.recordChangeAction({ + kind: 'rollback', + agent: change.agent, + ...(change.requester ? { requester: change.requester } : {}), + params: { + changeId: change.id, + messageId: change.tail.id, + requested: change.requestedMessages, + marks: describeMarksChoice(change.marks), + ...(admission ? { admission: admission.id } : {}), + }, + result: { sourceBranch: change.sourceBranch, targetBranch: destination, messagesRemoved: established.removed, markers: established.markers }, + }); + }, }); - this.recordOperatorAction({ - kind: 'undo-turns', + return { + requested: change.requestedMessages, + messagesRemoved: outcome.removed, + fromBranch: change.sourceBranch, + toBranch: destination, + ...(alreadyApplied ? { alreadyApplied: true as const } : {}), + markers: outcome.markers, + }; + } + + /** + * Apply a staged hide under the held lease, or recover it. A fresh attempt + * records its exact present messages (each checked unchanged) before + * removing them. An interrupted attempt is finished, never re-planned: + * exactly its own messages, so what it removed before the interruption + * still counts in its outcome. If the active branch left the source + * meanwhile, its outcome is unknown and it is refused, never resumed + * elsewhere. + */ + private applyHide(change: ResolvedHideChange, admission?: OperatorAdmission): { + hidden: number; + alreadyApplied?: true; + markers: SurgeryMarkerReceipt; + } { + const stale = (why: string): never => { + throw new OperatorActionError('stale', `Hide ${change.id} for ${change.agent} no longer applies: ${why}`); + }; + const agent = this.agents.get(change.agent); + if (!agent) throw new OperatorActionError('unknown-agent', `Unknown agent: ${change.agent}`); + if (change.storeId !== this.storeIdentity()) stale('it was resolved against another store'); + const record = () => this.changesLedger().records.get(change.id); + const settle = (outcome: OperatorChangeOutcome, alreadyApplied: boolean) => { + if (!record()?.completed) { + const ids = capIds((record()!.attempts.find((a) => a.n === outcome.n)?.evidence.ids ?? []).map((p) => p.id)); + this.recordChangeAction({ + kind: 'hide', + agent: change.agent, + ...(change.requester ? { requester: change.requester } : {}), + params: { + changeId: change.id, + fromMessageId: change.fromMessageId, + ...(change.toMessageId ? { toMessageId: change.toMessageId } : {}), + marks: describeMarksChoice(change.marks), + ...(admission ? { admission: admission.id } : {}), + }, + result: { + branch: change.sourceBranch, + hidden: outcome.removed, + removedIds: ids.ids, + ...(ids.truncated ? { removedIdsTruncated: true } : {}), + markers: outcome.markers, + }, + }); + this.journalChange({ kind: 'completed', changeId: change.id }); + } + return { hidden: outcome.removed, ...(alreadyApplied ? { alreadyApplied: true as const } : {}), markers: outcome.markers }; + }; + const known = record(); + if (known?.outcome) return settle(known.outcome, true); + const current = this.store.currentBranch().name; + const cm = agent.getContextManager(); + const last = known?.attempts[known.attempts.length - 1]; + if (last) { + if (current !== change.sourceBranch) { + stale(`the active branch moved from ${change.sourceBranch} to ${current} during an interrupted hide, so its outcome is unknown`); + } + const removedNow = this.removeHideMessages(cm, last.evidence.ids ?? [], stale); + return settle(this.establishOutcome(change, last, 'hide'), removedNow === 0); + } + if (current !== change.sourceBranch) stale(`the active branch moved from ${change.sourceBranch} to ${current}`); + const byId = new Map(cm.getAllMessages().map((m) => [String(m.id), m])); + const present = change.messages.filter((p) => byId.has(p.id)); + for (const p of present) { + if (this.messageFingerprint(byId.get(p.id)!) !== p.fingerprint) stale(`message ${p.id} changed since it was staged`); + } + this.journalChange({ + kind: 'attempt', + changeId: change.id, + changeKind: 'hide', agent: change.agent, - ...(change.requester ? { requester: change.requester } : {}), - params: { - changeId: change.id, - requested: change.requestedTurns, - resolved: change.checkpoints.length, - turnIndexes: change.checkpoints.map((c) => c.turnIndex), - ...(admission ? { admission: admission.id } : {}), + n: 1, + at: Date.now(), + evidence: { + target: change.sourceBranch, + sourceHead: this.store.currentBranch().head, + ids: present.map((p) => ({ id: p.id, fingerprint: p.fingerprint })), + removed: present.length, + marks: this.removalMarks(change, present.map((p) => byId.get(p.id)!)), + staged: this.hasStagedMarks(change), + ...(change.requester ? { requester: change.requester } : {}), }, - result: { sourceBranch: change.sourceBranch, targetBranch: destination }, - }); - this.completedOperatorCuts.add(change.id); - this.completeUndoTurnsCut(change, markers); - return markers; - } - - /** Mark an admitted undo cut complete on its destination and make that - * durable: branch state persists only on sync(), so completion is - * reported only after the sync succeeds (a failure throws, and a retry - * confirms it again). */ - private completeUndoTurnsCut(change: ResolvedUndoTurnsChange, markers?: SurgeryMarkerReceipt): void { - const framework = this.store.getStateJson(FRAMEWORK_STATE_ID); - this.store.setStateJson(FRAMEWORK_STATE_ID, { - ...(framework && typeof framework === 'object' ? framework as Record : {}), - operatorCut: { changeId: change.id, at: Date.now(), marked: true, ...(markers ? { markers } : {}), completed: true }, }); - this.store.sync(); + this.removeHideMessages(cm, present, stale); + return settle(this.establishOutcome(change, record()!.attempts[0]!, 'hide'), present.length === 0); + } + + /** Remove those of `ids` that remain, each only while unchanged: one range + * removal when they are contiguous, else one by one. Returns how many. */ + private removeHideMessages( + cm: ContextManager, + ids: Array<{ id: string; fingerprint: string }>, + stale: (why: string) => never, + ): number { + const all = cm.getAllMessages(); + const byId = new Map(all.map((m) => [String(m.id), m])); + const present = ids.filter((p) => byId.has(p.id)); + for (const p of present) { + if (this.messageFingerprint(byId.get(p.id)!) !== p.fingerprint) stale(`message ${p.id} changed since it was staged`); + } + if (present.length === 0) return 0; + const order = all.map((m) => String(m.id)); + const first = order.indexOf(present[0]!.id); + const lastIndex = order.indexOf(present[present.length - 1]!.id); + if (present.length > 1 && lastIndex - first + 1 === present.length) { + cm.removeMessages(present[0]!.id as MessageId, present[present.length - 1]!.id as MessageId); + } else { + for (const p of present) cm.removeMessage(p.id as MessageId); + } + return present.length; } + /** + * At startup, establish the outcome of every gated body change that + * committed before a crash recorded it: a cut whose switch was recorded or + * whose destination is active or used, and an interrupted hide, finished + * on its own branch. This runs before modules, connections or traffic, so + * a committed body never waits on its host to settle its marks. Anything + * not provably committed is left for the host's retry. + */ + private async reconcileOperatorChanges(): Promise { + let records: OperatorChangeRecord[]; + try { + records = [...this.changesLedger().records.values()]; + } catch (error) { + console.error('[operator-changes] journal unreadable at startup:', error instanceof Error ? error.message : error); + return; + } + for (const record of records) { + if (record.outcome || record.dropped || record.attempts.length === 0) continue; + const last = record.attempts[record.attempts.length - 1]!; + try { + const current = this.store.currentBranch().name; + if (record.kind === 'hide') { + const agent = this.agents.get(record.agent); + if (!agent || current !== last.evidence.target) continue; + this.removeHideMessages(agent.getContextManager(), last.evidence.ids ?? [], (why) => { + throw new OperatorActionError('stale', why); + }); + } else { + const destination = last.evidence.target; + if (!(record.switched === last.n || current === destination || this.destinationUsed(destination))) continue; + if (record.switched !== last.n) this.journalChange({ kind: 'switched', changeId: record.changeId, n: last.n }); + } + const outcome = this.establishOutcome( + { id: record.changeId, agent: record.agent, kind: record.kind }, + last, + record.kind === 'hide' ? 'hide' : 'undo', + ); + this.recordChangeAction({ + kind: record.kind === 'undo-messages' ? 'rollback' : record.kind, + agent: record.agent, + ...(last.evidence.requester ? { requester: last.evidence.requester } : {}), + params: { changeId: record.changeId, recovered: 'startup' }, + result: { target: last.evidence.target, removed: outcome.removed, markers: outcome.markers }, + }); + this.journalChange({ kind: 'completed', changeId: record.changeId }); + console.error(`[operator-changes] ${record.kind} ${record.changeId} for ${record.agent}: committed before a crash recorded it; outcome established at startup`); + } catch (error) { + console.error( + `[operator-changes] ${record.kind} ${record.changeId} for ${record.agent}: not recovered at startup (left for the host's retry):`, + error instanceof Error ? error.message : error, + ); + } + } + } + + /** * Apply a staged change at a safe boundary, under the host's held lease. * Revalidates first and refuses with OperatorActionError('stale') when the @@ -7572,6 +8173,9 @@ export class AgentFramework { opts: { lease: SafeBoundaryLease; admission: OperatorAdmission; step?: number }, ): Promise { this.heldLeaseTokens(opts.lease, `apply ${change.kind} for ${change.agent}`); + if (this.changesLedger().records.get(change.id)?.dropped) { + throw new OperatorActionError('stale', `Operator change ${change.id} was dropped; it no longer applies`); + } if (change.kind === 'unstick') { if (opts.step === undefined) { throw new OperatorActionError('invalid', `Applying unstick ${change.id} takes a step (1 to ${change.plan.length})`); @@ -7581,9 +8185,11 @@ export class AgentFramework { if (change.kind === 'undo-turns') { // Marks are scheduled within the cut's bookkeeping, under the lease; // their delivery runs in the background and is never this undo's. - const applied = this.applyUndoTurns(change, opts.admission); + const applied = await this.applyUndoTurns(change, opts.admission); return { kind: 'undo-turns', ...applied }; } + if (change.kind === 'hide') return { kind: 'hide', ...this.applyHide(change, opts.admission) }; + if (change.kind === 'undo-messages') return { kind: 'undo-messages', ...(await this.applyUndoMessages(change, opts.admission)) }; this.revalidateSelfChange(change); const tool = change.kind === 'agent-settings' ? 'agent_settings' : change.tool; this.activeAdmissions.set(opts.admission.id, change); @@ -7695,6 +8301,13 @@ export class AgentFramework { branchName?: string; /** Publication choice for awareness marks; default `none`. */ marks?: DiscordAwarenessMarks; + /** Run under this held lease (runAtSafeBoundary) instead of reserving + * the store, as a host applying a staged change does. */ + lease?: SafeBoundaryLease; + /** The context it was chosen in (a preview's `context`): refused as + * stale, before any change, if the store or branch differs under the + * reservation. */ + expected?: { storeId?: string; branch?: string }; }, ): Promise<{ agentName: string; @@ -7736,10 +8349,11 @@ export class AgentFramework { // until the switch has landed and its marker scheduling has been // recorded or reported, or the switch has been rolled back. Remote // delivery is not waited on. - const release = this.reserveStoreForSurgery('roll back', agentName); + const release = this.surgeryReservation('roll back', agentName, opts.lease); let branchName: string; let markers: SurgeryMarkerReceipt; try { + this.checkExpectedContext(opts.expected, `Rollback of ${agentName} to ${opts.messageId}`); // Prepare the external side effect before switching Chronicle. If the // process dies after the switch but before activate(), startup // promotes this batch by matching targetBranch to the active branch. @@ -7846,6 +8460,13 @@ export class AgentFramework { branchName?: string; /** Publication choice for awareness marks; default `none`. */ marks?: DiscordAwarenessMarks; + /** Run under this held lease (runAtSafeBoundary) instead of reserving + * the store, as a host applying a staged change does. */ + lease?: SafeBoundaryLease; + /** The context it was chosen in (a preview's `context`): refused as + * stale, before any change, if the store or branch differs under the + * reservation. */ + expected?: { storeId?: string; branch?: string }; }, ): Promise<{ agentName: string; @@ -7884,10 +8505,11 @@ export class AgentFramework { // until the fork is fully redacted and its marker scheduling has been // recorded or reported, or the source is restored. Remote delivery is // not waited on. - const release = this.reserveStoreForSurgery('suppress', agentName); + const release = this.surgeryReservation('suppress', agentName, opts.lease); let createdBranch: string; let markers: SurgeryMarkerReceipt; try { + this.checkExpectedContext(opts.expected, `Suppression for ${agentName}`); const markerBatch = this.discordAwarenessOutbox?.prepare({ agentName, sourceBranch, @@ -8079,6 +8701,7 @@ export class AgentFramework { }; }; return { + context: { storeId: this.storeIdentity(), branch: this.store.currentBranch().name }, messagesRemoved: messages.length, addressable: extractDiscordAwarenessRefs(messages).length, emoji: this.discordAwarenessEmoji, diff --git a/src/operator-change.ts b/src/operator-change.ts index 5adb52860..178267aee 100644 --- a/src/operator-change.ts +++ b/src/operator-change.ts @@ -77,12 +77,11 @@ export interface ResolvedUndoTurnsChange extends ResolvedOperatorChangeBase { /** The checkpoints it undoes, newest first. Application makes one cut at * the oldest one's sequenceBefore, onto a branch named for this change. */ checkpoints: Array<{ turnIndex: number; sequenceBefore: number; branchName: string }>; - /** The operator's awareness-marks choice, frozen at staging: 'none', or a - * scope with exactly the refs the cut would have removed then. The cut - * can also reach messages that arrive later; application marks only the - * frozen refs it actually removed, reports later removals as unmarked, - * and never widens the set. */ - marks: 'none' | { scope: 'addressed' | 'all'; refs: Array<{ serverId: string; channelId: string; messageId: string }> }; + /** The operator's awareness-marks choice, frozen at staging to the refs + * the cut would have removed then. The cut can also reach messages that + * arrive later; application marks only the frozen refs it actually + * removed, reports later removals as unmarked, and never widens the set. */ + marks: FrozenMarks; /** The MCPL server the command came from: marks whose message names no * server are routed through it. */ serverId?: string; @@ -108,11 +107,101 @@ export interface ResolvedUnstickChange extends ResolvedOperatorChangeBase { plan: Array<{ step: number; messageIds: string[]; fingerprints: string[] }>; } +/** An operator's awareness-marks choice frozen at staging: 'none', or a + * scope with exactly the refs the change would have removed then. With + * refs, staging records the choice in the awareness journal (a staged + * batch), whose position is its authorization for life. */ +export type FrozenMarks = + | 'none' + | { scope: 'addressed' | 'all'; refs: Array<{ serverId: string; channelId: string; messageId: string }> }; + +/** host/command hide: a redaction in place, on the source branch. */ +export interface ResolvedHideChange extends ResolvedOperatorChangeBase { + kind: 'hide'; + /** The Discord message ids the operator named. */ + fromMessageId: string; + toMessageId?: string; + /** The messages it hides, oldest first: exactly these, each only while + * its content is unchanged. */ + messages: Array<{ id: string; fingerprint: string }>; + /** Frozen to the refs among those messages. */ + marks: FrozenMarks; + /** The MCPL server the command came from (marks without a server route there). */ + serverId?: string; +} + +/** host/command undo by messages: a branch cut at the message that becomes + * the tail. Like undo-turns, the cut also takes messages that arrive after + * staging; marks cover only the refs frozen here. */ +export interface ResolvedUndoMessagesChange extends ResolvedOperatorChangeBase { + kind: 'undo-messages'; + requestedMessages: number; + /** The message that becomes the tail (the end of its body group), with its + * content fingerprint. */ + tail: { id: string; fingerprint: string }; + /** How many messages followed it at staging. */ + messagesAfter: number; + /** Frozen to the refs among the messages that followed it at staging. */ + marks: FrozenMarks; + /** The MCPL server the command came from (marks without a server route there). */ + serverId?: string; +} + export type ResolvedOperatorChange = | ResolvedSettingsChange | ResolvedPresentationChange | ResolvedUndoTurnsChange - | ResolvedUnstickChange; + | ResolvedUnstickChange + | ResolvedHideChange + | ResolvedUndoMessagesChange; + +/** What one application attempt's body change will do, captured under its + * lease before the change. A committed attempt's outcome is established + * from exactly this, never by rereading a branch later. */ +export interface OperatorChangeEvidence { + /** The branch the body lands on: a cut's destination, or the hidden + * messages' own branch. */ + target: string; + /** The source branch's head when the attempt began. */ + sourceHead: number; + /** A hide's exact messages, with their fingerprints. */ + ids?: Array<{ id: string; fingerprint: string }>; + /** Messages the body change removes. */ + removed: number; + /** Its marks facts: the frozen refs it removes (`refs`), the addressable + * removals it leaves unmarked, and the frozen refs it doesn't remove. */ + marks: { scope: 'none' | 'addressed' | 'all'; refs: Array<{ serverId: string; channelId: string; messageId: string }>; unmarked: number; notRemoved: number }; + /** Whether staging recorded a publication choice to activate. */ + staged: boolean; + requester?: OperatorRequester; +} + +/** A gated body change's established outcome: recorded once, immutable. */ +export interface OperatorChangeOutcome { + /** The attempt it established. */ + n: number; + at: number; + removed: number; + /** The awareness receipt: the staged choice's activation, or none. */ + markers: SurgeryMarkerReceipt; +} + +/** The body record of one gated operator change (journal operator/changes). */ +export interface OperatorChangeRecord { + changeId: string; + kind: ResolvedOperatorChange['kind']; + agent: string; + /** Application attempts, in order, each with its evidence. */ + attempts: Array<{ n: number; at: number; evidence: OperatorChangeEvidence }>; + /** The attempt whose cut switched to its destination: recorded right + * after the switch, so it proves the cut whatever is active later. */ + switched?: number; + outcome?: OperatorChangeOutcome; + /** Its bookkeeping (the operator-log record) is done. */ + completed?: true; + /** The host dropped it before it applied; its staged choice was discarded. */ + dropped?: { at: number }; +} /** One step of an unstick operation, as journaled: `intent` before its shed * (with exactly what it will remove), `shed` once it is done. */ @@ -153,6 +242,24 @@ export type UnstickAttemptOutcome = export type AppliedOperatorChange = | { kind: 'agent-settings'; result: unknown } | { kind: 'tool-presentation'; result: unknown } + | { + kind: 'hide'; + /** Messages its established outcome removed. */ + hidden: number; + /** Established earlier: nothing was removed by this call. */ + alreadyApplied?: true; + markers: SurgeryMarkerReceipt; + } + | { + kind: 'undo-messages'; + requested: number; + /** Messages the cut removed, later arrivals included. */ + messagesRemoved: number; + fromBranch: string; + toBranch: string; + alreadyApplied?: true; + markers: SurgeryMarkerReceipt; + } | { kind: 'undo-turns'; requested: number; @@ -162,9 +269,8 @@ export type AppliedOperatorChange = /** The cut had already been applied (the active branch is its * destination): nothing changed this time. */ alreadyApplied?: true; - /** The awareness marks receipt: scheduled once, by the first - * application to get that far, and reported again by any retry. */ - markers?: SurgeryMarkerReceipt; + /** The awareness receipt of its established outcome. */ + markers: SurgeryMarkerReceipt; } | { kind: 'unstick'; diff --git a/src/operator-log.ts b/src/operator-log.ts index 1112bc546..41ab60b54 100644 --- a/src/operator-log.ts +++ b/src/operator-log.ts @@ -162,6 +162,11 @@ export type SurgeryMarkerReceipt = SurgeryMarkerFacts & ( /** What a surgery would remove and which messages each marks scope covers. */ export interface SurgeryMarksPreview { + /** The store and branch it was computed on. Pass it back as the + * surgery's `expected`: the surgery is refused, before any change, if + * either differs under its reservation (a session switch or branch move + * since the preview). */ + context: { storeId: string; branch: string }; messagesRemoved: number; /** Removed messages that carry a Discord address at all. */ addressable: number; diff --git a/test/operator-change-gate.test.ts b/test/operator-change-gate.test.ts index 7860eaa60..cbc49f19d 100644 --- a/test/operator-change-gate.test.ts +++ b/test/operator-change-gate.test.ts @@ -544,7 +544,7 @@ describe('operator-change gate: host/command undo by turns', () => { assert.deepEqual(texts(), ['one', 'reply to one', 'two']); }); - it('recognizes a cut that was already applied, and refuses one whose branch moved', async () => { + it('recognizes a cut that was already applied, and reports it as established after the branch moved on', async () => { await turn('one'); await turn('two'); await undo(1); const change = asked[0]!; @@ -559,7 +559,11 @@ describe('operator-change gate: host/command undo by turns', () => { assert.equal(framework.getOperatorLog().filter((e) => e.kind === 'undo-turns').length, 1, 'no second log record'); framework.redo('scout'); // the active branch moves away from the destination - await assert.rejects(apply(), (e: Error & { code?: string }) => e.code === 'stale' && /already applied/.test(e.message)); + const source = branch(); + const established = await apply(); + assert.equal(established.kind === 'undo-turns' && established.alreadyApplied, true, 'its established outcome, reported as recorded'); + assert.equal(branch(), source, 'and nothing cut again'); + assert.equal(framework.getOperatorLog().filter((e) => e.kind === 'undo-turns').length, 1); }); it('refuses a staged undo whose branch moved or whose checkpoint is gone', async () => { @@ -574,52 +578,72 @@ describe('operator-change gate: host/command undo by turns', () => { ); }); - for (const [seam, later] of (['marker', 'log', 'completion', 'completion sync'] as const).flatMap((x) => [[x, false], [x, true]] as const)) { - it(`finishes the cut's bookkeeping on retry after a failure at the ${seam} write${later ? ', keeping later work' : ', redo included'}`, async () => { + const failJournalOnce = (kind: string) => { + const fw = framework as unknown as { journalChange: (entry: { kind: string }) => void }; + const real = fw.journalChange.bind(fw); + let armed = true; + fw.journalChange = (entry) => { + if (armed && entry.kind === kind) { armed = false; throw new Error(`injected ${kind} failure`); } + return real(entry); + }; + return () => { fw.journalChange = real; }; + }; + const failSwitchOnce = () => { + const store = framework.getStore() as unknown as { switchBranch: (name: string) => unknown }; + const real = store.switchBranch.bind(store); + let armed = true; + store.switchBranch = (name) => { if (armed) { armed = false; throw new Error('injected cut failure'); } return real(name); }; + return () => { store.switchBranch = real; }; + }; + const failLogOnce = (kind: string) => { + const fw = framework as unknown as { recordOperatorAction: (entry: { kind: string }) => void }; + const real = fw.recordOperatorAction.bind(fw); + let armed = true; + fw.recordOperatorAction = (entry) => { if (armed && entry.kind === kind) { armed = false; throw new Error('injected log failure'); } return real(entry); }; + return () => { fw.recordOperatorAction = real; }; + }; + const applyIt = (change: ResolvedOperatorChange) => framework.runAtSafeBoundary({ verb: 'apply' }, (lease) => + framework.applyResolvedOperatorChange(change, { lease, admission: { id: 'rev-u' } })); + + for (const [seam, later] of ([ + ['attempt', false], ['cut', false], + ['switched', false], ['switched', true], + ['outcome', false], ['outcome', true], + ['log', false], ['log', true], + ['completed', false], ['completed', true], + ] as const)) { + it(`recovers from a failure at its ${seam} step${later ? ', keeping later work' : ''}`, async () => { await turn('one'); await turn('two'); await turn('three'); const source = branch(); await undo(2); const change = asked[0]!; - const fw = framework as unknown as { recordOperatorAction: (entry: { kind: string }) => void }; - const store = framework.getStore() as unknown as { setStateJson: (id: string, v: unknown) => void; sync: () => void }; - const real = { setState: store.setStateJson.bind(store), record: fw.recordOperatorAction.bind(fw), sync: store.sync.bind(store) }; - let armed = true; - const fail = () => { armed = false; throw new Error(`injected ${seam} failure`); }; - const cut = (v: unknown) => (v as { operatorCut?: { completed?: boolean } })?.operatorCut; - if (seam === 'marker') store.setStateJson = (id, v) => (armed && cut(v) && !cut(v)!.completed ? fail() : real.setState(id, v)); - if (seam === 'completion') store.setStateJson = (id, v) => (armed && cut(v)?.completed ? fail() : real.setState(id, v)); - if (seam === 'log') fw.recordOperatorAction = (entry) => (armed && entry.kind === 'undo-turns' ? fail() : real.record(entry)); - if (seam === 'completion sync') { - // The sync right after the completion marker is the barrier that - // makes completion durable: fail it once. - let marked = false; - store.setStateJson = (id, v) => { if (cut(v)?.completed) marked = true; return real.setState(id, v); }; - store.sync = () => (armed && marked ? fail() : real.sync()); - } - const apply = () => framework.runAtSafeBoundary({ verb: 'apply' }, (lease) => - framework.applyResolvedOperatorChange(change, { lease, admission: { id: 'rev-u' } })); + const destination = `undo/scout/op-${change.id}`; + const restore = seam === 'cut' ? failSwitchOnce() : seam === 'log' ? failLogOnce('undo-turns') : failJournalOnce(seam); try { - await assert.rejects(apply(), new RegExp(`injected ${seam} failure`)); + await assert.rejects(applyIt(change), new RegExp(`injected ${seam} failure`)); } finally { - store.setStateJson = real.setState; - fw.recordOperatorAction = real.record; - store.sync = real.sync; + restore(); } - assert.equal(branch(), `undo/scout/op-${change.id}`, 'the cut landed before the failure'); - if (later) await turn('after the failure'); // legitimate work on the destination before the retry - const retried = await apply(); - assert.equal(retried.kind === 'undo-turns' && retried.alreadyApplied, true, 'not cut again'); + const committed = seam !== 'attempt' && seam !== 'cut'; + assert.equal(branch() === destination, committed, committed ? 'the cut committed before the failure' : 'nothing was cut'); + if (later) await turn('after the failure'); + const retried = await applyIt(change); + assert.equal(retried.kind === 'undo-turns' && retried.alreadyApplied === true, committed, 'a committed cut is recovered, never cut again'); + assert.equal(branch(), destination); assert.equal(framework.getOperatorLog().filter((e) => e.kind === 'undo-turns').length, 1, 'logged once'); + const record = framework.getOperatorChangeRecord(change.id)!; + assert.equal(record.attempts.length, seam === 'cut' ? 2 : 1, seam === 'cut' ? 'the unapplied attempt, then a fresh one' : 'one attempt'); + assert.ok(record.outcome && record.completed); if (later) { - assert.equal(host().getTurnCheckpoints('scout').length, 2, 'the older turn and the later one: repair kept later work'); + assert.equal(host().getTurnCheckpoints('scout').length, 2, 'the older turn and the later one: later work kept'); assert.equal(texts().at(-1), 'reply to after the failure'); - assert.equal(host().redoStacks.get('scout')?.length ?? 0, 0, 'the later work invalidated redo, and repair did not restore it'); + assert.equal(host().redoStacks.get('scout')?.length ?? 0, 0, 'the later work invalidated redo'); return; } assert.equal(host().getTurnCheckpoints('scout').length, 1); - assert.equal(host().redoStacks.get('scout')!.length, 1, 'one redo entry, not one per attempt'); + assert.equal(host().redoStacks.get('scout')!.length, 1, 'one redo entry'); const redone = framework.redo('scout'); - assert.deepEqual([redone.redone, redone.toBranch], [true, source], 'the promised one-step redo exists'); + assert.deepEqual([redone.redone, redone.toBranch], [true, source], 'the one-step redo'); assert.equal(host().getTurnCheckpoints('scout').length, 3); }); } @@ -704,35 +728,279 @@ describe('operator-change gate: host/command undo by turns', () => { assert.deepEqual(again.kind === 'undo-turns' && again.markers, m, 'a retry reports the same receipt'); }); - it('schedules marks once even when the first attempt fails after scheduling them', async () => { + type OutboxView = { + batches(): Array<{ id: string; status: string }>; + retract(target: string): unknown; + cancel(target: string): unknown; + pendingDispatches(): Array<{ action: string }>; + }; + const outbox = () => (framework as unknown as { discordAwarenessOutbox: OutboxView }).discordAwarenessOutbox; + const arrive = (messageId: string) => framework.getAgent('scout')!.getContextManager().addMessage('Member', [{ type: 'text', text: messageId }], { + serverId: 'discord', channelId: 'discord:g1:c1', messageId, tags: ['chat:addressed'], + }); + const receipt = (applied: Awaited>) => { + const m = applied.kind === 'undo-turns' || applied.kind === 'undo-messages' || applied.kind === 'hide' ? applied.markers : null; + return m && { status: m.status, queued: m.queued, unmarked: m.unmarked, notRemoved: m.notRemoved }; + }; + + it("recovers the activation's recorded receipt after a failure at the outcome record, late arrival included, with no second batch", async () => { await say('a0'); await say('a1'); await say('a2'); await undoWithMarks(2, 'all'); const change = asked[0]!; - const fw = framework as unknown as { - scheduleAppliedMarks: (...args: unknown[]) => unknown; - recordOperatorAction: (entry: { kind: string }) => void; - }; - const realSchedule = fw.scheduleAppliedMarks.bind(fw); - const realRecord = fw.recordOperatorAction.bind(fw); - let scheduled = 0; - fw.scheduleAppliedMarks = (...args) => { scheduled++; return realSchedule(...args); }; - let armed = true; - fw.recordOperatorAction = (entry) => { - if (armed && entry.kind === 'undo-turns') { armed = false; throw new Error('injected failure after marks'); } - return realRecord(entry); - }; - const apply = () => framework.runAtSafeBoundary({ verb: 'apply' }, (lease) => - framework.applyResolvedOperatorChange(change, { lease, admission: { id: 'rev-u' } })); + arrive('late'); + const restore = failJournalOnce('outcome'); try { - await assert.rejects(apply(), /injected failure after marks/); - const retried = await apply(); - const m = retried.kind === 'undo-turns' ? retried.markers : undefined; - assert.deepEqual(m && [m.status, m.queued], ['queued', 1], 'the retry reports the receipt the failed attempt recorded'); - assert.equal(scheduled, 1, 'scheduled once'); + await assert.rejects(applyIt(change), /injected outcome failure/); } finally { - fw.scheduleAppliedMarks = realSchedule; - fw.recordOperatorAction = realRecord; + restore(); } + const retried = await applyIt(change); + assert.deepEqual(receipt(retried), { status: 'queued', queued: 1, unmarked: 1, notRemoved: 0 }, 'a2 marked; the late arrival counted unmarked'); + assert.equal(retried.kind === 'undo-turns' && retried.alreadyApplied, true); + assert.deepEqual(outbox().batches().filter((x) => x.id.startsWith(`op-${change.id}`)).map((x) => x.status), ['active'], 'one batch'); + }); + + it('recovers the established outcome after its destination was used and left and the source restored, never cutting again', async () => { + await say('a0'); await say('a1'); await say('a2'); + await undoWithMarks(2, 'all'); + const change = asked[0]!; + arrive('late'); + const restore = failJournalOnce('outcome'); + try { + await assert.rejects(applyIt(change), /injected outcome failure/); + } finally { + restore(); + } + assert.equal(branch(), `undo/scout/op-${change.id}`); + await turn('on the destination'); // the destination is used + framework.getStore().switchBranch(change.sourceBranch); // and an operator restores the source + const branches = () => framework.getStore().listBranches().map((x) => x.name).sort(); + const before = branches(); + const retried = await applyIt(change); + assert.equal(retried.kind === 'undo-turns' && retried.alreadyApplied, true); + assert.equal(branch(), change.sourceBranch, 'the restored source stays active: source active is not permission to cut again'); + assert.deepEqual(branches(), before, 'no new branch'); + assert.deepEqual(receipt(retried), { status: 'queued', queued: 1, unmarked: 1, notRemoved: 0 }, 'from the attempt that committed'); + }); + + it('never cuts again after its destination was left at once, before any turn ran there', async () => { + await turn('one'); await turn('two'); await turn('three'); + await undo(2); + const change = asked[0]!; + const destination = `undo/scout/op-${change.id}`; + const restore = failJournalOnce('outcome'); + try { + await assert.rejects(applyIt(change), /injected outcome failure/); + } finally { + restore(); + } + // Left before any turn ran there. (Its switch record, a typed record, is + // itself a write on the destination, so the switch record and the + // used-destination evidence prove the cut together.) + framework.getStore().switchBranch(change.sourceBranch); + assert.equal(framework.getOperatorChangeRecord(change.id)!.switched, 1); + void destination; + const retried = await applyIt(change); + assert.equal(retried.kind === 'undo-turns' && retried.alreadyApplied, true, 'never cut again'); + assert.equal(branch(), change.sourceBranch); + }); + + it('treats a used destination as proof of the cut when its switch record was lost, after the source is restored', async () => { + await turn('one'); await turn('two'); await turn('three'); + await undo(2); + const change = asked[0]!; + const restore = failJournalOnce('switched'); + try { + await assert.rejects(applyIt(change), /injected switched failure/); + } finally { + restore(); + } + await turn('on the destination'); // used + framework.getStore().switchBranch(change.sourceBranch); // left, and the source restored + const branches = () => framework.getStore().listBranches().map((x) => x.name).sort(); + const before = branches(); + const retried = await applyIt(change); + assert.equal(retried.kind === 'undo-turns' && retried.alreadyApplied, true, 'committed: never cut again'); + assert.equal(branch(), change.sourceBranch); + assert.deepEqual(branches(), before); + assert.equal(framework.getOperatorChangeRecord(change.id)!.switched, 1, 'its switch recorded now, from the evidence'); + }); + + for (const target of ['all', 'by id'] as const) { + it(`keeps a retract made while the cut waits (${target}): its keys are born superseded`, async () => { + await say('a0'); await say('a1'); await say('a2'); + await undoWithMarks(2, 'all'); + const change = asked[0]!; + outbox().retract(target === 'all' ? 'all' : `op-${change.id}`); + const applied = await applyIt(change); + assert.deepEqual(receipt(applied), { status: 'queued', queued: 0, unmarked: 0, notRemoved: 0 }, 'the retract outranks the staged choice'); + assert.equal(outbox().pendingDispatches().filter((d) => d.action === 'add').length, 0, 'no add is sent'); + }); + } + + it('reports a choice cancelled while the cut waited as not scheduled, with the cut applied', async () => { + await say('a0'); await say('a1'); await say('a2'); + await undoWithMarks(2, 'all'); + const change = asked[0]!; + outbox().cancel(`op-${change.id}`); + const applied = await applyIt(change); + assert.equal(receipt(applied)!.status, 'not-scheduled'); + assert.equal(branch(), `undo/scout/op-${change.id}`, 'the cut applied'); + assert.equal(outbox().pendingDispatches().filter((d) => d.action === 'add').length, 0); + }); + + it('forms a new attempt when the first never cut, counting what the cut actually removes', async () => { + await say('a0'); await say('a1'); await say('a2'); + await undoWithMarks(2, 'all'); + const change = asked[0]!; + arrive('late-1'); + const restore = failSwitchOnce(); + try { + await assert.rejects(applyIt(change), /injected cut failure/); + } finally { + restore(); + } + arrive('late-2'); // arrives on the source before the retry + const applied = await applyIt(change); + assert.deepEqual(receipt(applied), { status: 'queued', queued: 1, unmarked: 2, notRemoved: 0 }, 'both arrivals, as the cut that committed removed them'); + const record = framework.getOperatorChangeRecord(change.id)!; + assert.deepEqual(record.attempts.map((x) => x.evidence.marks.unmarked), [1, 2], 'the first prediction is history, not the outcome'); + assert.equal(record.outcome!.n, 2); + }); + + it("establishes a committed cut's outcome at startup when a crash came before its record", async () => { + await say('a0'); await say('a1'); await say('a2'); + await undoWithMarks(2, 'all'); + const change = asked[0]!; + const restore = failJournalOnce('outcome'); + try { + await assert.rejects(applyIt(change), /injected outcome failure/); + } finally { + restore(); + } + await framework.stop(); + framework = await AgentFramework.create({ + storePath: join(tempDir, 'test.chronicle'), + membrane: membrane.asMembrane(), + agents: [{ name: 'scout', model: 'test-model', systemPrompt: 'You are scout.', maxTokens: 1000 }], + modules: [], + operatorChangeGate: async (c) => { asked.push(structuredClone(c)); return decide(c); }, + }); + const record = framework.getOperatorChangeRecord(change.id)!; + assert.ok(record.outcome && record.completed, 'established before modules or traffic'); + assert.deepEqual([record.outcome.markers.status, record.outcome.markers.queued], ['queued', 1]); + assert.equal(outbox().batches().find((x) => x.id === `op-${change.id}`)?.status, 'active'); + }); + + it('discards a staged choice when the gate refuses its change or the host drops it', async () => { + await say('a0'); await say('a1'); await say('a2'); + decide = async () => { throw new Error('host says no'); }; + const refusal = await undoWithMarks(2, 'all'); + assert.equal(refusal.code, 'gate-failed'); + const refused = asked[0]!; + assert.equal(outbox().batches().find((x) => x.id === `op-${refused.id}`)?.status, 'discarded'); + decide = async (c) => ({ id: 'rev-x', text: `staged ${c.kind}` }); + await undoWithMarks(2, 'all'); + const dropped = asked[1]!; + framework.dropResolvedOperatorChange(dropped); + assert.equal(outbox().batches().find((x) => x.id === `op-${dropped.id}`)?.status, 'discarded'); + await assert.rejects(applyIt(dropped), (e: Error & { code?: string }) => e.code === 'stale' && /dropped/.test(e.message)); + }); + + it('stages a hide with its exact messages and frozen refs, applies it under the lease, and reports one receipt', async () => { + await say('h0'); await say('h1'); await say('h2'); + const r = await host().handleHostCommand('discord', { command: 'hide', agentName: 'scout', fromMessageId: 'h1', toMessageId: 'h2', marks: 'all', requesterName: 'nissa' }); + assert.equal(r.code, 'staged'); + const change = asked[0]!; + assert.equal(change.kind === 'hide' && change.messages.length, 3, 'h1, its reply, and h2'); + assert.deepEqual(change.kind === 'hide' && change.marks, { + scope: 'all', + refs: ['h1', 'h2'].map((messageId) => ({ serverId: 'discord', channelId: 'discord:g1:c1', messageId })), + }); + const applied = await applyIt(change); + assert.deepEqual(applied.kind === 'hide' && applied.hidden, 3); + assert.deepEqual(receipt(applied), { status: 'queued', queued: 2, unmarked: 0, notRemoved: 0 }); + assert.deepEqual(texts().filter((t) => /h[12]/.test(t)), ['reply to h2'], 'exactly the range is gone: the reply after it stays'); + const again = await applyIt(change); + assert.equal(again.kind === 'hide' && again.alreadyApplied, true); + assert.deepEqual(receipt(again), receipt(applied)); + assert.equal(framework.getOperatorLog().filter((e) => e.kind === 'hide').length, 1); + }); + + it("counts an interrupted hide's removals from its own evidence when finishing it", async () => { + await say('h0'); await say('h1'); await say('h2'); + await host().handleHostCommand('discord', { command: 'hide', agentName: 'scout', fromMessageId: 'h1', toMessageId: 'h2', marks: 'all', requesterName: 'nissa' }); + const change = asked[0]!; + const restore = failJournalOnce('outcome'); + try { + await assert.rejects(applyIt(change), /injected outcome failure/); + } finally { + restore(); + } + const finished = await applyIt(change); + assert.equal(finished.kind === 'hide' && finished.hidden, 3, 'what it removed before the interruption still counts'); + assert.deepEqual(receipt(finished), { status: 'queued', queued: 2, unmarked: 0, notRemoved: 0 }); + }); + + it('refuses a hide whose message changed, or whose interrupted attempt the active branch left', async () => { + await say('h0'); await say('h1'); await say('h2'); + await host().handleHostCommand('discord', { command: 'hide', agentName: 'scout', fromMessageId: 'h1', toMessageId: 'h2', marks: 'none', requesterName: 'nissa' }); + const edited = asked[0]!; + const cmx = framework.getAgent('scout')!.getContextManager(); + const h1 = cmx.getAllMessages().find((m) => (m.metadata as { messageId?: string } | undefined)?.messageId === 'h1')!; + cmx.editMessage(h1.id, [{ type: 'text', text: 'h1, edited' }]); + await assert.rejects(applyIt(edited), (e: Error & { code?: string }) => e.code === 'stale' && /changed since it was staged/.test(e.message)); + + await host().handleHostCommand('discord', { command: 'hide', agentName: 'scout', fromMessageId: 'h2', marks: 'none', requesterName: 'nissa' }); + const interrupted = asked[1]!; + const restore = failJournalOnce('outcome'); + try { + await assert.rejects(applyIt(interrupted), /injected outcome failure/); + } finally { + restore(); + } + framework.getStore().createBranch('elsewhere'); + framework.getStore().switchBranch('elsewhere'); + await assert.rejects(applyIt(interrupted), (e: Error & { code?: string }) => e.code === 'stale' && /outcome is unknown/.test(e.message)); + }); + + it('stages an undo by messages at its tail, cuts onto its own branch, and counts a later arrival unmarked', async () => { + await say('m0'); await say('m1'); + const r = await host().handleHostCommand('discord', { command: 'undo', agentName: 'scout', messages: 2, marks: 'all', requesterName: 'nissa' }); + assert.equal(r.code, 'staged'); + const change = asked[0]!; + assert.deepEqual(change.kind === 'undo-messages' && change.marks, { + scope: 'all', + refs: [{ serverId: 'discord', channelId: 'discord:g1:c1', messageId: 'm1' }], + }, 'm1 and its reply follow the tail'); + arrive('late'); + const applied = await applyIt(change); + assert.deepEqual(applied.kind === 'undo-messages' && [applied.toBranch, applied.messagesRemoved], [`undo-msgs/scout/op-${change.id}`, 3]); + assert.deepEqual(receipt(applied), { status: 'queued', queued: 1, unmarked: 1, notRemoved: 0 }); + assert.deepEqual(texts(), ['m0', 'reply to m0']); + const again = await applyIt(change); + assert.equal(again.kind === 'undo-messages' && again.alreadyApplied, true); + }); + + it('refuses an undo by messages whose tail changed since staging', async () => { + await say('m0'); await say('m1'); + await host().handleHostCommand('discord', { command: 'undo', agentName: 'scout', messages: 2, marks: 'none', requesterName: 'nissa' }); + const change = asked[0]!; + const cmx = framework.getAgent('scout')!.getContextManager(); + cmx.editMessage(cmx.getAllMessages()[1]!.id, [{ type: 'text', text: 'reply, edited' }]); + await assert.rejects(applyIt(change), (e: Error & { code?: string }) => e.code === 'stale' && /tail message .* changed/.test(e.message)); + }); + + it("runs the public rollback under a host's held lease, and refuses it inside one without the lease", async () => { + await turn('one'); await turn('two'); + const target = String(framework.getAgent('scout')!.getContextManager().getAllMessages()[1]!.id); + await assert.rejects( + framework.runAtSafeBoundary({ verb: 'nested' }, () => framework.rollbackToMessage('scout', { messageId: target })), + (e: Error & { code?: string }) => e.code === 'agent-busy', + ); + const rolled = await framework.runAtSafeBoundary({ verb: 'held' }, (lease) => + framework.rollbackToMessage('scout', { messageId: target, lease })); + assert.equal(rolled.messagesRemoved, 2); }); it("keeps a staged undo without a marks choice local, still counting what it removed", async () => { @@ -1156,3 +1424,75 @@ describe('operator-change gate: host/command unstick, planned at staging', () => assert.deepEqual(texts(), ['one', 'reply to one', 'back'], 'exactly the planned range went, then the agent answered'); }); }); + +describe('surgery bound to the context it was previewed in', () => { + let dir: string; + let a: AgentFramework; + let b: AgentFramework; + let quiet: { log: typeof console.log; error: typeof console.error }; + const make = (name: string) => AgentFramework.create({ + storePath: join(dir, name), + membrane: new MockMembrane().asMembrane(), + agents: [{ name: 'scout', model: 'test-model', systemPrompt: 'You are scout.', maxTokens: 1000 }], + modules: [], + }); + const seed = (fw: AgentFramework, n: number) => { + const cm = fw.getAgent('scout')!.getContextManager(); + for (let i = 0; i < n; i++) cm.addMessage('user', [{ type: 'text', text: `m${i}` }]); + return cm; + }; + const stale = (pattern: RegExp) => (e: Error & { code?: string }) => e.code === 'stale' && pattern.test(e.message); + + beforeEach(async () => { + dir = mkdtempSync(join(tmpdir(), 'surgery-context-')); + quiet = { log: console.log, error: console.error }; + console.log = () => {}; + console.error = () => {}; + a = await make('a'); + b = await make('b'); + }); + afterEach(async () => { + await a.stop(); + await b.stop(); + console.log = quiet.log; + console.error = quiet.error; + rmSync(dir, { recursive: true, force: true }); + }); + + it('refuses a rollback previewed in one store when it reaches another, before any change', async () => { + const ca = seed(a, 5); + const cb = seed(b, 5); + const target = String(ca.getAllMessages()[1]!.id); + assert.equal(String(cb.getAllMessages()[1]!.id), target, 'both stores assign the same id'); + const preview = a.previewSurgeryMarks('scout', { rollbackTo: target }); + assert.equal(preview.context.storeId, a.getStoreIdentity()); + assert.notEqual(a.getStoreIdentity(), b.getStoreIdentity()); + await assert.rejects(b.rollbackToMessage('scout', { messageId: target, expected: preview.context }), stale(/another store/)); + assert.equal(cb.getAllMessages().length, 5, 'the other store is untouched'); + const rolled = await a.rollbackToMessage('scout', { messageId: target, expected: preview.context }); + assert.equal(rolled.messagesRemoved, 3); + }); + + it('refuses a rollback whose branch moved since its preview, and keeps same-branch arrivals within it', async () => { + const ca = seed(a, 5); + const target = String(ca.getAllMessages()[1]!.id); + const preview = a.previewSurgeryMarks('scout', { rollbackTo: target }); + ca.addMessage('user', [{ type: 'text', text: 'arrived since' }]); + const store = a.getStore(); + store.createBranch('moved'); + store.switchBranch('moved'); + await assert.rejects(a.rollbackToMessage('scout', { messageId: target, expected: preview.context }), stale(/active branch moved/)); + store.switchBranch(preview.context.branch); + const rolled = await a.rollbackToMessage('scout', { messageId: target, expected: preview.context }); + assert.equal(rolled.messagesRemoved, 4, 'the arrival on the same branch is part of the chosen rollback'); + }); + + it('checks the same context for a suppression', async () => { + const ca = seed(a, 5); + seed(b, 5); + const target = String(ca.getAllMessages()[2]!.id); + const preview = a.previewSurgeryMarks('scout', { suppress: [target] }); + await assert.rejects(b.suppressMessages('scout', { messageIds: [target], expected: preview.context }), stale(/another store/)); + assert.equal(b.getAgent('scout')!.getContextManager().getAllMessages().length, 5); + }); +}); From 987a175a604fc9e2568d38f28b0b05b30fad8083 Mon Sep 17 00:00:00 2001 From: Agnes-1009 Date: Wed, 7 Oct 2026 16:49:56 -0400 Subject: [PATCH 48/73] fix(framework): a failed cut is never a commitment; unreadable operator journals stop startup; body-asserting records follow the store's commit (review) June's three R2 recovery blockers on c5a07bc (room-222 #48574, #48629). Ada's suggestions are taken for the first two (#48590). 1. **A failed, restored cut was certified as committed.** The cut's failure path (strategy initialization failing after cm.switchBranch moved the store and wrote on the destination) restored the source. destinationUsed() then read that initializer write as commitment: the retry reported alreadyApplied and published marks while the source still held every message. - A failed attempt now has a durable disposition. applyCut records `failed {n, error}` before the restore runs, and the restore is now a step of its own, so it is never inside the cut. - Recovery never reads an attempt with `failed` as committed, whatever its destination holds. If a crash came between the record and the restore (the destination still active), recovery finishes the restore, then starts attempt n+1 under the same staging authorization. - A completed cut later restored by an operator is still proven by its `switched` record and its outcome, which a failed attempt never reaches. - Startup reconciliation skips failed attempts. - If the failure record itself can't be written, the restore still runs and the error says so. 2. **An unreadable operator/changes journal became a ready, empty ledger.** changesLedger() installed empty state before loading, and startup swallowed the load error. - Both operator journals (operator/changes, operator/unstick) now load before they're installed. A read failure throws OperatorJournalUnreadableError (exported), and a failed reload after an ambiguous write uninstalls, so the next use loads again. - Startup reads both before modules or traffic, and an unreadable one stops AgentFramework.create, as an unreadable awareness journal already does. - Mid-run, every use fails rather than answering as if there were no operations. 3. **An outcome could survive a hard kill without its body.** journalChange appended with `durable` only, so the sync came after the append. `switched` and `outcome`, the records asserting that the body committed, now append with `afterCommittedState`: the store's committed state is synced before the record is written. The marked path already got this barrier inside activateStaged; zero-ref and refused publications now get it from operator/changes itself. Tests: - A failed cut that wrote on its destination is retried as a fresh attempt: really cut, marks queued once, attempts [failed, committed]. The same holds when the crash came before its restore. - Startup is refused when either journal is unreadable. Mid-run, both a first and a second read throw rather than returning null. - SIGKILL in a child at the sync right after the outcome append (June's probe): on reopen the outcome {removed: 2} and the removed messages agree. Mutants caught: - the failed record ignored; - the failed record not written; - an empty ledger installed before the load; - no commit barrier (reproduces June's kill result); - the unstick journal not read at startup. 24 suites: 408/408. Co-Authored-By: Agnes (Claude Opus 5.5) --- src/framework.ts | 142 ++++++++++++++++++------- src/index.ts | 1 + src/operator-change.ts | 6 +- test/operator-change-gate.test.ts | 167 +++++++++++++++++++++++++++++- 4 files changed, 277 insertions(+), 39 deletions(-) diff --git a/src/framework.ts b/src/framework.ts index 230f70ed0..e58678846 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -606,6 +606,7 @@ type UnstickJournalSnapshot = Record; type OperatorChangesEntry = | { kind: 'attempt'; changeId: string; changeKind: OperatorChangeRecord['kind']; agent: string; n: number; at: number; evidence: OperatorChangeEvidence } | { kind: 'switched'; changeId: string; n: number } + | { kind: 'failed'; changeId: string; n: number; error: string } | { kind: 'outcome'; changeId: string; outcome: OperatorChangeOutcome } | { kind: 'completed'; changeId: string } | { kind: 'dropped'; changeId: string; changeKind: OperatorChangeRecord['kind']; agent: string; at: number }; @@ -628,7 +629,10 @@ function reduceOperatorChangesEntry(records: Map, const record = records.get(entry.changeId); if (!record) return; if (entry.kind === 'switched') record.switched = entry.n; - else if (entry.kind === 'outcome') record.outcome ??= entry.outcome; + else if (entry.kind === 'failed') { + const attempt = record.attempts.find((a) => a.n === entry.n); + if (attempt) attempt.failed = entry.error; + } else if (entry.kind === 'outcome') record.outcome ??= entry.outcome; else record.completed = true; } @@ -706,6 +710,17 @@ function normalizeDiscordAwarenessDeadline(value: number | undefined): number { ); } +/** An operator journal (operator/changes, operator/unstick) could not be + * read. It may hold a body change that still needs recovery, so neither + * startup nor an operation proceeds as if it were empty. */ +export class OperatorJournalUnreadableError extends Error { + constructor(journal: string, cause: unknown) { + const detail = cause instanceof Error ? cause.message : String(cause); + super(`The ${journal} journal could not be read: ${detail}`, { cause }); + this.name = 'OperatorJournalUnreadableError'; + } +} + class DiscordAwarenessAccountingError extends Error { constructor(operation: string, cause: unknown) { const detail = cause instanceof Error ? cause.message : String(cause); @@ -7102,19 +7117,32 @@ export class AgentFramework { } { if (!this.unstickJournalState) { const journal = new RecordJournal(this.store, { type: 'operator/unstick' }); - this.unstickJournalState = { journal, ops: new Map() }; - this.reloadUnstickJournal(); + this.unstickJournalState = { journal, ops: this.loadUnstickJournal(journal) }; } return this.unstickJournalState; } - /** Rebuild the reduced operations from exactly what the journal holds: at - * first use, and to reconcile after an ambiguous write. */ + private loadUnstickJournal(journal: RecordJournal): Map { + try { + const { snapshot, entries } = journal.load(); + const ops = new Map(Object.entries(snapshot ?? {}).map(([id, op]) => [id, structuredClone(op)])); + for (const { entry } of entries) reduceUnstickEntry(ops, entry); + return ops; + } catch (error) { + throw new OperatorJournalUnreadableError('operator/unstick', error); + } + } + + /** Rebuild the reduced operations from exactly what the journal holds, + * after an ambiguous write. If it can't be read, nothing stays installed. */ private reloadUnstickJournal(): void { const state = this.unstickJournalState!; - const { snapshot, entries } = state.journal.load(); - state.ops = new Map(Object.entries(snapshot ?? {}).map(([id, op]) => [id, structuredClone(op)])); - for (const { entry } of entries) reduceUnstickEntry(state.ops, entry); + try { + state.ops = this.loadUnstickJournal(state.journal); + } catch (error) { + this.unstickJournalState = null; + throw error; + } } /** Append one journal entry and reduce it. A failure after the entry may @@ -7151,19 +7179,35 @@ export class AgentFramework { records: Map; } { if (!this.changesJournalState) { + // Loaded before it is installed: a journal that can't be read is never + // mistaken for one with no operations. const journal = new RecordJournal(this.store, { type: 'operator/changes' }); - this.changesJournalState = { journal, records: new Map() }; - this.reloadChangesJournal(); + this.changesJournalState = { journal, records: this.loadChangesJournal(journal) }; } return this.changesJournalState; } - /** Rebuild the reduced records from exactly what the journal holds. */ + private loadChangesJournal(journal: RecordJournal): Map { + try { + const { snapshot, entries } = journal.load(); + const records = new Map(Object.entries(snapshot ?? {}).map(([id, record]) => [id, structuredClone(record)])); + for (const { entry } of entries) reduceOperatorChangesEntry(records, entry); + return records; + } catch (error) { + throw new OperatorJournalUnreadableError('operator/changes', error); + } + } + + /** Rebuild the reduced records from exactly what the journal holds. If it + * can't be read, nothing stays installed: the next use loads again. */ private reloadChangesJournal(): void { const state = this.changesJournalState!; - const { snapshot, entries } = state.journal.load(); - state.records = new Map(Object.entries(snapshot ?? {}).map(([id, record]) => [id, structuredClone(record)])); - for (const { entry } of entries) reduceOperatorChangesEntry(state.records, entry); + try { + state.records = this.loadChangesJournal(state.journal); + } catch (error) { + this.changesJournalState = null; + throw error; + } } /** Append one entry, durable, and reduce it. A failure after the entry may @@ -7172,7 +7216,11 @@ export class AgentFramework { private journalChange(entry: OperatorChangesEntry): void { const ledger = this.changesLedger(); try { - ledger.journal.append(entry, { durable: true }); + // A record asserting the body committed (its switch, its outcome) is + // appended only after the store's committed state is synced, so a + // hard kill can never keep the record without the body it certifies. + const assertsBody = entry.kind === 'switched' || entry.kind === 'outcome'; + ledger.journal.append(entry, { durable: true, ...(assertsBody ? { afterCommittedState: true } : {}) }); } catch (error) { if (ledger.journal.needsReconcile) this.reloadChangesJournal(); throw error; @@ -7786,6 +7834,8 @@ export class AgentFramework { revalidate: (stale: (why: string) => never) => void; removal: () => Array<{ metadata?: unknown }>; cut: () => Promise; + /** Put the source back if the cut left the destination active. */ + restore: () => Promise; finish: (outcome: OperatorChangeOutcome) => void; }, ): Promise<{ outcome: OperatorChangeOutcome; alreadyApplied: boolean }> { @@ -7807,9 +7857,17 @@ export class AgentFramework { }; const known = record(); if (known?.outcome) return settle(known.outcome, true); - const current = this.store.currentBranch().name; + let current = this.store.currentBranch().name; const last = known?.attempts[known.attempts.length - 1]; - if (last) { + if (last && last.failed !== undefined) { + // That attempt failed, and its disposition was recorded before its + // source was restored: never a commitment, whatever its destination + // holds. If a crash came before the restore, finish it now. + if (current === destination) { + await steps.restore(); + current = this.store.currentBranch().name; + } + } else if (last) { if (known!.switched === last.n || current === destination || this.destinationUsed(destination)) { if (known!.switched !== last.n) this.journalChange({ kind: 'switched', changeId: change.id, n: last.n }); return settle(this.establishOutcome(change, last, 'undo'), true); @@ -7840,7 +7898,21 @@ export class AgentFramework { ...(change.requester ? { requester: change.requester } : {}), }, }); - await steps.cut(); + try { + await steps.cut(); + } catch (error) { + const detail = error instanceof Error ? error.message : String(error); + // Recorded before the source is restored, so recovery never reads a + // destination this attempt touched as its cut. + let recorded = ''; + try { + this.journalChange({ kind: 'failed', changeId: change.id, n, error: detail }); + } catch (journalError) { + recorded = `; its failure could not be recorded (${journalError instanceof Error ? journalError.message : String(journalError)})`; + } + const restored = await steps.restore(); + throw new OperatorActionError('failed', `${label} ${change.id} failed; ${restored}${recorded}: ${detail}`, { cause: error }); + } this.journalChange({ kind: 'switched', changeId: change.id, n }); return settle(this.establishOutcome(change, record()!.attempts.find((a) => a.n === n)!, 'undo'), false); } @@ -7881,6 +7953,12 @@ export class AgentFramework { this.store.switchBranch(destination); this.materializeConfigMountAfterBranchSwitch(); }, + restore: async () => { + if (this.store.currentBranch().name !== destination) return `active branch is ${this.store.currentBranch().name}`; + this.store.switchBranch(change.sourceBranch); + this.materializeConfigMountAfterBranchSwitch(); + return `active branch restored to ${change.sourceBranch}`; + }, finish: (established) => { // The one redo entry back to the source tip, while the destination is // active and no turn has run on it (new work invalidates redo). @@ -7952,19 +8030,11 @@ export class AgentFramework { }, removal: () => this.planRollback(change.agent, change.tail.id).discarded, cut: async () => { - try { - const exists = this.store.listBranches().some((b) => b.name === destination); - await cm.switchBranch(exists ? destination : cm.branchAt(change.tail.id as MessageId, destination)); - } catch (error) { - const restored = await this.restoreSourceBranch(cm, change.sourceBranch, destination, change.agent); - throw new OperatorActionError( - 'failed', - `Undo of ${change.requestedMessages} message(s) ${change.id} failed; ${restored}: ${error instanceof Error ? error.message : String(error)}`, - { cause: error }, - ); - } + const exists = this.store.listBranches().some((b) => b.name === destination); + await cm.switchBranch(exists ? destination : cm.branchAt(change.tail.id as MessageId, destination)); this.materializeConfigMountAfterBranchSwitch(); }, + restore: () => this.restoreSourceBranch(cm, change.sourceBranch, destination, change.agent), finish: (established) => { this.recordChangeAction({ kind: 'rollback', @@ -8111,16 +8181,16 @@ export class AgentFramework { * not provably committed is left for the host's retry. */ private async reconcileOperatorChanges(): Promise { - let records: OperatorChangeRecord[]; - try { - records = [...this.changesLedger().records.values()]; - } catch (error) { - console.error('[operator-changes] journal unreadable at startup:', error instanceof Error ? error.message : error); - return; - } + // Both operator journals are read before anything else runs: either may + // hold a body change that still needs recovery, so an unreadable one + // stops startup (OperatorJournalUnreadableError) rather than reading as + // empty. + this.unstickLedger(); + const records = [...this.changesLedger().records.values()]; for (const record of records) { if (record.outcome || record.dropped || record.attempts.length === 0) continue; const last = record.attempts[record.attempts.length - 1]!; + if (last.failed !== undefined) continue; // never committed: the host's retry starts a fresh attempt try { const current = this.store.currentBranch().name; if (record.kind === 'hide') { diff --git a/src/index.ts b/src/index.ts index 2ee714ad0..81477f241 100644 --- a/src/index.ts +++ b/src/index.ts @@ -4,6 +4,7 @@ export * from './types/index.js'; // Core classes export { AgentFramework, BudgetPreflightError, ResumeBlockedError } from './framework.js'; export type { RuntimeSettingsPreview, HostModeStatus, SafeBoundaryLease, SafeBoundaryOptions } from './framework.js'; +export { OperatorJournalUnreadableError } from './framework.js'; export type { OperatorChangeReceipt, ResolvedOperatorChange, diff --git a/src/operator-change.ts b/src/operator-change.ts index 178267aee..cad4a3de8 100644 --- a/src/operator-change.ts +++ b/src/operator-change.ts @@ -191,8 +191,10 @@ export interface OperatorChangeRecord { changeId: string; kind: ResolvedOperatorChange['kind']; agent: string; - /** Application attempts, in order, each with its evidence. */ - attempts: Array<{ n: number; at: number; evidence: OperatorChangeEvidence }>; + /** Application attempts, in order, each with its evidence. `failed` is set + * (with the error) when the attempt failed and restored its source: such + * an attempt is never read as committed. */ + attempts: Array<{ n: number; at: number; evidence: OperatorChangeEvidence; failed?: string }>; /** The attempt whose cut switched to its destination: recorded right * after the switch, so it proves the cut whatever is active later. */ switched?: number; diff --git a/test/operator-change-gate.test.ts b/test/operator-change-gate.test.ts index cbc49f19d..75b973191 100644 --- a/test/operator-change-gate.test.ts +++ b/test/operator-change-gate.test.ts @@ -13,7 +13,10 @@ import { describe, it, beforeEach, afterEach } from 'node:test'; import assert from 'node:assert/strict'; import { tmpdir } from 'node:os'; -import { mkdtempSync, rmSync } from 'node:fs'; +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { spawnSync } from 'node:child_process'; +import { fileURLToPath } from 'node:url'; +import { JsStore } from '@animalabs/chronicle'; import { join } from 'node:path'; import { AgentFramework, AutobiographicalStrategy, WorkspaceModule } from '../src/index.js'; import type { @@ -982,6 +985,44 @@ describe('operator-change gate: host/command undo by turns', () => { assert.equal(again.kind === 'undo-messages' && again.alreadyApplied, true); }); + for (const crashBeforeRestore of [false, true]) { + it(`never reads a failed cut as committed, though it wrote on its destination${crashBeforeRestore ? ', even when the crash came before its restore' : ''}`, async () => { + await say('m0'); await say('m1'); + await host().handleHostCommand('discord', { command: 'undo', agentName: 'scout', messages: 2, marks: 'all', requesterName: 'nissa' }); + const change = asked[0]!; + const destination = `undo-msgs/scout/op-${change.id}`; + const cmx = framework.getAgent('scout')!.getContextManager() as unknown as { switchBranch: (name: string) => Promise }; + const realSwitch = cmx.switchBranch.bind(cmx); + let armed = true; + // The store moves and strategy initialization writes on the destination + // before failing, as June's probe did. + cmx.switchBranch = async (name) => { + if (!armed) return realSwitch(name); + armed = false; + framework.getStore().switchBranch(name); + framework.getStore().setStateJson('probe/initializer', { wrote: true }); + throw new Error('injected strategy initialization failure'); + }; + const fw = framework as unknown as { restoreSourceBranch: (...args: unknown[]) => Promise }; + const realRestore = fw.restoreSourceBranch.bind(fw); + if (crashBeforeRestore) fw.restoreSourceBranch = async () => { throw new Error('the process died before restoring'); }; + try { + await assert.rejects(applyIt(change), crashBeforeRestore ? /died before restoring/ : /injected strategy initialization failure/); + } finally { + cmx.switchBranch = realSwitch; + fw.restoreSourceBranch = realRestore; + } + assert.equal(branch(), crashBeforeRestore ? destination : change.sourceBranch); + assert.equal(framework.getOperatorChangeRecord(change.id)!.attempts[0]!.failed !== undefined, true, 'its failure recorded first'); + const retried = await applyIt(change); + assert.equal(retried.kind === 'undo-messages' && retried.alreadyApplied, undefined, 'a fresh attempt, not a recovered commitment'); + assert.deepEqual(texts(), ['m0', 'reply to m0'], 'the cut really happened this time'); + assert.deepEqual(receipt(retried), { status: 'queued', queued: 1, unmarked: 0, notRemoved: 0 }, 'marks from the attempt that committed'); + const record = framework.getOperatorChangeRecord(change.id)!; + assert.deepEqual([record.attempts.length, record.outcome!.n], [2, 2]); + }); + } + it('refuses an undo by messages whose tail changed since staging', async () => { await say('m0'); await say('m1'); await host().handleHostCommand('discord', { command: 'undo', agentName: 'scout', messages: 2, marks: 'none', requesterName: 'nissa' }); @@ -1496,3 +1537,127 @@ describe('surgery bound to the context it was previewed in', () => { assert.equal(b.getAgent('scout')!.getContextManager().getAllMessages().length, 5); }); }); + + +describe('operator journals that cannot be read', () => { + let dir: string; + let quiet: { log: typeof console.log; error: typeof console.error }; + beforeEach(() => { + dir = mkdtempSync(join(tmpdir(), 'operator-journal-unreadable-')); + quiet = { log: console.log, error: console.error }; + console.log = () => {}; + console.error = () => {}; + }); + afterEach(() => { + console.log = quiet.log; + console.error = quiet.error; + rmSync(dir, { recursive: true, force: true }); + }); + const unreadable = (store: JsStore, type: string) => { + const real = store.getRecordIdsByType.bind(store); + store.getRecordIdsByType = (t: string) => { + if (t === type) throw new Error('injected read failure'); + return real(t); + }; + }; + + for (const type of ['operator/changes', 'operator/unstick']) { + it(`refuses to start when ${type} can't be read, rather than reading it as empty`, async () => { + const store = JsStore.openOrCreate({ path: join(dir, 'store') }); + unreadable(store, type); + await assert.rejects( + AgentFramework.create({ + store, + membrane: new MockMembrane().asMembrane(), + agents: [{ name: 'scout', model: 'test-model', systemPrompt: 'You are scout.', maxTokens: 1000 }], + modules: [], + }), + (e: Error) => e.name === 'OperatorJournalUnreadableError' && e.message.includes(type), + ); + }); + } + + it('fails an operation mid-run when operator/changes becomes unreadable, never answering as if it were empty', async () => { + const framework = await AgentFramework.create({ + storePath: join(dir, 'live'), + membrane: new MockMembrane().asMembrane(), + agents: [{ name: 'scout', model: 'test-model', systemPrompt: 'You are scout.', maxTokens: 1000 }], + modules: [], + }); + try { + (framework as unknown as { changesJournalState: unknown }).changesJournalState = null; // the next use loads again + unreadable(framework.getStore() as JsStore, 'operator/changes'); + for (const attempt of ['first', 'again']) { + assert.throws(() => framework.getOperatorChangeRecord('prior-hide'), (e: Error) => e.name === 'OperatorJournalUnreadableError', + `${attempt}: an unreadable journal is never installed as an empty one`); + } + } finally { + await framework.stop(); + } + }); +}); + +describe('a hard kill between a body change and its outcome record', () => { + it('never keeps an outcome without the body it certifies', () => { + const dir = mkdtempSync(join(tmpdir(), 'operator-outcome-kill-')); + try { + const index = fileURLToPath(new URL('../src/index.js', import.meta.url)); + const mock = fileURLToPath(new URL('./helpers/mock-membrane.js', import.meta.url)); + const script = join(dir, 'child.mjs'); + writeFileSync(script, ` + import { AgentFramework } from ${JSON.stringify(index)}; + import { MockMembrane } from ${JSON.stringify(mock)}; + import { writeFileSync } from 'node:fs'; + import { join } from 'node:path'; + const [mode, dir] = process.argv.slice(2); + console.log = () => {}; console.error = () => {}; + const staged = []; + const fw = await AgentFramework.create({ + storePath: join(dir, 'store'), + membrane: new MockMembrane().asMembrane(), + agents: [{ name: 'scout', model: 'test-model', systemPrompt: 'x', maxTokens: 1000 }], + modules: [], + operatorChangeGate: async (c) => { staged.push(c); return { id: 'rev', text: 'staged' }; }, + }); + const cm = fw.getAgent('scout').getContextManager(); + const texts = () => cm.getAllMessages().map((m) => m.content[0]?.text); + if (mode === 'check') { + const ids = JSON.parse((await import('node:fs')).readFileSync(join(dir, 'change.json'), 'utf8')); + const record = fw.getOperatorChangeRecord(ids.id); + writeFileSync(join(dir, 'after.json'), JSON.stringify({ removed: record?.outcome?.removed ?? null, texts: texts() })); + await fw.stop(); + process.exit(0); + } + for (const id of ['m0', 'm1', 'm2', 'm3']) { + cm.addMessage('Member', [{ type: 'text', text: id }], { serverId: 'discord', channelId: 'discord:g1:c1', messageId: id }); + } + fw.getStore().sync(); + await fw.handleHostCommand('discord', { command: 'hide', agentName: 'scout', fromMessageId: 'm1', toMessageId: 'm2', marks: 'none' }); + const change = staged[0]; + writeFileSync(join(dir, 'change.json'), JSON.stringify({ id: change.id })); + const store = fw.getStore(); + const realAppend = store.appendJson.bind(store); + const realSync = store.sync.bind(store); + let outcomeAppended = false; + store.appendJson = (type, data) => { + const record = realAppend(type, data); + if (type === 'operator/changes' && data && data.kind === 'outcome') outcomeAppended = true; + return record; + }; + store.sync = () => { if (outcomeAppended) process.kill(process.pid, 'SIGKILL'); return realSync(); }; + await fw.runAtSafeBoundary({ verb: 'apply' }, (lease) => fw.applyResolvedOperatorChange(change, { lease, admission: { id: 'rev' } })); + process.exit(3); // not reached: killed at the sync right after the outcome append + `); + const run = (mode: string) => spawnSync(process.execPath, [script, mode, dir], { encoding: 'utf8' }); + const applied = run('apply'); + assert.equal(applied.signal, 'SIGKILL', applied.stderr); + const check = run('check'); + assert.equal(check.status, 0, check.stderr); + const after = JSON.parse(readFileSync(join(dir, 'after.json'), 'utf8')) as { removed: number | null; texts: string[] }; + assert.equal(after.removed, 2, 'the outcome is known after reopen (recorded, or established at startup)'); + assert.deepEqual(after.texts, ['m0', 'm3'], 'and the body it certifies is there with it'); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); +}); From 3c333d8afef99b9a1140ea842ec32d96fc76a08a Mon Sep 17 00:00:00 2001 From: Agnes-1009 Date: Wed, 7 Oct 2026 17:33:39 -0400 Subject: [PATCH 49/73] fix+feat(framework): only records or what the process saw decide a cut; unresolved attempts held and settled by attested verdicts, live or offline; whole-group hides; per-attempt destinations (review) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Kit's two body-path findings on c5a07bc (room-222 #48729). The disposition model agreed after June's storage-failure probe on e52bc54 (#48790): Ada #48798, #48827, #48846, #48861, #48886; June #48815, #48836, #48858, #48883; Kit #48831, #48854, #48878; mine #48819. Kit's findings - **A gated hide failed on a one-shard body group.** Removal now takes contiguous runs as range removals, singletons included, since Chronicle removes a body group only whole and only by range. Resolution expands the named range to whole groups (bodyGroupRun), so the frozen plan names every shard affected. A middle shard hides its whole group. - **A foreign branch under the generated destination name was trusted.** Every attempt now cuts onto a branch of its own: `base` first, then `base~n`. It always creates that branch and never reuses one. If the name already exists, the attempt is refused (`invalid`) before anything is recorded or cut. Dispositions (cutAttemptDisposition, pure and shared with the CLI) - **Committed:** a framework-observed `switched`, an operator's `committed` resolution, or this process's own knowledge of a completed cut. - **Not committed:** a recorded `failed`, an operator's `not-committed` resolution, or this process's knowledge of a failure. - **Unresolved:** anything else, whatever the destination's state. The used-destination check is gone as evidence (Kit withdrew it, #48831), and so is any inference from a branch's existence, since branches can be restored from or deleted after an unrecorded switch (June #48836). Process knowledge - cutKnowledge holds, per attempt, what this process saw. It binds even when the record can't be written: a failure whose `failed` append failed still yields a fresh attempt in process (June's 205 probe). - An attempt record whose own write threw counts as `failed`: this process knows that cut never ran. Held attempts - Applying a held change is refused with the new OperatorActionError code `unresolved`. It is never certified and never cut again, and its staged marks stay unpublished. - getOperatorChangeRecord reports `unresolved {n, target}`, computed at read time and never journaled. Settling, as the operator's own attestation: a `resolved` record carrying the verdict, reason, requester, time and `via` ('live' | 'offline'), never a `switched` or `failed`. - **Live:** `resolveOperatorChange(changeId, attempt, verdict, { lease, reason, requester })`. It needs the held lease and a reason, and it settles only the current unresolved attempt; any other is `stale`. - `not-committed` abandons the attempt and restores the source if its destination is the active body. - `committed` attests that the attempt applied. The outcome is established, and the marks activated, on the host's next retry or at the next start, never by the call itself. - The receipt (OperatorChangeResolutionReceipt) names what was recorded and what was restored, when it settles, and anything that remains. - **Offline:** `agent-framework-recover --operator-change list | resolve --attempt n --verdict committed|not-committed --reason … [--requester …]`, with the host stopped (single writer). It records the attestation only. Its receipt says the next start establishes a committed outcome and activates its marks before traffic, or, for not-committed, that the next start restores the source before any agent initializes. Startup - restoreAbandonedCutSources runs right after the store opens and before any agent initializes. An abandoned attempt (failed, or settled not-committed) whose destination is the active branch gets its source back, synced, so a body that can't even initialize never blocks its own recovery. A failed restore stops startup with OperatorChangeRecoveryError (exported). This is also the first read of both operator journals, so an unreadable one stops startup there. - An unresolved attempt whose destination is the active body boots **quiesced**, with an ops alert. The flags are in memory only, so it's re-derived at every boot. A held attempt elsewhere leaves traffic alone, since the body is the original. - Committed attempts, by switch or by attestation, have their outcomes established before modules or traffic, as before. Shared model: the journal entry types, reduceOperatorChangesEntry, OPERATOR_CHANGES_JOURNAL and cutAttemptDisposition move to operator-change.ts, so the CLI reads the same journal the framework writes. Tests (operator-change-gate, 91): - A one-shard group and a three-shard group, each hidden whole. - A foreign branch under the generated name: refused, nothing cut, nothing recorded. - A failure whose record was lost, after a restart: held with traffic allowed, refused as `unresolved`, a wrong attempt refused as `stale`, settled not-committed; the retry is then a fresh attempt on `~2`. - A lost switch record with the destination active, after a restart: boots quiesced and refused, with marks still staged; settled committed, marks still staged; the retry establishes the outcome {queued 1}, recorded as an attestation rather than a switch. - A recorded failure whose process died before restoring: the source is restored at startup and nothing is quiesced. - Kit's end-to-end case, offline: a context strategy that can't initialize on the cut body makes create() fail. `--operator-change list` shows the attempt with its destination active. `resolve not-committed` records it offline. The next start initializes on the restored source, unquiesced. - An offline committed verdict: the next start establishes the outcome, activates the marks, and doesn't quiesce. Mutants caught: - missing evidence read as success; - process knowledge ignored; - no quiesce on an active unresolved body; - an attestation recorded as a framework switch; - no restore before the agents. A mutant dropping the CLI's immediate restore survived, because startup restored anyway. That showed two mechanisms where one belongs, and a gap where creating the agent itself fails. So the restore moved into startup, ahead of the agents, and the CLI only records. 24 suites: 415/415. Co-Authored-By: Agnes (Claude Opus 5.5) --- src/framework.ts | 348 +++++++++++++++++++++--------- src/index.ts | 2 +- src/operator-change.ts | 107 ++++++++- src/operator-log.ts | 2 +- src/recovery/recover-cli.ts | 133 ++++++++++++ test/operator-change-gate.test.ts | 248 ++++++++++++++++++++- 6 files changed, 721 insertions(+), 119 deletions(-) diff --git a/src/framework.ts b/src/framework.ts index e58678846..360fd09a5 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -19,7 +19,14 @@ import { type ResolvedPresentationChange, type ResolvedSettingsChange, type FrozenMarks, + cutAttemptDisposition, + OPERATOR_CHANGES_JOURNAL, + reduceOperatorChangesEntry, + type OperatorChangesEntry, + type OperatorChangesSnapshot, type OperatorChangeEvidence, + type OperatorChangeResolution, + type OperatorChangeResolutionReceipt, type OperatorChangeOutcome, type OperatorChangeRecord, type ResolvedHideChange, @@ -603,39 +610,6 @@ type UnstickJournalEntry = | { kind: 'attempt-done'; operationId: string; step: number; outcome: 'responded' | 'refused' | 'failed'; category?: string; error?: string }; type UnstickJournalSnapshot = Record; -type OperatorChangesEntry = - | { kind: 'attempt'; changeId: string; changeKind: OperatorChangeRecord['kind']; agent: string; n: number; at: number; evidence: OperatorChangeEvidence } - | { kind: 'switched'; changeId: string; n: number } - | { kind: 'failed'; changeId: string; n: number; error: string } - | { kind: 'outcome'; changeId: string; outcome: OperatorChangeOutcome } - | { kind: 'completed'; changeId: string } - | { kind: 'dropped'; changeId: string; changeKind: OperatorChangeRecord['kind']; agent: string; at: number }; -type OperatorChangesSnapshot = Record; - -/** Fold one operator/changes entry in. An outcome is immutable: the first - * recorded stands. */ -function reduceOperatorChangesEntry(records: Map, entry: OperatorChangesEntry): void { - if (entry.kind === 'attempt' || entry.kind === 'dropped') { - const record = records.get(entry.changeId) - ?? { changeId: entry.changeId, kind: entry.changeKind, agent: entry.agent, attempts: [] }; - if (entry.kind === 'attempt') { - if (!record.attempts.some((a) => a.n === entry.n)) record.attempts.push({ n: entry.n, at: entry.at, evidence: entry.evidence }); - } else { - record.dropped ??= { at: entry.at }; - } - records.set(entry.changeId, record); - return; - } - const record = records.get(entry.changeId); - if (!record) return; - if (entry.kind === 'switched') record.switched = entry.n; - else if (entry.kind === 'failed') { - const attempt = record.attempts.find((a) => a.n === entry.n); - if (attempt) attempt.failed = entry.error; - } else if (entry.kind === 'outcome') record.outcome ??= entry.outcome; - else record.completed = true; -} - function reduceUnstickEntry(ops: Map, entry: UnstickJournalEntry): void { const op = ops.get(entry.operationId) ?? { operationId: entry.operationId, agent: 'agent' in entry ? entry.agent : '', steps: [], attempts: [] }; @@ -721,6 +695,17 @@ export class OperatorJournalUnreadableError extends Error { } } +/** Startup could not bring a gated operator change's body to a known + * state (restoring the source of a failed attempt), so the framework does + * not start serving it. */ +export class OperatorChangeRecoveryError extends Error { + constructor(changeId: string, step: string, cause: unknown) { + const detail = cause instanceof Error ? cause.message : String(cause); + super(`Operator change ${changeId}: ${step} failed at startup: ${detail}`, { cause }); + this.name = 'OperatorChangeRecoveryError'; + } +} + class DiscordAwarenessAccountingError extends Error { constructor(operation: string, cause: unknown) { const detail = cause instanceof Error ? cause.message : String(cause); @@ -1674,6 +1659,9 @@ export class AgentFramework { private activeAdmissions: Map = new Map(); /** This store's identity (storeIdentity), once read or minted. */ private storeIdentityValue: string | null = null; + /** What this process saw of each cut attempt (change id -> attempt -> + * result). It binds even when the record of it couldn't be written. */ + private cutKnowledge: Map> = new Map(); /** The operator/changes journal, loaded on first use (changesLedger). */ private changesJournalState: { journal: RecordJournal; @@ -1910,6 +1898,12 @@ export class AgentFramework { } } + // Before any agent initializes: an abandoned cut that left its destination + // active gets its source back, so a body that can't initialize never + // stands in the way of its own recovery. Also the first read of the + // operator journals: an unreadable one stops startup here. + framework.restoreAbandonedCutSources(); + // Restore persisted usage data (if any) from prior session framework.restoreUsageState(); @@ -7181,7 +7175,7 @@ export class AgentFramework { if (!this.changesJournalState) { // Loaded before it is installed: a journal that can't be read is never // mistaken for one with no operations. - const journal = new RecordJournal(this.store, { type: 'operator/changes' }); + const journal = new RecordJournal(this.store, { type: OPERATOR_CHANGES_JOURNAL }); this.changesJournalState = { journal, records: this.loadChangesJournal(journal) }; } return this.changesJournalState; @@ -7241,7 +7235,81 @@ export class AgentFramework { * this after a restart. */ getOperatorChangeRecord(changeId: string): OperatorChangeRecord | null { const record = this.changesLedger().records.get(changeId); - return record ? structuredClone(record) : null; + if (!record) return null; + const copy = structuredClone(record); + const unresolved = this.unresolvedAttempt(record); + if (unresolved) copy.unresolved = unresolved; + return copy; + } + + /** A cut whose latest attempt has neither its switch nor its failure + * recorded, and wasn't seen by this process: whether it applied is + * unknown, whatever its destination's state. */ + private unresolvedAttempt(record: OperatorChangeRecord): { n: number; target: string } | null { + if (record.outcome || record.dropped || (record.kind !== 'undo-turns' && record.kind !== 'undo-messages')) return null; + const last = record.attempts[record.attempts.length - 1]; + if (!last || cutAttemptDisposition(record, last, this.cutKnowledge.get(record.changeId)?.get(last.n)) !== 'unresolved') return null; + return { n: last.n, target: last.evidence.target }; + } + + /** + * Settle a held cut with an operator's verdict, under the held lease. The + * verdict is recorded as the operator's attestation (who, why, which + * attempt), distinct from anything the framework observed. + * - `not-committed` abandons the attempt and restores the source if its + * destination is the active body. A fresh attempt, under the same staging + * authorization, is the host's decision on its next retry. + * - `committed` attests that the attempt applied. Its outcome is + * established, and its staged marks activated, on the host's next retry + * (or at the next start), never by this call. + * The receipt names what it did and what remains. + */ + async resolveOperatorChange( + changeId: string, + attempt: number, + verdict: 'committed' | 'not-committed', + opts: { lease: SafeBoundaryLease; reason: string; requester?: OperatorRequester }, + ): Promise { + this.heldLeaseTokens(opts.lease, `resolve operator change ${changeId}`); + if (!opts.reason?.trim()) throw new OperatorActionError('invalid', 'Settling an operator change needs a reason'); + const record = this.changesLedger().records.get(changeId); + const unresolved = record ? this.unresolvedAttempt(record) : null; + if (!record || !unresolved) throw new OperatorActionError('invalid', `Operator change ${changeId} has no unresolved attempt to settle`); + if (unresolved.n !== attempt) { + throw new OperatorActionError('stale', `Operator change ${changeId}: attempt ${attempt} is not its unresolved attempt (${unresolved.n} is)`); + } + const resolution: OperatorChangeResolution = { + verdict, reason: opts.reason, ...(opts.requester ? { requester: opts.requester } : {}), at: Date.now(), via: 'live', + }; + this.journalChange({ kind: 'resolved', changeId, n: attempt, resolution }); + const target = record.attempts.find((a) => a.n === attempt)!.evidence; + let restored: string | undefined; + let remaining: string | undefined; + if (verdict === 'not-committed' && this.store.currentBranch().name === target.target) { + const agent = this.agents.get(record.agent); + if (!agent) throw new OperatorActionError('unknown-agent', `Unknown agent: ${record.agent}`); + restored = await this.restoreSourceBranch(agent.getContextManager(), target.source, target.target, record.agent); + this.materializeConfigMountAfterBranchSwitch(); + if (this.store.currentBranch().name === target.target) remaining = `the source ${target.source} could not be restored: ${restored}`; + } + const receipt: OperatorChangeResolutionReceipt = { + changeId, + attempt, + recorded: resolution, + ...(restored ? { restored } : {}), + settlement: verdict === 'committed' + ? 'its outcome is established, and its staged marks activated, on the host\'s next retry or the next start' + : 'abandoned; a fresh attempt is the host\'s decision on its next retry', + ...(remaining ? { remaining } : {}), + }; + this.recordOperatorAction({ + kind: 'resolve-operator-change', + agent: record.agent, + ...(opts.requester ? { requester: opts.requester } : {}), + params: { changeId, attempt, verdict, reason: opts.reason, via: 'live' }, + result: { ...(restored ? { restored } : {}), settlement: receipt.settlement, ...(remaining ? { remaining } : {}) }, + }); + return receipt; } /** @@ -7703,6 +7771,11 @@ export class AgentFramework { if (toIdx < 0) throw new OperatorActionError('unknown-message', `Message ${toMessageId} is not an addressable message in context.`); [lo, hi] = fromIdx <= toIdx ? [fromIdx, toIdx] : [toIdx, fromIdx]; } + // Whole body groups: chronicle never removes part of one, so the frozen + // plan names every shard the hide affects. + const cm = agent.getContextManager(); + lo = bodyGroupRun(cm, lo).from; + hi = bodyGroupRun(cm, hi).to; const range = all.slice(lo, hi + 1); return { id: randomUUID(), @@ -7765,13 +7838,6 @@ export class AgentFramework { this.recordOperatorAction(entry); } - /** A cut's destination exists and was written past its branch point: the - * cut was applied there, whatever is active now. */ - private destinationUsed(destination: string): boolean { - const dest = this.store.listBranches().find((b) => b.name === destination); - return !!dest && dest.head !== (dest.branchPoint ?? dest.head); - } - /** * Establish a committed body change's outcome from its attempt's evidence: * activate its staged publication choice with exactly what the body @@ -7829,16 +7895,22 @@ export class AgentFramework { */ private async applyCut( change: ResolvedUndoTurnsChange | ResolvedUndoMessagesChange, - destination: string, + base: string, steps: { revalidate: (stale: (why: string) => never) => void; removal: () => Array<{ metadata?: unknown }>; - cut: () => Promise; - /** Put the source back if the cut left the destination active. */ - restore: () => Promise; - finish: (outcome: OperatorChangeOutcome) => void; + /** Create `destination` (it never exists yet) and switch to it. */ + cut: (destination: string) => Promise; + /** Put the source back if the cut left `destination` active. */ + restore: (destination: string) => Promise; + finish: (outcome: OperatorChangeOutcome, target: string) => void; }, - ): Promise<{ outcome: OperatorChangeOutcome; alreadyApplied: boolean }> { + ): Promise<{ outcome: OperatorChangeOutcome; alreadyApplied: boolean; target: string }> { + // Each attempt cuts onto a branch of its own, created by it and never + // reused: `base` for the first, `base~n` after. A branch already bearing + // an attempt's name was made by someone else, so it is refused. + const destinationFor = (n: number) => (n === 1 ? base : `${base}~${n}`); + const exists = (name: string) => this.store.listBranches().some((b) => b.name === name); const label = change.kind === 'undo-turns' ? `Undo of ${change.requestedTurns} turn(s)` : `Undo of ${change.requestedMessages} message(s)`; @@ -7848,40 +7920,70 @@ export class AgentFramework { if (!this.agents.has(change.agent)) throw new OperatorActionError('unknown-agent', `Unknown agent: ${change.agent}`); if (change.storeId !== this.storeIdentity()) stale('it was resolved against another store'); const record = () => this.changesLedger().records.get(change.id); + const targetOf = (outcome: OperatorChangeOutcome) => record()!.attempts.find((a) => a.n === outcome.n)!.evidence.target; const settle = (outcome: OperatorChangeOutcome, alreadyApplied: boolean) => { + const target = targetOf(outcome); if (!record()?.completed) { - steps.finish(outcome); + steps.finish(outcome, target); this.journalChange({ kind: 'completed', changeId: change.id }); } - return { outcome, alreadyApplied }; + return { outcome, alreadyApplied, target }; }; const known = record(); if (known?.outcome) return settle(known.outcome, true); let current = this.store.currentBranch().name; const last = known?.attempts[known.attempts.length - 1]; - if (last && last.failed !== undefined) { - // That attempt failed, and its disposition was recorded before its - // source was restored: never a commitment, whatever its destination - // holds. If a crash came before the restore, finish it now. - if (current === destination) { - await steps.restore(); - current = this.store.currentBranch().name; - } - } else if (last) { - if (known!.switched === last.n || current === destination || this.destinationUsed(destination)) { - if (known!.switched !== last.n) this.journalChange({ kind: 'switched', changeId: change.id, n: last.n }); + if (last) { + // Only a record, or what this process itself saw, decides an attempt. + const target = last.evidence.target; + const disposition = cutAttemptDisposition(known!, last, this.cutKnowledge.get(change.id)?.get(last.n)); + if (disposition === 'committed') { + // A switch this process saw but couldn't record is recorded now; an + // operator's attestation stands as its own record. + if (known!.switched !== last.n && last.resolution?.verdict !== 'committed') { + this.journalChange({ kind: 'switched', changeId: change.id, n: last.n }); + } return settle(this.establishOutcome(change, last, 'undo'), true); } - } else if (current === destination || this.destinationUsed(destination)) { + if (disposition === 'not-committed') { + // Never a commitment, whatever its destination holds. If the + // process died before restoring the source, restore it now. + if (current === target) { + await steps.restore(target); + current = this.store.currentBranch().name; + } + } else { + // Neither its switch nor its failure was recorded, and this process + // didn't see it: whether it applied is unknown, whatever its + // destination's state now (a branch can be restored from or deleted + // after an unrecorded switch). Held, never certified and never cut + // again, until an operator settles it. + throw new OperatorActionError( + 'unresolved', + `${label} ${change.id}: attempt ${last.n} has neither a recorded switch nor a recorded failure, ` + + `so whether it applied (to ${target}) is unknown; settle it with resolveOperatorChange`, + ); + } + } + if (current !== change.sourceBranch) stale(`the active branch moved from ${change.sourceBranch} to ${current}`); + const n = (last?.n ?? 0) + 1; + const destination = destinationFor(n); + if (exists(destination)) { throw new OperatorActionError( - 'failed', - `${label} ${change.id}: its destination ${destination} exists but no attempt was recorded, so its outcome is unknown`, + 'invalid', + `${label} ${change.id}: a branch named ${destination} already exists, and no attempt of this change made it; nothing was cut`, ); } - if (current !== change.sourceBranch) stale(`the active branch moved from ${change.sourceBranch} to ${current}`); steps.revalidate(stale); const removed = steps.removal(); - const n = (last?.n ?? 0) + 1; + const note = (result: 'committed' | 'failed') => { + const attempts = this.cutKnowledge.get(change.id) ?? new Map(); + attempts.set(n, result); + this.cutKnowledge.set(change.id, attempts); + }; + // If writing the attempt throws, it may still have landed; either way + // this process knows its cut never ran. + note('failed'); this.journalChange({ kind: 'attempt', changeId: change.id, @@ -7891,6 +7993,7 @@ export class AgentFramework { at: Date.now(), evidence: { target: destination, + source: change.sourceBranch, sourceHead: this.store.currentBranch().head, removed: removed.length, marks: this.removalMarks(change, removed), @@ -7899,8 +8002,9 @@ export class AgentFramework { }, }); try { - await steps.cut(); + await steps.cut(destination); } catch (error) { + note('failed'); const detail = error instanceof Error ? error.message : String(error); // Recorded before the source is restored, so recovery never reads a // destination this attempt touched as its cut. @@ -7910,9 +8014,10 @@ export class AgentFramework { } catch (journalError) { recorded = `; its failure could not be recorded (${journalError instanceof Error ? journalError.message : String(journalError)})`; } - const restored = await steps.restore(); + const restored = await steps.restore(destination); throw new OperatorActionError('failed', `${label} ${change.id} failed; ${restored}${recorded}: ${detail}`, { cause: error }); } + note('committed'); this.journalChange({ kind: 'switched', changeId: change.id, n }); return settle(this.establishOutcome(change, record()!.attempts.find((a) => a.n === n)!, 'undo'), false); } @@ -7931,9 +8036,8 @@ export class AgentFramework { alreadyApplied?: true; markers: SurgeryMarkerReceipt; }> { - const destination = this.undoTurnsDestination(change); const oldest = change.checkpoints[change.checkpoints.length - 1]!; - const { outcome, alreadyApplied } = await this.applyCut(change, destination, { + const { outcome, alreadyApplied, target } = await this.applyCut(change, this.undoTurnsDestination(change), { revalidate: (stale) => { const list = this.getTurnCheckpoints(change.agent); for (const c of change.checkpoints) { @@ -7946,20 +8050,18 @@ export class AgentFramework { // the cut removes, read within the lease that holds the switch. removal: () => this.agents.get(change.agent)!.getContextManager().getAllMessages() .filter((m) => Number(m.sequence) > oldest.sequenceBefore), - cut: async () => { - if (!this.store.listBranches().some((b) => b.name === destination)) { - this.store.createBranchAt(destination, change.sourceBranch, oldest.sequenceBefore); - } + cut: async (destination) => { + this.store.createBranchAt(destination, change.sourceBranch, oldest.sequenceBefore); this.store.switchBranch(destination); this.materializeConfigMountAfterBranchSwitch(); }, - restore: async () => { + restore: async (destination) => { if (this.store.currentBranch().name !== destination) return `active branch is ${this.store.currentBranch().name}`; this.store.switchBranch(change.sourceBranch); this.materializeConfigMountAfterBranchSwitch(); return `active branch restored to ${change.sourceBranch}`; }, - finish: (established) => { + finish: (established, destination) => { // The one redo entry back to the source tip, while the destination is // active and no turn has run on it (new work invalidates redo). if (this.store.currentBranch().name === destination @@ -7998,7 +8100,7 @@ export class AgentFramework { requested: change.requestedTurns, undone: change.checkpoints.length, fromBranch: change.sourceBranch, - toBranch: destination, + toBranch: target, ...(alreadyApplied ? { alreadyApplied: true as const } : {}), markers: outcome.markers, }; @@ -8018,24 +8120,22 @@ export class AgentFramework { alreadyApplied?: true; markers: SurgeryMarkerReceipt; }> { - const destination = this.undoMessagesDestination(change); const agent = this.agents.get(change.agent); if (!agent) throw new OperatorActionError('unknown-agent', `Unknown agent: ${change.agent}`); const cm = agent.getContextManager(); - const { outcome, alreadyApplied } = await this.applyCut(change, destination, { + const { outcome, alreadyApplied, target } = await this.applyCut(change, this.undoMessagesDestination(change), { revalidate: (stale) => { const tail = cm.getAllMessages().find((m) => String(m.id) === change.tail.id); if (!tail) stale(`its tail message ${change.tail.id} is gone`); if (this.messageFingerprint(tail!) !== change.tail.fingerprint) stale(`its tail message ${change.tail.id} changed since it was staged`); }, removal: () => this.planRollback(change.agent, change.tail.id).discarded, - cut: async () => { - const exists = this.store.listBranches().some((b) => b.name === destination); - await cm.switchBranch(exists ? destination : cm.branchAt(change.tail.id as MessageId, destination)); + cut: async (destination) => { + await cm.switchBranch(cm.branchAt(change.tail.id as MessageId, destination)); this.materializeConfigMountAfterBranchSwitch(); }, - restore: () => this.restoreSourceBranch(cm, change.sourceBranch, destination, change.agent), - finish: (established) => { + restore: (destination) => this.restoreSourceBranch(cm, change.sourceBranch, destination, change.agent), + finish: (established, destination) => { this.recordChangeAction({ kind: 'rollback', agent: change.agent, @@ -8055,7 +8155,7 @@ export class AgentFramework { requested: change.requestedMessages, messagesRemoved: outcome.removed, fromBranch: change.sourceBranch, - toBranch: destination, + toBranch: target, ...(alreadyApplied ? { alreadyApplied: true as const } : {}), markers: outcome.markers, }; @@ -8135,6 +8235,7 @@ export class AgentFramework { at: Date.now(), evidence: { target: change.sourceBranch, + source: change.sourceBranch, sourceHead: this.store.currentBranch().head, ids: present.map((p) => ({ id: p.id, fingerprint: p.fingerprint })), removed: present.length, @@ -8147,8 +8248,8 @@ export class AgentFramework { return settle(this.establishOutcome(change, record()!.attempts[0]!, 'hide'), present.length === 0); } - /** Remove those of `ids` that remain, each only while unchanged: one range - * removal when they are contiguous, else one by one. Returns how many. */ + /** Remove those of `ids` that remain, each only while unchanged, as range + * removals of their contiguous runs. Returns how many. */ private removeHideMessages( cm: ContextManager, ids: Array<{ id: string; fingerprint: string }>, @@ -8161,17 +8262,47 @@ export class AgentFramework { if (this.messageFingerprint(byId.get(p.id)!) !== p.fingerprint) stale(`message ${p.id} changed since it was staged`); } if (present.length === 0) return 0; - const order = all.map((m) => String(m.id)); - const first = order.indexOf(present[0]!.id); - const lastIndex = order.indexOf(present[present.length - 1]!.id); - if (present.length > 1 && lastIndex - first + 1 === present.length) { - cm.removeMessages(present[0]!.id as MessageId, present[present.length - 1]!.id as MessageId); - } else { - for (const p of present) cm.removeMessage(p.id as MessageId); + // Each contiguous run goes as one range removal, a single message + // included: chronicle removes a body group only whole and only by + // range, even a one-shard group. + const order = new Map(all.map((m, i) => [String(m.id), i])); + const sorted = [...present].sort((a, b) => order.get(a.id)! - order.get(b.id)!); + let runStart = 0; + for (let i = 1; i <= sorted.length; i++) { + if (i === sorted.length || order.get(sorted[i]!.id)! !== order.get(sorted[i - 1]!.id)! + 1) { + cm.removeMessages(sorted[runStart]!.id as MessageId, sorted[i - 1]!.id as MessageId); + runStart = i; + } } return present.length; } + /** + * Before any agent initializes on it: when an abandoned cut (its attempt + * failed, or an operator settled it as not committed) left its destination + * as the active branch, switch the store back to the source and sync it. + */ + private restoreAbandonedCutSources(): void { + this.unstickLedger(); + const current = this.store.currentBranch().name; + for (const record of this.changesLedger().records.values()) { + if (record.outcome || record.dropped || (record.kind !== 'undo-turns' && record.kind !== 'undo-messages')) continue; + const last = record.attempts[record.attempts.length - 1]; + if (!last || current !== last.evidence.target || cutAttemptDisposition(record, last) !== 'not-committed') continue; + try { + this.store.switchBranch(last.evidence.source); + this.store.sync(); + } catch (error) { + throw new OperatorChangeRecoveryError(record.changeId, `restoring the source of abandoned attempt ${last.n}`, error); + } + console.error( + `[operator-changes] ${record.kind} ${record.changeId}: abandoned attempt ${last.n} had left ${last.evidence.target} active; ` + + `restored ${last.evidence.source} before any agent initializes`, + ); + return; // one active branch + } + } + /** * At startup, establish the outcome of every gated body change that * committed before a crash recorded it: a cut whose switch was recorded or @@ -8181,16 +8312,16 @@ export class AgentFramework { * not provably committed is left for the host's retry. */ private async reconcileOperatorChanges(): Promise { - // Both operator journals are read before anything else runs: either may - // hold a body change that still needs recovery, so an unreadable one - // stops startup (OperatorJournalUnreadableError) rather than reading as - // empty. - this.unstickLedger(); + // Both journals were first read before the agents (an unreadable one + // stopped startup there, rather than reading as empty). const records = [...this.changesLedger().records.values()]; for (const record of records) { if (record.outcome || record.dropped || record.attempts.length === 0) continue; const last = record.attempts[record.attempts.length - 1]!; - if (last.failed !== undefined) continue; // never committed: the host's retry starts a fresh attempt + const disposition = record.kind === 'hide' ? null : cutAttemptDisposition(record, last); + // Abandoned: its source was restored before the agents initialized + // (restoreAbandonedCutSources); a fresh attempt is the host's retry. + if (disposition === 'not-committed') continue; try { const current = this.store.currentBranch().name; if (record.kind === 'hide') { @@ -8199,11 +8330,20 @@ export class AgentFramework { this.removeHideMessages(agent.getContextManager(), last.evidence.ids ?? [], (why) => { throw new OperatorActionError('stale', why); }); - } else { - const destination = last.evidence.target; - if (!(record.switched === last.n || current === destination || this.destinationUsed(destination))) continue; - if (record.switched !== last.n) this.journalChange({ kind: 'switched', changeId: record.changeId, n: last.n }); + } else if (disposition === 'unresolved') { + const target = last.evidence.target; + if (current === target) { + // Unresolved, and its destination is the active body: nobody can + // vouch for it, so no turn starts until an operator settles it. + this.quiesced = true; + this.quiesceReason = `operator change ${record.changeId} is unresolved: its attempt ${last.n} left ${target} active with neither its switch nor its failure recorded`; + this.quiescedAt = Date.now(); + this.eventGate?.setQuiesced(true); + this.opsAlert('operator-change-unresolved', record.agent, `${this.quiesceReason}; booting quiesced until it is settled (resolveOperatorChange) and the host resumes`); + } + continue; // held until resolved; anything that isn't the active body waits for the host } + // Committed (its switch, or an operator's attestation): its outcome is established below. const outcome = this.establishOutcome( { id: record.changeId, agent: record.agent, kind: record.kind }, last, diff --git a/src/index.ts b/src/index.ts index 81477f241..944862085 100644 --- a/src/index.ts +++ b/src/index.ts @@ -4,7 +4,7 @@ export * from './types/index.js'; // Core classes export { AgentFramework, BudgetPreflightError, ResumeBlockedError } from './framework.js'; export type { RuntimeSettingsPreview, HostModeStatus, SafeBoundaryLease, SafeBoundaryOptions } from './framework.js'; -export { OperatorJournalUnreadableError } from './framework.js'; +export { OperatorJournalUnreadableError, OperatorChangeRecoveryError } from './framework.js'; export type { OperatorChangeReceipt, ResolvedOperatorChange, diff --git a/src/operator-change.ts b/src/operator-change.ts index cad4a3de8..e11f45a1a 100644 --- a/src/operator-change.ts +++ b/src/operator-change.ts @@ -160,8 +160,11 @@ export type ResolvedOperatorChange = * from exactly this, never by rereading a branch later. */ export interface OperatorChangeEvidence { /** The branch the body lands on: a cut's destination, or the hidden - * messages' own branch. */ + * messages' own branch. A cut's destination is created by its attempt + * alone (a name that already exists is refused before it begins). */ target: string; + /** The branch the change was resolved on. */ + source: string; /** The source branch's head when the attempt began. */ sourceHead: number; /** A hide's exact messages, with their fingerprints. */ @@ -192,9 +195,16 @@ export interface OperatorChangeRecord { kind: ResolvedOperatorChange['kind']; agent: string; /** Application attempts, in order, each with its evidence. `failed` is set - * (with the error) when the attempt failed and restored its source: such - * an attempt is never read as committed. */ - attempts: Array<{ n: number; at: number; evidence: OperatorChangeEvidence; failed?: string }>; + * (with the error) when the framework saw the attempt fail. `resolution` + * is an operator's attestation about an attempt whose result was never + * recorded: kept as such, never made to look like a framework record. */ + attempts: Array<{ + n: number; + at: number; + evidence: OperatorChangeEvidence; + failed?: string; + resolution?: OperatorChangeResolution; + }>; /** The attempt whose cut switched to its destination: recorded right * after the switch, so it proves the cut whatever is active later. */ switched?: number; @@ -203,6 +213,11 @@ export interface OperatorChangeRecord { completed?: true; /** The host dropped it before it applied; its staged choice was discarded. */ dropped?: { at: number }; + /** Read-time only, never journaled: the latest attempt has neither its + * switch nor its failure recorded (and this process didn't see it), so + * whether it applied is unknown, whatever its destination's state. It is + * held until resolveOperatorChange records an operator's verdict. */ + unresolved?: { n: number; target: string }; } /** One step of an unstick operation, as journaled: `intent` before its shed @@ -306,3 +321,87 @@ function stableJson(value: unknown): string { const keys = Object.keys(value as Record).sort(); return `{${keys.map((k) => `${JSON.stringify(k)}:${stableJson((value as Record)[k])}`).join(',')}}`; } + + +/** An operator's verdict on one attempt whose result was never recorded: + * `committed` attests that it applied; `not-committed` that it did not, + * abandoning it. Neither authorizes new work: a fresh attempt is the host's + * decision on its next retry. */ +export interface OperatorChangeResolution { + verdict: 'committed' | 'not-committed'; + reason: string; + requester?: OperatorRequester; + at: number; + /** Given through a running framework under its lease, or with the host + * stopped through agent-framework-recover. */ + via: 'live' | 'offline'; +} + +/** An operator/changes journal entry. */ +export type OperatorChangesEntry = + | { kind: 'attempt'; changeId: string; changeKind: OperatorChangeRecord['kind']; agent: string; n: number; at: number; evidence: OperatorChangeEvidence } + | { kind: 'switched'; changeId: string; n: number } + | { kind: 'failed'; changeId: string; n: number; error: string } + | { kind: 'resolved'; changeId: string; n: number; resolution: OperatorChangeResolution } + | { kind: 'outcome'; changeId: string; outcome: OperatorChangeOutcome } + | { kind: 'completed'; changeId: string } + | { kind: 'dropped'; changeId: string; changeKind: OperatorChangeRecord['kind']; agent: string; at: number }; +export type OperatorChangesSnapshot = Record; +export const OPERATOR_CHANGES_JOURNAL = 'operator/changes'; + +/** Fold one operator/changes entry in. An outcome, and an attempt's + * resolution, are immutable: the first recorded stands. */ +export function reduceOperatorChangesEntry(records: Map, entry: OperatorChangesEntry): void { + if (entry.kind === 'attempt' || entry.kind === 'dropped') { + const record = records.get(entry.changeId) + ?? { changeId: entry.changeId, kind: entry.changeKind, agent: entry.agent, attempts: [] }; + if (entry.kind === 'attempt') { + if (!record.attempts.some((a) => a.n === entry.n)) record.attempts.push({ n: entry.n, at: entry.at, evidence: entry.evidence }); + } else { + record.dropped ??= { at: entry.at }; + } + records.set(entry.changeId, record); + return; + } + const record = records.get(entry.changeId); + if (!record) return; + if (entry.kind === 'switched') record.switched = entry.n; + else if (entry.kind === 'failed' || entry.kind === 'resolved') { + const attempt = record.attempts.find((a) => a.n === entry.n); + if (attempt && entry.kind === 'failed') attempt.failed ??= entry.error; + if (attempt && entry.kind === 'resolved') attempt.resolution ??= entry.resolution; + } else if (entry.kind === 'outcome') record.outcome ??= entry.outcome; + else record.completed = true; +} + +/** + * What decides a cut attempt: a framework record (its switch, its failure), + * an operator's resolution, or what the running process itself saw (`seen`). + * Nothing else, so missing evidence is never read as success: an attempt + * with none of these is `unresolved`, whatever its destination's state. + */ +export function cutAttemptDisposition( + record: OperatorChangeRecord, + attempt: OperatorChangeRecord['attempts'][number], + seen?: 'committed' | 'failed', +): 'committed' | 'not-committed' | 'unresolved' { + if (record.switched === attempt.n || attempt.resolution?.verdict === 'committed' || seen === 'committed') return 'committed'; + if (attempt.failed !== undefined || attempt.resolution?.verdict === 'not-committed' || seen === 'failed') return 'not-committed'; + return 'unresolved'; +} + +/** What resolveOperatorChange (or agent-framework-recover --operator-change + * resolve) did, step by step, and what remains. */ +export interface OperatorChangeResolutionReceipt { + changeId: string; + attempt: number; + /** The attestation as recorded. */ + recorded: OperatorChangeResolution; + /** The source restored, when a not-committed attempt's destination was the + * active body. */ + restored?: string; + /** When the change settles from here. */ + settlement: string; + /** Anything this didn't repair. */ + remaining?: string; +} diff --git a/src/operator-log.ts b/src/operator-log.ts index 41ab60b54..b22a8c1a0 100644 --- a/src/operator-log.ts +++ b/src/operator-log.ts @@ -183,7 +183,7 @@ export interface SurgeryMarksPreview { * lets surfaces distinguish "agent busy — quiesce first" from bad input. */ export class OperatorActionError extends Error { constructor( - readonly code: 'unknown-agent' | 'agent-busy' | 'unknown-message' | 'invalid' | 'failed' | 'stale', + readonly code: 'unknown-agent' | 'agent-busy' | 'unknown-message' | 'invalid' | 'failed' | 'stale' | 'unresolved', message: string, options?: { cause?: unknown }, ) { diff --git a/src/recovery/recover-cli.ts b/src/recovery/recover-cli.ts index 4940165f8..35de0b404 100644 --- a/src/recovery/recover-cli.ts +++ b/src/recovery/recover-cli.ts @@ -14,6 +14,17 @@ import { type OperatorLogInput, type OperatorRequester, } from '../operator-log.js'; +import { RecordJournal } from '../record-journal.js'; +import { + cutAttemptDisposition, + OPERATOR_CHANGES_JOURNAL, + reduceOperatorChangesEntry, + type OperatorChangeRecord, + type OperatorChangeResolution, + type OperatorChangeResolutionReceipt, + type OperatorChangesEntry, + type OperatorChangesSnapshot, +} from '../operator-change.js'; interface CliOptions { storePath?: string; @@ -30,6 +41,11 @@ interface CliOptions { emoji?: string; marks: 'none' | 'addressed' | 'all'; awareness?: { verb: 'list' | 'cancel' | 'retract' | 'release'; target?: string }; + operatorChange?: { verb: 'list' } | { verb: 'resolve'; changeId: string }; + attempt?: number; + verdict?: 'committed' | 'not-committed'; + reason?: string; + requester?: string; dryRun: boolean; } @@ -38,6 +54,9 @@ function usage(): string { agent-framework-recover --store --agent [options] agent-framework-recover --store --awareness (cancel takes a batch id or a retract request id) + agent-framework-recover --store --operator-change list + agent-framework-recover --store --operator-change resolve --attempt + --verdict --reason [--requester ] Creates and activates a Chronicle recovery branch while the normal host is down. The recovery is local to the resident: by default no Discord reaction is @@ -76,6 +95,21 @@ Awareness journal (host stopped): --awareness release Queue a held batch's recorded operations Each acts at once: inspect first with --awareness list. +Gated operator changes (host stopped): + --operator-change list Each held change: an attempt whose switch and + failure were never recorded, with its evidence + --operator-change resolve + Record your verdict on its unresolved attempt + (--attempt, --verdict, --reason required), as + your attestation, distinct from what the + framework observed. not-committed abandons the + attempt; if its destination is the active + branch, the next start restores the source + before any agent initializes. A fresh attempt + is the host's decision. committed attests that it applied: + the next start establishes its outcome and + activates its staged marks before any traffic. + Queued work is delivered when the host next connects to the Discord server. Each recovery, and each cancel, retract or release, is recorded in /operator-actions.jsonl, the host's operator log, with the OS account @@ -140,6 +174,22 @@ function parseArgs(args: string[]): CliOptions { } break; } + case '--operator-change': { + const verb = value(); + if (verb === 'list') options.operatorChange = { verb }; + else if (verb === 'resolve') options.operatorChange = { verb, changeId: value() }; + else throw new Error('--operator-change must be list or resolve '); + break; + } + case '--attempt': options.attempt = Number(value()); break; + case '--verdict': { + const verdict = value(); + if (verdict !== 'committed' && verdict !== 'not-committed') throw new Error('--verdict must be committed or not-committed'); + options.verdict = verdict; + break; + } + case '--reason': options.reason = value(); break; + case '--requester': options.requester = value(); break; case '--dry-run': options.dryRun = true; break; case '--help': case '-h': @@ -210,8 +260,91 @@ function awareness(options: CliOptions): void { } } +/** + * The operator/changes journal with the host stopped: list the held changes, + * or record an operator's verdict on one. This process is the store's only + * writer, which stands in for the running framework's lease. + */ +function operatorChange(options: CliOptions): void { + const store = JsStore.openOrCreate({ path: options.storePath! }); + try { + const journal = new RecordJournal(store, { type: OPERATOR_CHANGES_JOURNAL }); + const { snapshot, entries } = journal.load(); + const records = new Map( + Object.entries(snapshot ?? {}).map(([id, record]) => [id, structuredClone(record)]), + ); + for (const { entry } of entries) reduceOperatorChangesEntry(records, entry); + const active = store.currentBranch().name; + const held = (record: OperatorChangeRecord) => { + if (record.outcome || record.dropped || (record.kind !== 'undo-turns' && record.kind !== 'undo-messages')) return null; + const last = record.attempts[record.attempts.length - 1]; + return last && cutAttemptDisposition(record, last) === 'unresolved' ? last : null; + }; + const command = options.operatorChange!; + if (command.verb === 'list') { + const unresolved = [...records.values()].flatMap((record) => { + const attempt = held(record); + if (!attempt) return []; + const { target, source, removed, marks } = attempt.evidence; + return [{ + changeId: record.changeId, + kind: record.kind, + agent: record.agent, + attempt: attempt.n, + source, + target, + targetIsActive: active === target, + targetExists: store.listBranches().some((b) => b.name === target), + removed, + marks: { scope: marks.scope, refs: marks.refs.length, unmarked: marks.unmarked, notRemoved: marks.notRemoved }, + }]; + }); + console.log(JSON.stringify({ store: options.storePath, activeBranch: active, unresolved }, null, 2)); + return; + } + const record = records.get(command.changeId); + const attempt = record ? held(record) : null; + if (!record || !attempt) throw new Error(`Operator change ${command.changeId} has no unresolved attempt to settle`); + if (options.attempt !== attempt.n) throw new Error(`--attempt ${options.attempt} is not its unresolved attempt (${attempt.n} is)`); + if (!options.verdict) throw new Error('--verdict committed|not-committed is required'); + if (!options.reason?.trim()) throw new Error('--reason is required'); + const resolution: OperatorChangeResolution = { + verdict: options.verdict, + reason: options.reason, + requester: { via: 'agent-framework-recover', ...(options.requester ? { name: options.requester } : {}) }, + at: Date.now(), + via: 'offline', + }; + journal.append({ kind: 'resolved', changeId: record.changeId, n: attempt.n, resolution }, { durable: true }); + // Nothing in the store's body is touched here: a not-committed attempt + // whose destination is the active branch gets its source back at the + // next start, before any agent initializes on it. + const restoresAtStart = options.verdict === 'not-committed' && active === attempt.evidence.target; + const remaining = options.verdict === 'not-committed' && !restoresAtStart && active !== attempt.evidence.source + ? `the active branch is ${active}, neither the source ${attempt.evidence.source} nor the attempt's destination` + : undefined; + const receipt: OperatorChangeResolutionReceipt = { + changeId: record.changeId, + attempt: attempt.n, + recorded: resolution, + settlement: options.verdict === 'committed' + ? 'at the next start: startup establishes its outcome and activates its staged marks before any traffic' + : `abandoned${restoresAtStart ? `; the next start restores ${attempt.evidence.source} before any agent initializes` : ''}; a fresh attempt is the host's decision on its next retry`, + ...(remaining ? { remaining } : {}), + }; + console.log(JSON.stringify(receipt, null, 2)); + } finally { + store.close(); + } +} + async function main(): Promise { const options = parseArgs(process.argv.slice(2)); + if (options.operatorChange) { + if (!options.storePath) throw new Error(`--operator-change needs --store\n\n${usage()}`); + operatorChange(options); + return; + } if (options.awareness) { if (!options.storePath) { throw new Error(`--awareness needs --store\n\n${usage()}`); diff --git a/test/operator-change-gate.test.ts b/test/operator-change-gate.test.ts index 75b973191..c59e6410b 100644 --- a/test/operator-change-gate.test.ts +++ b/test/operator-change-gate.test.ts @@ -532,18 +532,28 @@ describe('operator-change gate: host/command undo by turns', () => { assert.ok(framework.getOperatorLog().some((e) => e.kind === 'undo-turns' && (e.params as { admission?: string }).admission === 'rev-u')); }); - it('finishes a cut an earlier attempt created but never switched to', async () => { + it('starts a fresh attempt on a branch of its own after one that created its destination but never switched', async () => { await turn('one'); await turn('two'); await undo(1); const change = asked[0]!; - // As if a crash came between creating the destination and switching to it. - const store = framework.getStore(); - const oldest = change.kind === 'undo-turns' ? change.checkpoints.at(-1)! : null; - store.createBranchAt(`undo/scout/op-${change.id}`, change.sourceBranch, oldest!.sequenceBefore); - const applied = await framework.runAtSafeBoundary({ verb: 'retry' }, (lease) => - framework.applyResolvedOperatorChange(change, { lease, admission: { id: 'rev-u' } })); + // As if the process died between creating the destination and switching + // to it: the switch fails, and so does recording the failure. + const store = framework.getStore() as unknown as { switchBranch: (name: string) => unknown }; + const realSwitch = store.switchBranch.bind(store); + store.switchBranch = () => { throw new Error('the process died before switching'); }; + const restore = failJournalOnce('failed'); + try { + await assert.rejects(applyIt(change), /died before switching/); + } finally { + store.switchBranch = realSwitch; + restore(); + } + const record = framework.getOperatorChangeRecord(change.id)!; + assert.deepEqual([record.attempts.length, record.attempts[0]!.failed, record.switched], [1, undefined, undefined], 'an attempt with no disposition'); + assert.ok(framework.getStore().listBranches().some((x) => x.name === `undo/scout/op-${change.id}`), 'its branch was created'); + const applied = await applyIt(change); assert.equal(applied.kind === 'undo-turns' && applied.alreadyApplied, undefined, 'applied now'); - assert.equal(branch(), `undo/scout/op-${change.id}`); + assert.equal(branch(), `undo/scout/op-${change.id}~2`, 'on its own branch, the unused one left alone'); assert.deepEqual(texts(), ['one', 'reply to one', 'two']); }); @@ -632,7 +642,9 @@ describe('operator-change gate: host/command undo by turns', () => { if (later) await turn('after the failure'); const retried = await applyIt(change); assert.equal(retried.kind === 'undo-turns' && retried.alreadyApplied === true, committed, 'a committed cut is recovered, never cut again'); - assert.equal(branch(), destination); + // A fresh attempt cuts onto a branch of its own; one that never got as + // far as creating its branch reuses the first name. + assert.equal(branch(), seam === 'cut' ? `${destination}~2` : destination); assert.equal(framework.getOperatorLog().filter((e) => e.kind === 'undo-turns').length, 1, 'logged once'); const record = framework.getOperatorChangeRecord(change.id)!; assert.equal(record.attempts.length, seam === 'cut' ? 2 : 1, seam === 'cut' ? 'the unapplied attempt, then a fresh one' : 'one attempt'); @@ -1023,6 +1035,224 @@ describe('operator-change gate: host/command undo by turns', () => { }); } + it('hides a one-shard body group as a range, and a shard of a larger group as its whole group', async () => { + type ShardStore = { messageStore: { append: (p: string, c: unknown[], m?: unknown, cb?: unknown, extra?: unknown) => { id: string } } }; + const shards = (group: string, texts: string[], messageId: string) => texts.map((text, shardIndex) => + (framework.getAgent('scout')!.getContextManager() as unknown as ShardStore).messageStore + .append('Member', [{ type: 'text', text }], { serverId: 'discord', channelId: 'discord:g1:c1', messageId }, undefined, { bodyGroupId: group, shardIndex }).id); + await say('before'); + shards('g1', ['lone'], 'lone'); + shards('g3', ['big 0', 'big 1', 'big 2'], 'big'); + for (const [fromMessageId, expected] of [['lone', 1], ['big', 3]] as const) { + await host().handleHostCommand('discord', { command: 'hide', agentName: 'scout', fromMessageId, toMessageId: fromMessageId, marks: 'none', requesterName: 'nissa' }); + const change = asked.at(-1)!; + assert.equal(change.kind === 'hide' && change.messages.length, expected, 'the frozen plan names the whole group'); + const applied = await applyIt(change); + assert.equal(applied.kind === 'hide' && applied.hidden, expected); + } + assert.deepEqual(texts().filter((t) => /lone|big/.test(t)), []); + }); + + it("refuses a branch already bearing its destination's name, made by no attempt of the change", async () => { + await turn('one'); await turn('two'); + await undo(1); + const change = asked[0]!; + framework.getStore().createBranch(`undo/scout/op-${change.id}`); // someone else's branch, at the current tip + const before = texts(); + await assert.rejects(applyIt(change), (e: Error & { code?: string }) => e.code === 'invalid' && /already exists, and no attempt/.test(e.message)); + assert.deepEqual(texts(), before, 'nothing cut'); + assert.equal(branch(), change.sourceBranch); + assert.equal(framework.getOperatorChangeRecord(change.id), null, 'nothing recorded either'); + }); + + const reopen = async () => { + await framework.stop(); + framework = await AgentFramework.create({ + storePath: join(tempDir, 'test.chronicle'), + membrane: membrane.asMembrane(), + agents: [{ name: 'scout', model: 'test-model', systemPrompt: 'You are scout.', maxTokens: 1000 }], + modules: [], + operatorChangeGate: async (c) => { asked.push(structuredClone(c)); return decide(c); }, + }); + }; + const quiesced = () => (framework as unknown as { quiesced: boolean }).quiesced; + const failCutAfterInitializerWrite = () => { + const cmx = framework.getAgent('scout')!.getContextManager() as unknown as { switchBranch: (name: string) => Promise }; + const real = cmx.switchBranch.bind(cmx); + let armed = true; + cmx.switchBranch = async (name) => { + if (!armed) return real(name); + armed = false; + framework.getStore().switchBranch(name); + framework.getStore().setStateJson('probe/initializer', { wrote: true }); + throw new Error('injected strategy initialization failure'); + }; + return () => { cmx.switchBranch = real; }; + }; + const resolve = (changeId: string, attempt: number, verdict: 'committed' | 'not-committed', reason: string) => + framework.runAtSafeBoundary({ verb: 'resolve' }, (lease) => + framework.resolveOperatorChange(changeId, attempt, verdict, { lease, reason, requester: { via: 'test', name: 'nissa' } })); + + it('holds an attempt whose failure record was lost once the process restarts, and settles it as not committed', async () => { + await say('m0'); await say('m1'); + await host().handleHostCommand('discord', { command: 'undo', agentName: 'scout', messages: 2, marks: 'all', requesterName: 'nissa' }); + const change = asked[0]!; + const undoCut = failCutAfterInitializerWrite(); + const undoRecord = failJournalOnce('failed'); + try { + await assert.rejects(applyIt(change), /injected strategy initialization failure/); + } finally { + undoCut(); undoRecord(); + } + await reopen(); // what this process saw is gone + assert.equal(branch(), change.sourceBranch); + assert.equal(quiesced(), false, 'the original body is active, so traffic is safe'); + assert.deepEqual(framework.getOperatorChangeRecord(change.id)!.unresolved, { n: 1, target: `undo-msgs/scout/op-${change.id}` }); + await assert.rejects(applyIt(change), (e: Error & { code?: string }) => e.code === 'unresolved', 'never certified, never cut again'); + await assert.rejects(resolve(change.id, 2, 'not-committed', 'wrong attempt'), (e: Error & { code?: string }) => e.code === 'stale'); + const settled = await resolve(change.id, 1, 'not-committed', 'its initializer failed; nothing applied'); + assert.deepEqual([settled.recorded.verdict, settled.recorded.via, settled.restored], ['not-committed', 'live', undefined]); + assert.match(settled.settlement, /abandoned/); + const applied = await applyIt(change); + assert.equal(applied.kind === 'undo-messages' && applied.toBranch, `undo-msgs/scout/op-${change.id}~2`, 'a fresh attempt, on its own branch'); + assert.deepEqual(texts(), ['m0', 'reply to m0']); + assert.equal(framework.getOperatorChangeRecord(change.id)!.attempts[0]!.resolution!.reason, 'its initializer failed; nothing applied'); + }); + + it('boots quiesced while an unresolved attempt left its destination active, and settles it as committed without publishing by itself', async () => { + await say('a0'); await say('a1'); await say('a2'); + await undoWithMarks(2, 'all'); + const change = asked[0]!; + const undoRecord = failJournalOnce('switched'); + try { + await assert.rejects(applyIt(change), /injected switched failure/); + } finally { + undoRecord(); + } + await reopen(); + assert.equal(branch(), `undo/scout/op-${change.id}`); + assert.equal(quiesced(), true, 'no turn starts on a body nobody can vouch for'); + await assert.rejects(applyIt(change), (e: Error & { code?: string }) => e.code === 'unresolved'); + const staged = () => outbox().batches().find((x) => x.id === `op-${change.id}`)?.status; + assert.equal(staged(), 'staged', 'its marks wait'); + const settled = await resolve(change.id, 1, 'committed', 'the switch completed; its record was lost'); + assert.match(settled.settlement, /next retry or the next start/); + assert.equal(staged(), 'staged', 'the attestation publishes nothing'); + const applied = await applyIt(change); + assert.equal(applied.kind === 'undo-turns' && applied.alreadyApplied, true); + assert.deepEqual(receipt(applied), { status: 'queued', queued: 1, unmarked: 0, notRemoved: 0 }); + const record = framework.getOperatorChangeRecord(change.id)!; + assert.deepEqual([record.switched, record.attempts[0]!.resolution!.verdict], [undefined, 'committed'], 'an attestation, not a framework-observed switch'); + }); + + it("restores a recorded failure's source at startup, before any traffic, when the process died before restoring it", async () => { + await say('m0'); await say('m1'); + await host().handleHostCommand('discord', { command: 'undo', agentName: 'scout', messages: 2, marks: 'none', requesterName: 'nissa' }); + const change = asked[0]!; + const undoCut = failCutAfterInitializerWrite(); + const fw = framework as unknown as { restoreSourceBranch: (...args: unknown[]) => Promise }; + const realRestore = fw.restoreSourceBranch.bind(fw); + fw.restoreSourceBranch = async () => { throw new Error('the process died before restoring'); }; + try { + await assert.rejects(applyIt(change), /died before restoring/); + } finally { + undoCut(); fw.restoreSourceBranch = realRestore; + } + assert.equal(branch(), `undo-msgs/scout/op-${change.id}`, 'left on the failed destination'); + await reopen(); + assert.equal(branch(), change.sourceBranch, 'restored before traffic'); + assert.equal(quiesced(), false); + const applied = await applyIt(change); // the host's retry + assert.equal(applied.kind === 'undo-messages' && applied.toBranch, `undo-msgs/scout/op-${change.id}~2`); + }); + + it('settles a held attempt offline when the destination cannot even start, then starts normally', async () => { + await turn('one'); await turn('two'); await turn('three'); + await undo(2); + const change = asked[0]!; + const undoRecord = failJournalOnce('switched'); + try { + await assert.rejects(applyIt(change), /injected switched failure/); + } finally { + undoRecord(); + } + await framework.stop(); + const storePath = join(tempDir, 'test.chronicle'); + const index = fileURLToPath(new URL('../src/index.js', import.meta.url)); + const mockPath = fileURLToPath(new URL('./helpers/mock-membrane.js', import.meta.url)); + const cli = fileURLToPath(new URL('../src/recovery/recover-cli.js', import.meta.url)); + const script = join(tempDir, 'start.mjs'); + // A context strategy that can't initialize on the cut body (it needs a + // message only the source has): the agent itself fails to start there. + writeFileSync(script, ` + import { AgentFramework, AutobiographicalStrategy } from ${JSON.stringify(index)}; + import { MockMembrane } from ${JSON.stringify(mockPath)}; + console.log = () => {}; console.error = () => {}; + class Picky extends AutobiographicalStrategy { + async initialize(ctx) { + if (!ctx.messageStore.getAll().some((m) => m.content?.[0]?.text === 'three')) throw new Error('cannot initialize on this body'); + return super.initialize?.(ctx); + } + } + try { + const fw = await AgentFramework.create({ storePath: ${JSON.stringify(storePath)}, membrane: new MockMembrane().asMembrane(), + agents: [{ name: 'scout', model: 'test-model', systemPrompt: 'x', maxTokens: 1000, strategy: new Picky() }], modules: [], + operatorChangeGate: async () => ({ id: 'rev', text: 'staged' }) }); + process.stdout.write(JSON.stringify({ started: true, branch: fw.getStore().currentBranch().name, quiesced: fw.quiesced })); + await fw.stop(); + } catch (error) { + process.stdout.write(JSON.stringify({ started: false, error: error.message })); + } + `); + const start = () => JSON.parse(spawnSync(process.execPath, [script], { encoding: 'utf8' }).stdout) as { started: boolean; branch?: string; quiesced?: boolean; error?: string }; + const recover = (...args: string[]) => { + const run = spawnSync(process.execPath, [cli, '--store', storePath, '--operator-change', ...args], { encoding: 'utf8' }); + assert.equal(run.status, 0, run.stderr); + return JSON.parse(run.stdout); + }; + const failed = start(); + assert.equal(failed.started, false, 'normal startup cannot finish'); + assert.match(String(failed.error), /cannot initialize on this body/); + const listed = recover('list'); + assert.deepEqual(listed.unresolved.map((u: { changeId: string; attempt: number; targetIsActive: boolean }) => [u.changeId, u.attempt, u.targetIsActive]), + [[change.id, 1, true]], 'inspected without starting the host'); + const settled = recover('resolve', change.id, '--attempt', '1', '--verdict', 'not-committed', '--reason', 'the cut body cannot start', '--requester', 'nissa'); + assert.deepEqual([settled.recorded.via, settled.recorded.verdict, settled.restored], ['offline', 'not-committed', undefined]); + assert.match(settled.settlement, new RegExp(`abandoned; the next start restores ${change.sourceBranch} before any agent initializes`)); + assert.deepEqual(start(), { started: true, branch: change.sourceBranch, quiesced: false }, 'normal readiness: restored before the agent initialized'); + await reopen().catch(() => {}); // the framework in this process was stopped above + }); + + it('establishes an offline committed verdict at the next start, as its receipt says', async () => { + await say('a0'); await say('a1'); await say('a2'); + await undoWithMarks(2, 'all'); + const change = asked[0]!; + const undoRecord = failJournalOnce('switched'); + try { + await assert.rejects(applyIt(change), /injected switched failure/); + } finally { + undoRecord(); + } + await framework.stop(); + const cli = fileURLToPath(new URL('../src/recovery/recover-cli.js', import.meta.url)); + const run = spawnSync(process.execPath, [cli, '--store', join(tempDir, 'test.chronicle'), '--operator-change', 'resolve', change.id, + '--attempt', '1', '--verdict', 'committed', '--reason', 'the switch completed'], { encoding: 'utf8' }); + assert.equal(run.status, 0, run.stderr); + const settled = JSON.parse(run.stdout); + assert.match(settled.settlement, /next start: startup establishes its outcome and activates its staged marks before any traffic/); + framework = await AgentFramework.create({ + storePath: join(tempDir, 'test.chronicle'), + membrane: membrane.asMembrane(), + agents: [{ name: 'scout', model: 'test-model', systemPrompt: 'You are scout.', maxTokens: 1000 }], + modules: [], + operatorChangeGate: async (c) => { asked.push(structuredClone(c)); return decide(c); }, + }); + const record = framework.getOperatorChangeRecord(change.id)!; + assert.deepEqual([record.outcome?.markers.status, record.outcome?.markers.queued], ['queued', 1], 'established at startup'); + assert.equal(outbox().batches().find((x) => x.id === `op-${change.id}`)?.status, 'active'); + assert.equal(quiesced(), false); + }); + it('refuses an undo by messages whose tail changed since staging', async () => { await say('m0'); await say('m1'); await host().handleHostCommand('discord', { command: 'undo', agentName: 'scout', messages: 2, marks: 'none', requesterName: 'nissa' }); From 231e3cde23627da694b7cc5ac0b88eb66a69e20c Mon Sep 17 00:00:00 2001 From: Agnes-1009 Date: Wed, 7 Oct 2026 18:09:33 -0400 Subject: [PATCH 50/73] fix(framework): an operator journal that parses but can't be read as its ledger is refused like unreadable bytes (review) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit June's remaining fail-closed gap at 2420545 (room-222 #49291). A JSON-valid checkpoint with a null snapshot (`{through: , snapshot: null}`) looked like no checkpoint at all to RecordJournal.load(), which returned `snapshot: null` in both cases while skipping the entries the checkpoint covered. The consumer's `snapshot ?? {}` then read a recorded pending attempt as empty history: startup succeeded, getOperatorChangeRecord returned null, and the framework reported ready. - **RecordJournal.load() reports `checkpointed`** (JournalLoad). A null snapshot from a found checkpoint is distinguishable from no checkpoint. This is additive. - **readOperatorChanges(load)**, pure and shared by the framework and the offline CLI, is the owner boundary. If a checkpoint exists, its snapshot must be a record map. Each record must have: - changeId matching its key, a known kind, and an agent; - attempts, each with n and at, full evidence (target, source, sourceHead, removed, staged, marks facts, hide ids), and a valid failed or resolution; - a well-formed switched, outcome, completed and dropped where present. Every entry must carry what its kind requires. Anything else throws "malformed operator/changes …". - **readUnstickJournal(load)** applies the same rule to operator/unstick: a checkpoint's snapshot must be an operation map with well-formed steps and attempts, and each entry must fit its kind. - **Both readers feed the existing load-first paths**, so a malformed ledger stops startup with OperatorJournalUnreadableError and fails every use mid-run, exactly as unreadable bytes do. The offline `agent-framework-recover --operator-change` refuses it too. Tests: - A pending attempt covered by a null-snapshot checkpoint: startup refused. - An attempt entry missing its fields: refused. - A null-snapshot checkpoint in the unstick journal: refused. - The offline CLI exits 1 with "malformed operator/changes checkpoint". - RecordJournal's own suite asserts `checkpointed` on its loads, plus a null-payload checkpoint reported as a checkpoint with its covered entries skipped. Mutants caught: a null checkpoint read as empty, entries left unvalidated, and an unstick null checkpoint read as empty. 24 suites: 420/420. Co-Authored-By: Agnes (Claude Opus 5.5) --- src/framework.ts | 46 ++++++++++--- src/operator-change.ts | 105 ++++++++++++++++++++++++++++++ src/record-journal.ts | 6 +- src/recovery/recover-cli.ts | 9 +-- test/operator-change-gate.test.ts | 46 +++++++++++++ test/record-journal.test.ts | 16 ++++- 6 files changed, 210 insertions(+), 18 deletions(-) diff --git a/src/framework.ts b/src/framework.ts index 360fd09a5..4b95d9388 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -21,6 +21,7 @@ import { type FrozenMarks, cutAttemptDisposition, OPERATOR_CHANGES_JOURNAL, + readOperatorChanges, reduceOperatorChangesEntry, type OperatorChangesEntry, type OperatorChangesSnapshot, @@ -610,6 +611,41 @@ type UnstickJournalEntry = | { kind: 'attempt-done'; operationId: string; step: number; outcome: 'responded' | 'refused' | 'failed'; category?: string; error?: string }; type UnstickJournalSnapshot = Record; +/** + * Read the unstick journal into its operations, refusing anything parseable + * that can't be interpreted as this ledger (a null-snapshot checkpoint, or + * a record or entry missing what it requires), exactly as unreadable bytes. + */ +function readUnstickJournal(load: { snapshot: unknown; entries: Array<{ entry: unknown }>; checkpointed: boolean }): Map { + const malformed = (what: string): never => { throw new Error(`malformed operator/unstick ${what}`); }; + const isObject = (v: unknown): v is Record => typeof v === 'object' && v !== null && !Array.isArray(v); + const isCount = (v: unknown): v is number => typeof v === 'number' && Number.isFinite(v); + const ops = new Map(); + if (load.checkpointed) { + const snapshot = load.snapshot; + if (!isObject(snapshot)) throw new Error('malformed operator/unstick checkpoint: its snapshot is not an operation map'); + for (const [id, op] of Object.entries(snapshot)) { + if (!isObject(op) || op.operationId !== id || typeof op.agent !== 'string' || !Array.isArray(op.steps) || !Array.isArray(op.attempts) + || !op.steps.every((x) => isObject(x) && isCount(x.step) && typeof x.status === 'string' && Array.isArray(x.messageIds)) + || !op.attempts.every((x) => isObject(x) && isCount(x.step) && typeof x.status === 'string')) { + malformed(`operation ${id}`); + } + ops.set(id, structuredClone(op as unknown as UnstickOperationRecord)); + } + } + for (const { entry } of load.entries) { + const e = entry as Record; + const ok = isObject(e) && typeof e.operationId === 'string' && e.operationId.length > 0 && isCount(e.step) && ( + (e.kind === 'step-intent' && typeof e.agent === 'string' && Array.isArray(e.messageIds)) + || e.kind === 'step-done' + || (e.kind === 'attempt-launched' && typeof e.agent === 'string') + || (e.kind === 'attempt-done' && ['responded', 'refused', 'failed'].includes(e.outcome as string))); + if (!ok) malformed(`entry ${isObject(e) ? String(e.kind) : typeof e}`); + reduceUnstickEntry(ops, entry as UnstickJournalEntry); + } + return ops; +} + function reduceUnstickEntry(ops: Map, entry: UnstickJournalEntry): void { const op = ops.get(entry.operationId) ?? { operationId: entry.operationId, agent: 'agent' in entry ? entry.agent : '', steps: [], attempts: [] }; @@ -7118,10 +7154,7 @@ export class AgentFramework { private loadUnstickJournal(journal: RecordJournal): Map { try { - const { snapshot, entries } = journal.load(); - const ops = new Map(Object.entries(snapshot ?? {}).map(([id, op]) => [id, structuredClone(op)])); - for (const { entry } of entries) reduceUnstickEntry(ops, entry); - return ops; + return readUnstickJournal(journal.load()); } catch (error) { throw new OperatorJournalUnreadableError('operator/unstick', error); } @@ -7183,10 +7216,7 @@ export class AgentFramework { private loadChangesJournal(journal: RecordJournal): Map { try { - const { snapshot, entries } = journal.load(); - const records = new Map(Object.entries(snapshot ?? {}).map(([id, record]) => [id, structuredClone(record)])); - for (const { entry } of entries) reduceOperatorChangesEntry(records, entry); - return records; + return readOperatorChanges(journal.load()); } catch (error) { throw new OperatorJournalUnreadableError('operator/changes', error); } diff --git a/src/operator-change.ts b/src/operator-change.ts index e11f45a1a..29571817a 100644 --- a/src/operator-change.ts +++ b/src/operator-change.ts @@ -405,3 +405,108 @@ export interface OperatorChangeResolutionReceipt { /** Anything this didn't repair. */ remaining?: string; } + +const CHANGE_KINDS = new Set(['agent-settings', 'tool-presentation', 'undo-turns', 'unstick', 'hide', 'undo-messages']); +const isObject = (v: unknown): v is Record => typeof v === 'object' && v !== null && !Array.isArray(v); +const isText = (v: unknown): v is string => typeof v === 'string' && v.length > 0; +const isCount = (v: unknown): v is number => typeof v === 'number' && Number.isFinite(v); + +function malformed(what: string): never { + throw new Error(`malformed operator/changes ${what}`); +} + +function checkEvidence(e: unknown, where: string): void { + if (!isObject(e)) malformed(`${where}: evidence`); + if (!isText(e.target) || !isText(e.source) || !isCount(e.sourceHead) || !isCount(e.removed) || typeof e.staged !== 'boolean') { + malformed(`${where}: evidence fields`); + } + const marks = e.marks; + if (!isObject(marks) || !['none', 'addressed', 'all'].includes(marks.scope as string) || !Array.isArray(marks.refs) + || !isCount(marks.unmarked) || !isCount(marks.notRemoved)) malformed(`${where}: marks facts`); + if (e.ids !== undefined && (!Array.isArray(e.ids) || !e.ids.every((i) => isObject(i) && isText(i.id) && isText(i.fingerprint)))) { + malformed(`${where}: hide ids`); + } +} + +function checkResolution(r: unknown, where: string): void { + if (!isObject(r) || (r.verdict !== 'committed' && r.verdict !== 'not-committed') || !isText(r.reason) || !isCount(r.at) + || (r.via !== 'live' && r.via !== 'offline')) malformed(`${where}: resolution`); +} + +function checkOutcome(o: unknown, where: string): void { + if (!isObject(o) || !isCount(o.n) || !isCount(o.at) || !isCount(o.removed) || !isObject(o.markers) || !isText(o.markers.status)) { + malformed(`${where}: outcome`); + } +} + +function checkRecord(r: unknown, id: string): asserts r is OperatorChangeRecord { + const where = `record ${id}`; + if (!isObject(r) || r.changeId !== id || !CHANGE_KINDS.has(r.kind as string) || !isText(r.agent) || !Array.isArray(r.attempts)) { + malformed(where); + } + for (const a of r.attempts) { + if (!isObject(a) || !isCount(a.n) || !isCount(a.at)) malformed(`${where}: attempt`); + checkEvidence(a.evidence, `${where} attempt ${a.n}`); + if (a.failed !== undefined && typeof a.failed !== 'string') malformed(`${where} attempt ${a.n}: failed`); + if (a.resolution !== undefined) checkResolution(a.resolution, `${where} attempt ${a.n}`); + } + if (r.switched !== undefined && !isCount(r.switched)) malformed(`${where}: switched`); + if (r.outcome !== undefined) checkOutcome(r.outcome, where); + if (r.completed !== undefined && r.completed !== true) malformed(`${where}: completed`); + if (r.dropped !== undefined && !(isObject(r.dropped) && isCount(r.dropped.at))) malformed(`${where}: dropped`); +} + +function checkEntry(e: unknown): asserts e is OperatorChangesEntry { + if (!isObject(e) || !isText(e.changeId)) malformed('entry'); + const where = `${String(e.kind)} entry for ${e.changeId}`; + switch (e.kind) { + case 'attempt': + if (!CHANGE_KINDS.has(e.changeKind as string) || !isText(e.agent) || !isCount(e.n) || !isCount(e.at)) malformed(where); + checkEvidence(e.evidence, where); + return; + case 'switched': + if (!isCount(e.n)) malformed(where); + return; + case 'failed': + if (!isCount(e.n) || typeof e.error !== 'string') malformed(where); + return; + case 'resolved': + if (!isCount(e.n)) malformed(where); + checkResolution(e.resolution, where); + return; + case 'outcome': + checkOutcome(e.outcome, where); + return; + case 'completed': + return; + case 'dropped': + if (!CHANGE_KINDS.has(e.changeKind as string) || !isText(e.agent) || !isCount(e.at)) malformed(where); + return; + default: + malformed(where); + } +} + +/** + * Read an operator/changes journal into its records. Anything parseable that + * can't be interpreted as this ledger is refused, exactly as unreadable + * bytes are: a checkpoint whose snapshot isn't a record map (a null one + * included, since it still covers the entries it skips), or a record or + * entry missing what its kind requires. + */ +export function readOperatorChanges(load: { snapshot: unknown; entries: Array<{ entry: unknown }>; checkpointed: boolean }): Map { + const records = new Map(); + if (load.checkpointed) { + const snapshot = load.snapshot; + if (!isObject(snapshot)) throw new Error('malformed operator/changes checkpoint: its snapshot is not a record map'); + for (const [id, record] of Object.entries(snapshot)) { + checkRecord(record, id); + records.set(id, structuredClone(record)); + } + } + for (const { entry } of load.entries) { + checkEntry(entry); + reduceOperatorChangesEntry(records, entry); + } + return records; +} diff --git a/src/record-journal.ts b/src/record-journal.ts index 0565e5422..b6cca4488 100644 --- a/src/record-journal.ts +++ b/src/record-journal.ts @@ -61,6 +61,10 @@ export interface JournalEntry extends JournalRecordRef { export interface JournalLoad { /** The latest checkpoint's snapshot, or null when none was written. */ snapshot: Snapshot | null; + /** Whether a checkpoint was found. A checkpoint whose snapshot is null + * still covers (and skips) its entries, so a consumer must not read a + * null snapshot with `checkpointed` as an empty history. */ + checkpointed: boolean; /** Every entry appended after that checkpoint, in append order. */ entries: Array>; } @@ -153,7 +157,7 @@ export class RecordJournal { this.lastEntryId = tail.length > 0 ? tail[tail.length - 1]!.value : through; this.sinceCheckpoint = entries.length; this.unreconciled = null; - return { snapshot, entries }; + return { snapshot, entries, checkpointed: latestCheckpointId !== null }; } /** True after an ambiguous write, until load() has replayed it. */ diff --git a/src/recovery/recover-cli.ts b/src/recovery/recover-cli.ts index 35de0b404..e4ead9d9d 100644 --- a/src/recovery/recover-cli.ts +++ b/src/recovery/recover-cli.ts @@ -18,7 +18,7 @@ import { RecordJournal } from '../record-journal.js'; import { cutAttemptDisposition, OPERATOR_CHANGES_JOURNAL, - reduceOperatorChangesEntry, + readOperatorChanges, type OperatorChangeRecord, type OperatorChangeResolution, type OperatorChangeResolutionReceipt, @@ -269,11 +269,8 @@ function operatorChange(options: CliOptions): void { const store = JsStore.openOrCreate({ path: options.storePath! }); try { const journal = new RecordJournal(store, { type: OPERATOR_CHANGES_JOURNAL }); - const { snapshot, entries } = journal.load(); - const records = new Map( - Object.entries(snapshot ?? {}).map(([id, record]) => [id, structuredClone(record)]), - ); - for (const { entry } of entries) reduceOperatorChangesEntry(records, entry); + // A journal that can't be interpreted is refused, never read as empty. + const records = readOperatorChanges(journal.load()); const active = store.currentBranch().name; const held = (record: OperatorChangeRecord) => { if (record.outcome || record.dropped || (record.kind !== 'undo-turns' && record.kind !== 'undo-messages')) return null; diff --git a/test/operator-change-gate.test.ts b/test/operator-change-gate.test.ts index c59e6410b..fdb9135a0 100644 --- a/test/operator-change-gate.test.ts +++ b/test/operator-change-gate.test.ts @@ -1807,6 +1807,52 @@ describe('operator journals that cannot be read', () => { }); } + const evidence = { target: 'undo/scout/op-x', source: 'main', sourceHead: 1, removed: 1, staged: false, + marks: { scope: 'none', refs: [], unmarked: 0, notRemoved: 0 } }; + const startOn = (store: JsStore) => AgentFramework.create({ + store, + membrane: new MockMembrane().asMembrane(), + agents: [{ name: 'scout', model: 'test-model', systemPrompt: 'You are scout.', maxTokens: 1000 }], + modules: [], + }); + const refusedAs = (type: string) => (e: Error) => e.name === 'OperatorJournalUnreadableError' && e.message.includes(type) && /malformed/.test(e.message); + + it('refuses a checkpoint whose snapshot is null rather than reading the entries it covers as gone', async () => { + const store = JsStore.openOrCreate({ path: join(dir, 'store') }); + const pending = store.appendJson('operator/changes', { kind: 'attempt', changeId: 'pending', changeKind: 'undo-turns', agent: 'scout', n: 1, at: 1, evidence }); + store.appendJson('operator/changes/checkpoint', { through: String(pending.id), snapshot: null }); + store.sync(); + await assert.rejects(startOn(store), refusedAs('operator/changes')); + }); + + it('refuses an entry missing what its kind requires', async () => { + const store = JsStore.openOrCreate({ path: join(dir, 'store') }); + store.appendJson('operator/changes', { kind: 'attempt', changeId: 'half' }); + store.sync(); + await assert.rejects(startOn(store), refusedAs('operator/changes')); + }); + + it('refuses a null-snapshot checkpoint in the unstick journal too', async () => { + const store = JsStore.openOrCreate({ path: join(dir, 'store') }); + const launched = store.appendJson('operator/unstick', { kind: 'attempt-launched', operationId: 'op', agent: 'scout', step: 1 }); + store.appendJson('operator/unstick/checkpoint', { through: String(launched.id), snapshot: null }); + store.sync(); + await assert.rejects(startOn(store), refusedAs('operator/unstick')); + }); + + it('refuses a malformed ledger offline as well', () => { + const path = join(dir, 'offline'); + const store = JsStore.openOrCreate({ path }); + const pending = store.appendJson('operator/changes', { kind: 'attempt', changeId: 'pending', changeKind: 'undo-turns', agent: 'scout', n: 1, at: 1, evidence }); + store.appendJson('operator/changes/checkpoint', { through: String(pending.id), snapshot: null }); + store.sync(); + store.close(); + const cli = fileURLToPath(new URL('../src/recovery/recover-cli.js', import.meta.url)); + const run = spawnSync(process.execPath, [cli, '--store', path, '--operator-change', 'list'], { encoding: 'utf8' }); + assert.equal(run.status, 1); + assert.match(run.stderr, /malformed operator\/changes checkpoint/); + }); + it('fails an operation mid-run when operator/changes becomes unreadable, never answering as if it were empty', async () => { const framework = await AgentFramework.create({ storePath: join(dir, 'live'), diff --git a/test/record-journal.test.ts b/test/record-journal.test.ts index 3618ae077..dcc030614 100644 --- a/test/record-journal.test.ts +++ b/test/record-journal.test.ts @@ -33,7 +33,7 @@ describe('RecordJournal', () => { it('an empty journal loads nothing', () => { withStore((store) => { const journal = new RecordJournal(store, { type: 'journal-test/entry' }); - assert.deepStrictEqual(journal.load(), { snapshot: null, entries: [] }); + assert.deepStrictEqual(journal.load(), { snapshot: null, entries: [], checkpointed: false }); assert.strictEqual(journal.entriesSinceCheckpoint, 0); }); }); @@ -105,7 +105,7 @@ describe('RecordJournal', () => { // A checkpoint after load covers what was loaded; the newest one wins. reader.checkpoint({ seen: ['a', 'b', 'c'] }); const second = new RecordJournal(store, { type: 'journal-test/entry' }).load(); - assert.deepStrictEqual(second, { snapshot: { seen: ['a', 'b', 'c'] }, entries: [] }); + assert.deepStrictEqual(second, { snapshot: { seen: ['a', 'b', 'c'] }, entries: [], checkpointed: true }); assert.deepStrictEqual(entriesOf(new RecordJournal(store, { type: 'journal-test/other' })), ['x', 'y']); }); }); @@ -153,6 +153,16 @@ describe('RecordJournal', () => { }); }); + it('reports a checkpoint whose snapshot is null as a checkpoint, distinct from none', () => { + withStore((store) => { + const journal = new RecordJournal(store, { type: 'journal-test/entry' }); + journal.append({ n: 'a' }); + journal.checkpoint(null as unknown); + const loaded = new RecordJournal(store, { type: 'journal-test/entry' }).load(); + assert.deepStrictEqual(loaded, { snapshot: null, entries: [], checkpointed: true }, 'its covered entries are skipped: a reader must not take this for empty'); + }); + }); + it('a write that fails after reaching the store blocks further writes until load() replays it', () => { withStore((store) => { let failNextSync = false; @@ -181,7 +191,7 @@ describe('RecordJournal', () => { journal.append({ n: 'B' }); journal.checkpoint({ seen: ['A', 'B'] }); assert.deepStrictEqual(new RecordJournal(store, { type: 'journal-test/entry' }).load(), - { snapshot: { seen: ['A', 'B'] }, entries: [] }); + { snapshot: { seen: ['A', 'B'] }, entries: [], checkpointed: true }); // A failure BEFORE the record reaches the store wrote nothing and leaves the journal usable. failNextSync = true; From d19a9d3a9b7136e3cc83ea835a3af5f11b812e1d Mon Sep 17 00:00:00 2001 From: Agnes-1009 Date: Wed, 7 Oct 2026 18:34:32 -0400 Subject: [PATCH 51/73] fix(recover): an offline operator-change resolution is logged like the CLI's other acts, under the CLI's requester convention (integration with #250) #250's 25891f0 made agent-framework-recover record its acts in the store's operator log: awareness controls and recoveries, done or refused, with the requester `{ via: 'cli', name: }`. R2's `--operator-change resolve` predates that and recorded nothing there, so after the merge it was the one silent operator act in that CLI. The live resolver already logs `resolve-operator-change`. - **Resolve logs `resolve-operator-change`**, with: - agent: the change's, or `*` when the change is unknown; - params: changeId, attempt, verdict, reason, and `via: 'offline'`, matching the live entry's `via: 'live'`; - on success, the result's settlement and any remaining requirement; - on refusal, the error: no unresolved attempt, wrong attempt, or a missing verdict or reason. - **One requester convention, in the attestation and in the log.** `--requester ` gives `{ via: 'cli', name }`; otherwise the CLI's own `cliRequester()`. The attestation's `requester` was `{ via: 'agent-framework-recover', name? }`; the attestation's own `via: 'offline'` already records how it was given. - **`list`** is a read, logged no more than `--awareness list`. Test: the offline end-to-end case now refuses a wrong attempt (exit 1), logged with its error, and logs the resolution by `--requester nissa` with its settlement. operator-change-gate, recover-cli-operator-log and record-journal: 107/107. Co-Authored-By: Agnes (Claude Opus 5.5) --- src/recovery/recover-cli.ts | 80 ++++++++++++++++++++----------- test/operator-change-gate.test.ts | 12 +++++ 2 files changed, 65 insertions(+), 27 deletions(-) diff --git a/src/recovery/recover-cli.ts b/src/recovery/recover-cli.ts index e4ead9d9d..79ece64f7 100644 --- a/src/recovery/recover-cli.ts +++ b/src/recovery/recover-cli.ts @@ -301,34 +301,60 @@ function operatorChange(options: CliOptions): void { } const record = records.get(command.changeId); const attempt = record ? held(record) : null; - if (!record || !attempt) throw new Error(`Operator change ${command.changeId} has no unresolved attempt to settle`); - if (options.attempt !== attempt.n) throw new Error(`--attempt ${options.attempt} is not its unresolved attempt (${attempt.n} is)`); - if (!options.verdict) throw new Error('--verdict committed|not-committed is required'); - if (!options.reason?.trim()) throw new Error('--reason is required'); - const resolution: OperatorChangeResolution = { - verdict: options.verdict, - reason: options.reason, - requester: { via: 'agent-framework-recover', ...(options.requester ? { name: options.requester } : {}) }, - at: Date.now(), - via: 'offline', - }; - journal.append({ kind: 'resolved', changeId: record.changeId, n: attempt.n, resolution }, { durable: true }); - // Nothing in the store's body is touched here: a not-committed attempt - // whose destination is the active branch gets its source back at the - // next start, before any agent initializes on it. - const restoresAtStart = options.verdict === 'not-committed' && active === attempt.evidence.target; - const remaining = options.verdict === 'not-committed' && !restoresAtStart && active !== attempt.evidence.source - ? `the active branch is ${active}, neither the source ${attempt.evidence.source} nor the attempt's destination` - : undefined; - const receipt: OperatorChangeResolutionReceipt = { - changeId: record.changeId, - attempt: attempt.n, - recorded: resolution, - settlement: options.verdict === 'committed' - ? 'at the next start: startup establishes its outcome and activates its staged marks before any traffic' - : `abandoned${restoresAtStart ? `; the next start restores ${attempt.evidence.source} before any agent initializes` : ''}; a fresh attempt is the host's decision on its next retry`, - ...(remaining ? { remaining } : {}), + // Who settles it: --requester names them; otherwise the CLI's user, as + // this CLI's other acts record theirs. The same requester goes in the + // attestation and in the operator log. + const requester: OperatorRequester = options.requester ? { via: 'cli', name: options.requester } : cliRequester(); + const logBase: OperatorLogInput = { + kind: 'resolve-operator-change', + agent: record?.agent ?? '*', + requester, + params: { + changeId: command.changeId, + ...(options.attempt !== undefined ? { attempt: options.attempt } : {}), + ...(options.verdict ? { verdict: options.verdict } : {}), + ...(options.reason ? { reason: options.reason } : {}), + via: 'offline', + }, }; + let receipt: OperatorChangeResolutionReceipt; + try { + if (!record || !attempt) throw new Error(`Operator change ${command.changeId} has no unresolved attempt to settle`); + if (options.attempt !== attempt.n) throw new Error(`--attempt ${options.attempt} is not its unresolved attempt (${attempt.n} is)`); + if (!options.verdict) throw new Error('--verdict committed|not-committed is required'); + if (!options.reason?.trim()) throw new Error('--reason is required'); + const resolution: OperatorChangeResolution = { + verdict: options.verdict, + reason: options.reason, + requester, + at: Date.now(), + via: 'offline', + }; + journal.append({ kind: 'resolved', changeId: record.changeId, n: attempt.n, resolution }, { durable: true }); + // Nothing in the store's body is touched here: a not-committed attempt + // whose destination is the active branch gets its source back at the + // next start, before any agent initializes on it. + const restoresAtStart = options.verdict === 'not-committed' && active === attempt.evidence.target; + const remaining = options.verdict === 'not-committed' && !restoresAtStart && active !== attempt.evidence.source + ? `the active branch is ${active}, neither the source ${attempt.evidence.source} nor the attempt's destination` + : undefined; + receipt = { + changeId: record.changeId, + attempt: attempt.n, + recorded: resolution, + settlement: options.verdict === 'committed' + ? 'at the next start: startup establishes its outcome and activates its staged marks before any traffic' + : `abandoned${restoresAtStart ? `; the next start restores ${attempt.evidence.source} before any agent initializes` : ''}; a fresh attempt is the host's decision on its next retry`, + ...(remaining ? { remaining } : {}), + }; + } catch (error) { + recordOperatorAction(options.storePath!, { ...logBase, error: errorMessage(error) }); + throw error; + } + recordOperatorAction(options.storePath!, { + ...logBase, + result: { settlement: receipt.settlement, ...(receipt.remaining ? { remaining: receipt.remaining } : {}) }, + }); console.log(JSON.stringify(receipt, null, 2)); } finally { store.close(); diff --git a/test/operator-change-gate.test.ts b/test/operator-change-gate.test.ts index fdb9135a0..c869ecb91 100644 --- a/test/operator-change-gate.test.ts +++ b/test/operator-change-gate.test.ts @@ -1216,8 +1216,20 @@ describe('operator-change gate: host/command undo by turns', () => { const listed = recover('list'); assert.deepEqual(listed.unresolved.map((u: { changeId: string; attempt: number; targetIsActive: boolean }) => [u.changeId, u.attempt, u.targetIsActive]), [[change.id, 1, true]], 'inspected without starting the host'); + // A wrong attempt is refused, and the refusal is logged like the CLI's other acts. + const wrong = spawnSync(process.execPath, [cli, '--store', storePath, '--operator-change', 'resolve', change.id, + '--attempt', '2', '--verdict', 'not-committed', '--reason', 'wrong attempt'], { encoding: 'utf8' }); + assert.equal(wrong.status, 1); + assert.match(wrong.stderr, /--attempt 2 is not its unresolved attempt \(1 is\)/); const settled = recover('resolve', change.id, '--attempt', '1', '--verdict', 'not-committed', '--reason', 'the cut body cannot start', '--requester', 'nissa'); assert.deepEqual([settled.recorded.via, settled.recorded.verdict, settled.restored], ['offline', 'not-committed', undefined]); + assert.deepEqual(settled.recorded.requester, { via: 'cli', name: 'nissa' }); + const logged = readFileSync(join(storePath, 'operator-actions.jsonl'), 'utf8').trim().split('\n').map((line) => JSON.parse(line)) + .filter((entry: { kind: string }) => entry.kind === 'resolve-operator-change'); + assert.deepEqual(logged.map((entry: { agent: string; params: { attempt: number; via: string }; error?: string; result?: { settlement: string } }) => + [entry.agent, entry.params.attempt, entry.params.via, entry.error !== undefined, entry.result !== undefined]), + [['scout', 2, 'offline', true, false], ['scout', 1, 'offline', false, true]], 'the refusal and the resolution, each logged'); + assert.deepEqual(logged[1].requester, { via: 'cli', name: 'nissa' }); assert.match(settled.settlement, new RegExp(`abandoned; the next start restores ${change.sourceBranch} before any agent initializes`)); assert.deepEqual(start(), { started: true, branch: change.sourceBranch, quiesced: false }, 'normal readiness: restored before the agent initialized'); await reopen().catch(() => {}); // the framework in this process was stopped above From 5d1c5d87b46d462a96f5d1f635bba0347bd8c454 Mon Sep 17 00:00:00 2001 From: Agnes-1009 Date: Wed, 7 Oct 2026 19:03:57 -0400 Subject: [PATCH 52/73] fix(framework,recover): validate operator/changes by what each kind's recovery consumes; --dry-run never resolves an operator change (review) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit June's two remaining findings on the merged head d19a9d3 (room-222 #50095, #50098), and Kit's test-label correction (#50100). **#50095, kind-specific consumed evidence.** `checkEvidence` treated `ids` as optional for every kind, and hide recovery substitutes `ids ?? []`. A hide attempt persisted without its ids was accepted at startup: it removed nothing, then recorded `outcome.removed: 2` while all four raw messages remained (her probe, 236.log). readOperatorChanges now validates each record as its kind's recovery reads it: - **Hide attempts:** `ids` must be present, each with an id and a fingerprint. `removed` must equal their count, and `target` must equal `source`. Empty ids remain valid, since none of the staged messages may have remained. - **Cut attempts** (undo-turns, undo-messages) carry no ids. Only the cuts and hide record attempts, and only a cut has a switch or a resolution. - **Every marks ref** needs a serverId, channelId and messageId, as delivery dereferences them. - **An outcome's markers** must be a whole SurgeryMarkerReceipt for its status: facts, plus queued with a batchId, not-scheduled with an error, or unresolved with both. - **References between entries:** - a switch, failure or resolution must name an attempt the ledger holds; - an outcome or completion must name a change it holds; - an attempt or drop must match its change's kind; - attempt numbers must be unique; - an outcome must name an attempt of its own. - **After folding the entries in, every record is checked whole** (snapshot records and reduced ones alike), so the ledger is validated exactly as recovery will read it. **#50098, `--dry-run` never resolves.** `--operator-change` ran before #250's dry-run refusal and ignored the flag. Her dry run recorded an attestation, and the next start restored the source. It's now refused before the store opens, worded as the awareness refusal is: "--dry-run previews a recovery; it does not apply to --operator-change (inspect with --operator-change list)". **Kit #50100.** The test "treats a used destination as proof…" passes because this process recorded its own observed commit (cutKnowledge), so it's renamed: "establishes a cut this process saw complete when its switch record was lost, whatever the branches show since". Its comment now says branch state is no evidence either way. Tests: - A hide attempt without ids is refused, so June's probe is refused at startup. - A hide whose removed count differs from its ids is refused. - A marks ref missing its messageId is refused. - A queued receipt without a batchId is refused. - An outcome naming no attempt of its own is refused, and so is a switch for a change never attempted. - The offline end-to-end case: `resolve --dry-run` exits 1 with that wording, and `list` still shows the attempt unresolved, so nothing was recorded. Mutants caught, each by exactly its own test: hide ids optional, removal count unchecked, ref elements unchecked, receipt unchecked, outcome cross-reference unchecked, dry-run resolving. 27 suites (the earlier 24, plus recover-cli-operator-log and the two MCPL reconnect suites): 445/445. Co-Authored-By: Agnes (Claude Opus 5.5) --- src/operator-change.ts | 86 ++++++++++++++++++++++++------- src/recovery/recover-cli.ts | 5 ++ test/operator-change-gate.test.ts | 55 ++++++++++++++++++-- 3 files changed, 124 insertions(+), 22 deletions(-) diff --git a/src/operator-change.ts b/src/operator-change.ts index 29571817a..ebac6e1c1 100644 --- a/src/operator-change.ts +++ b/src/operator-change.ts @@ -407,6 +407,8 @@ export interface OperatorChangeResolutionReceipt { } const CHANGE_KINDS = new Set(['agent-settings', 'tool-presentation', 'undo-turns', 'unstick', 'hide', 'undo-messages']); +/** The kinds whose application is journaled as attempts: the cuts and hide. */ +const CUT_KINDS = new Set(['undo-turns', 'undo-messages']); const isObject = (v: unknown): v is Record => typeof v === 'object' && v !== null && !Array.isArray(v); const isText = (v: unknown): v is string => typeof v === 'string' && v.length > 0; const isCount = (v: unknown): v is number => typeof v === 'number' && Number.isFinite(v); @@ -415,7 +417,8 @@ function malformed(what: string): never { throw new Error(`malformed operator/changes ${what}`); } -function checkEvidence(e: unknown, where: string): void { +/** An attempt's evidence, checked as its kind's recovery consumes it. */ +function checkEvidence(e: unknown, where: string, kind: string): void { if (!isObject(e)) malformed(`${where}: evidence`); if (!isText(e.target) || !isText(e.source) || !isCount(e.sourceHead) || !isCount(e.removed) || typeof e.staged !== 'boolean') { malformed(`${where}: evidence fields`); @@ -423,8 +426,19 @@ function checkEvidence(e: unknown, where: string): void { const marks = e.marks; if (!isObject(marks) || !['none', 'addressed', 'all'].includes(marks.scope as string) || !Array.isArray(marks.refs) || !isCount(marks.unmarked) || !isCount(marks.notRemoved)) malformed(`${where}: marks facts`); - if (e.ids !== undefined && (!Array.isArray(e.ids) || !e.ids.every((i) => isObject(i) && isText(i.id) && isText(i.fingerprint)))) { - malformed(`${where}: hide ids`); + if (!marks.refs.every((r) => isObject(r) && isText(r.serverId) && isText(r.channelId) && isText(r.messageId))) { + malformed(`${where}: a marks ref`); + } + if (kind === 'hide') { + // Recovery removes exactly these: missing ids must never read as an + // empty removal. (Empty is valid when none of them remained to hide.) + if (!Array.isArray(e.ids) || !e.ids.every((i) => isObject(i) && isText(i.id) && isText(i.fingerprint))) { + malformed(`${where}: a hide's ids`); + } + if (e.removed !== e.ids.length) malformed(`${where}: a hide removes exactly its ids`); + if (e.target !== e.source) malformed(`${where}: a hide stays on its branch`); + } else if (e.ids !== undefined) { + malformed(`${where}: a cut records no hide ids`); } } @@ -433,54 +447,88 @@ function checkResolution(r: unknown, where: string): void { || (r.via !== 'live' && r.via !== 'offline')) malformed(`${where}: resolution`); } -function checkOutcome(o: unknown, where: string): void { - if (!isObject(o) || !isCount(o.n) || !isCount(o.at) || !isCount(o.removed) || !isObject(o.markers) || !isText(o.markers.status)) { - malformed(`${where}: outcome`); +/** The awareness receipt an outcome hands back, by status. */ +function checkMarkers(m: unknown, where: string): void { + if (!isObject(m) || !['none', 'addressed', 'all'].includes(m.scope as string) || !isCount(m.unmarked) || !isCount(m.notRemoved)) { + malformed(`${where}: markers facts`); } + const ok = m.status === 'none' ? m.queued === 0 + : m.status === 'queued' ? isCount(m.queued) && isText(m.batchId) + : m.status === 'not-scheduled' ? m.queued === 0 && typeof m.error === 'string' + : m.status === 'unresolved' ? m.queued === 0 && isText(m.batchId) && typeof m.error === 'string' + : false; + if (!ok) malformed(`${where}: markers receipt`); } +function checkOutcome(o: unknown, where: string): void { + if (!isObject(o) || !isCount(o.n) || !isCount(o.at) || !isCount(o.removed)) malformed(`${where}: outcome`); + checkMarkers(o.markers, `${where} outcome`); +} + +/** A whole record, as reduced: its fields, each attempt's evidence by the + * record's kind, and its references to its own attempts. */ function checkRecord(r: unknown, id: string): asserts r is OperatorChangeRecord { const where = `record ${id}`; if (!isObject(r) || r.changeId !== id || !CHANGE_KINDS.has(r.kind as string) || !isText(r.agent) || !Array.isArray(r.attempts)) { malformed(where); } + const kind = r.kind as string; + if (r.attempts.length > 0 && !CUT_KINDS.has(kind) && kind !== 'hide') malformed(`${where}: a ${kind} records no attempts`); + const ns = new Set(); for (const a of r.attempts) { - if (!isObject(a) || !isCount(a.n) || !isCount(a.at)) malformed(`${where}: attempt`); - checkEvidence(a.evidence, `${where} attempt ${a.n}`); + if (!isObject(a) || !isCount(a.n) || !isCount(a.at) || ns.has(a.n)) malformed(`${where}: attempt`); + ns.add(a.n); + checkEvidence(a.evidence, `${where} attempt ${a.n}`, kind); if (a.failed !== undefined && typeof a.failed !== 'string') malformed(`${where} attempt ${a.n}: failed`); - if (a.resolution !== undefined) checkResolution(a.resolution, `${where} attempt ${a.n}`); + if (a.resolution !== undefined) { + if (!CUT_KINDS.has(kind)) malformed(`${where} attempt ${a.n}: only a cut is resolved`); + checkResolution(a.resolution, `${where} attempt ${a.n}`); + } + } + if (r.switched !== undefined && !(CUT_KINDS.has(kind) && isCount(r.switched) && ns.has(r.switched))) malformed(`${where}: switched`); + if (r.outcome !== undefined) { + checkOutcome(r.outcome, where); + if ((r.attempts.length > 0 || CUT_KINDS.has(kind) || kind === 'hide') && !ns.has((r.outcome as { n: number }).n)) { + malformed(`${where}: its outcome names no attempt of its own`); + } } - if (r.switched !== undefined && !isCount(r.switched)) malformed(`${where}: switched`); - if (r.outcome !== undefined) checkOutcome(r.outcome, where); if (r.completed !== undefined && r.completed !== true) malformed(`${where}: completed`); if (r.dropped !== undefined && !(isObject(r.dropped) && isCount(r.dropped.at))) malformed(`${where}: dropped`); } -function checkEntry(e: unknown): asserts e is OperatorChangesEntry { +/** One entry, checked before it's folded in: its own shape, and that it + * refers to a change (and attempt) the ledger already holds. */ +function checkEntry(e: unknown, records: Map): asserts e is OperatorChangesEntry { if (!isObject(e) || !isText(e.changeId)) malformed('entry'); const where = `${String(e.kind)} entry for ${e.changeId}`; + const record = records.get(e.changeId); + const hasAttempt = (n: unknown) => !!record && record.attempts.some((a) => a.n === n); switch (e.kind) { case 'attempt': - if (!CHANGE_KINDS.has(e.changeKind as string) || !isText(e.agent) || !isCount(e.n) || !isCount(e.at)) malformed(where); - checkEvidence(e.evidence, where); + if (!(CUT_KINDS.has(e.changeKind as string) || e.changeKind === 'hide') || !isText(e.agent) || !isCount(e.n) || !isCount(e.at)) malformed(where); + if (record && record.kind !== e.changeKind) malformed(`${where}: its kind differs from its change's`); + checkEvidence(e.evidence, where, e.changeKind as string); return; case 'switched': - if (!isCount(e.n)) malformed(where); + if (!isCount(e.n) || !hasAttempt(e.n)) malformed(where); return; case 'failed': - if (!isCount(e.n) || typeof e.error !== 'string') malformed(where); + if (!isCount(e.n) || typeof e.error !== 'string' || !hasAttempt(e.n)) malformed(where); return; case 'resolved': - if (!isCount(e.n)) malformed(where); + if (!isCount(e.n) || !hasAttempt(e.n)) malformed(where); checkResolution(e.resolution, where); return; case 'outcome': + if (!record) malformed(`${where}: no such change`); checkOutcome(e.outcome, where); return; case 'completed': + if (!record) malformed(`${where}: no such change`); return; case 'dropped': if (!CHANGE_KINDS.has(e.changeKind as string) || !isText(e.agent) || !isCount(e.at)) malformed(where); + if (record && record.kind !== e.changeKind) malformed(`${where}: its kind differs from its change's`); return; default: malformed(where); @@ -505,8 +553,10 @@ export function readOperatorChanges(load: { snapshot: unknown; entries: Array<{ } } for (const { entry } of load.entries) { - checkEntry(entry); + checkEntry(entry, records); reduceOperatorChangesEntry(records, entry); } + // The ledger as recovery will read it: every record whole, by its kind. + for (const [id, record] of records) checkRecord(record, id); return records; } diff --git a/src/recovery/recover-cli.ts b/src/recovery/recover-cli.ts index 79ece64f7..b411ebb8d 100644 --- a/src/recovery/recover-cli.ts +++ b/src/recovery/recover-cli.ts @@ -365,6 +365,11 @@ async function main(): Promise { const options = parseArgs(process.argv.slice(2)); if (options.operatorChange) { if (!options.storePath) throw new Error(`--operator-change needs --store\n\n${usage()}`); + // A resolution is a decision, not a preview: refuse before opening the + // store rather than record one for a command that asked to change nothing. + if (options.dryRun) { + throw new Error('--dry-run previews a recovery; it does not apply to --operator-change (inspect with --operator-change list)'); + } operatorChange(options); return; } diff --git a/test/operator-change-gate.test.ts b/test/operator-change-gate.test.ts index c869ecb91..9a547a796 100644 --- a/test/operator-change-gate.test.ts +++ b/test/operator-change-gate.test.ts @@ -820,7 +820,7 @@ describe('operator-change gate: host/command undo by turns', () => { assert.equal(branch(), change.sourceBranch); }); - it('treats a used destination as proof of the cut when its switch record was lost, after the source is restored', async () => { + it('establishes a cut this process saw complete when its switch record was lost, whatever the branches show since', async () => { await turn('one'); await turn('two'); await turn('three'); await undo(2); const change = asked[0]!; @@ -830,15 +830,17 @@ describe('operator-change gate: host/command undo by turns', () => { } finally { restore(); } - await turn('on the destination'); // used - framework.getStore().switchBranch(change.sourceBranch); // left, and the source restored + // Branch state is no evidence either way: the destination used, then + // left, and the source restored. + await turn('on the destination'); + framework.getStore().switchBranch(change.sourceBranch); const branches = () => framework.getStore().listBranches().map((x) => x.name).sort(); const before = branches(); const retried = await applyIt(change); assert.equal(retried.kind === 'undo-turns' && retried.alreadyApplied, true, 'committed: never cut again'); assert.equal(branch(), change.sourceBranch); assert.deepEqual(branches(), before); - assert.equal(framework.getOperatorChangeRecord(change.id)!.switched, 1, 'its switch recorded now, from the evidence'); + assert.equal(framework.getOperatorChangeRecord(change.id)!.switched, 1, 'its switch recorded now, from what this process saw'); }); for (const target of ['all', 'by id'] as const) { @@ -1216,6 +1218,12 @@ describe('operator-change gate: host/command undo by turns', () => { const listed = recover('list'); assert.deepEqual(listed.unresolved.map((u: { changeId: string; attempt: number; targetIsActive: boolean }) => [u.changeId, u.attempt, u.targetIsActive]), [[change.id, 1, true]], 'inspected without starting the host'); + // A dry run is no preview of a decision: refused before the store opens, nothing recorded. + const dry = spawnSync(process.execPath, [cli, '--store', storePath, '--operator-change', 'resolve', change.id, + '--attempt', '1', '--verdict', 'not-committed', '--reason', 'only looking', '--dry-run'], { encoding: 'utf8' }); + assert.equal(dry.status, 1); + assert.match(dry.stderr, /does not apply to --operator-change \(inspect with --operator-change list\)/); + assert.deepEqual(recover('list').unresolved.map((u: { attempt: number }) => u.attempt), [1], 'still unresolved: nothing was recorded'); // A wrong attempt is refused, and the refusal is logged like the CLI's other acts. const wrong = spawnSync(process.execPath, [cli, '--store', storePath, '--operator-change', 'resolve', change.id, '--attempt', '2', '--verdict', 'not-committed', '--reason', 'wrong attempt'], { encoding: 'utf8' }); @@ -1844,6 +1852,45 @@ describe('operator journals that cannot be read', () => { await assert.rejects(startOn(store), refusedAs('operator/changes')); }); + const hideEvidence = { ...evidence, target: 'main', source: 'main', removed: 2, ids: [{ id: 'm1', fingerprint: 'f1' }, { id: 'm2', fingerprint: 'f2' }] }; + let ledgers = 0; + const refuses = async (entries: object[], why: RegExp) => { + const store = JsStore.openOrCreate({ path: join(dir, `ledger-${++ledgers}`) }); + try { + for (const entry of entries) store.appendJson('operator/changes', entry); + store.sync(); + await assert.rejects(startOn(store), (e: Error) => refusedAs('operator/changes')(e) && why.test(e.message)); + } finally { + store.close(); + } + }; + const hideAttempt = (ev: object) => ({ kind: 'attempt', changeId: 'h', changeKind: 'hide', agent: 'scout', n: 1, at: 1, evidence: ev }); + + it("refuses a hide attempt without its ids, which recovery would read as an empty removal", async () => { + const { ids: _omitted, ...withoutIds } = hideEvidence; + await refuses([hideAttempt(withoutIds)], /a hide's ids/); + }); + + it('refuses a hide whose removal count differs from its ids', async () => { + await refuses([hideAttempt({ ...hideEvidence, removed: 3 })], /a hide removes exactly its ids/); + }); + + it('refuses a marks ref missing what delivery dereferences', async () => { + const marks = { scope: 'all', refs: [{ serverId: 's', channelId: 'c' }], unmarked: 0, notRemoved: 0 }; + await refuses([hideAttempt({ ...hideEvidence, marks })], /a marks ref/); + }); + + it('refuses an outcome whose receipt lacks what its status needs', async () => { + const markers = { scope: 'all', unmarked: 0, notRemoved: 0, status: 'queued', queued: 1 }; + await refuses([hideAttempt(hideEvidence), { kind: 'outcome', changeId: 'h', outcome: { n: 1, at: 2, removed: 2, markers } }], /markers receipt/); + }); + + it('refuses an outcome, or a switch, that names no attempt of its own', async () => { + const markers = { scope: 'none', unmarked: 0, notRemoved: 0, status: 'none', queued: 0 }; + await refuses([hideAttempt(hideEvidence), { kind: 'outcome', changeId: 'h', outcome: { n: 2, at: 2, removed: 2, markers } }], /names no attempt of its own/); + await refuses([{ kind: 'switched', changeId: 'never-attempted', n: 1 }], /switched entry/); + }); + it('refuses a null-snapshot checkpoint in the unstick journal too', async () => { const store = JsStore.openOrCreate({ path: join(dir, 'store') }); const launched = store.appendJson('operator/unstick', { kind: 'attempt-launched', operationId: 'op', agent: 'scout', step: 1 }); From 532146bcd4ab354d7989d0b4e5bfc3f6a0853f79 Mon Sep 17 00:00:00 2001 From: Petra-1779 Date: Wed, 7 Oct 2026 19:51:02 -0400 Subject: [PATCH 53/73] docs(framework): recovery comments state the rules the code enforces (Greptile #255 P2-7) Greptile's seventh finding on #255 (discussion_r4213085220): two doc comments described recovery rules the code had already discarded, and Theo-1778 found a third. - **`applyCut`** said an attempt committed if its destination was active or written to since. The code decides an attempt only by a record or by what this process saw (`cutAttemptDisposition`): committed (switch recorded or seen, or an operator's attestation), not committed (failure recorded or seen, or a not-committed verdict; abandoned, source restored if its destination is still active, then a fresh attempt), or unresolved (refused as `unresolved` until resolveOperatorChange). A fresh attempt cuts onto a destination of its own. - **`applyUndoMessages`** said an interrupted attempt's existing destination is reused. Each attempt's destination is its own (`base`, then `base~n`), a branch already bearing that name is refused, and a retry after an abandoned attempt cuts onto a fresh one. - **`reconcileOperatorChanges`** said startup establishes a cut "whose destination is active or used". It establishes one whose switch was recorded or that an operator attested committed. An unrecorded attempt is unresolved, never certified, and boots the host quiesced when its destination is the active body. Comments only; no behavior change. Co-Authored-By: Petra (Claude Opus 5.5) --- src/framework.ts | 43 +++++++++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 14 deletions(-) diff --git a/src/framework.ts b/src/framework.ts index 4b95d9388..124a7c22b 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -7916,12 +7916,21 @@ export class AgentFramework { /** * Apply a staged branch cut under the held lease, or recover it. * - An established outcome is reported as it was recorded. - * - The latest attempt committed if its switch was recorded, its - * destination is active, or its destination was written to since (being - * back on the source proves nothing): its outcome is established from - * its own evidence, and it is never cut again. - * - Otherwise this is a fresh attempt: revalidated on the source, its - * evidence recorded before the cut, its switch right after. + * - The latest attempt is decided only by a record or by what this process + * saw (cutAttemptDisposition), never by its destination's state, active + * or written to: + * - committed (its switch recorded or seen, or an operator's attestation): + * its outcome is established from its own evidence, and it is never + * cut again; + * - not committed (its failure recorded or seen, or an operator's + * verdict): abandoned, with its source restored if its destination is + * still active, and a fresh attempt follows; + * - unresolved (no switch or failure recorded or seen, and no operator + * verdict): refused as `unresolved` until an operator settles it with + * resolveOperatorChange. + * - A fresh attempt cuts onto a destination of its own, never one an + * earlier attempt touched: revalidated on the source, its evidence + * recorded before the cut, its switch right after. */ private async applyCut( change: ResolvedUndoTurnsChange | ResolvedUndoMessagesChange, @@ -8138,9 +8147,10 @@ export class AgentFramework { /** * Apply a staged undo by messages under the held lease: the rollback cut - * at its tail, onto the branch named for the change (an existing one an - * interrupted attempt created is reused), through the same branching the - * ungated rollback uses. + * at its tail, through the same branching the ungated rollback uses, onto + * the branch named for the change and attempt (applyCut). An interrupted + * attempt's destination is never reused: a retry after an abandoned + * attempt cuts onto a fresh one. */ private async applyUndoMessages(change: ResolvedUndoMessagesChange, admission?: OperatorAdmission): Promise<{ requested: number; @@ -8335,11 +8345,16 @@ export class AgentFramework { /** * At startup, establish the outcome of every gated body change that - * committed before a crash recorded it: a cut whose switch was recorded or - * whose destination is active or used, and an interrupted hide, finished - * on its own branch. This runs before modules, connections or traffic, so - * a committed body never waits on its host to settle its marks. Anything - * not provably committed is left for the host's retry. + * committed before a crash recorded it: a cut whose switch was recorded, + * or that an operator attested as committed (resolveOperatorChange, live + * or offline), and an interrupted hide, finished on its own branch. A + * cut's destination being active or written to proves nothing: a cut with + * no recorded switch, failure or operator verdict is unresolved, never + * certified, and if its destination is the active body, startup boots + * quiesced until an operator settles it. This runs before modules, + * connections or traffic, so a committed body never waits on its host to + * settle its marks. Anything else not provably committed is left for the + * host's retry. */ private async reconcileOperatorChanges(): Promise { // Both journals were first read before the agents (an unreadable one From f167efda4b69480f373be0a4da158402bc777def Mon Sep 17 00:00:00 2001 From: Petra-1779 Date: Wed, 7 Oct 2026 19:58:08 -0400 Subject: [PATCH 54/73] fix(agent): a settings preview names the default pace a paced descent installs (Greptile #255 P2-5) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Greptile's fifth finding on #255 (discussion_r4213085207). previewRuntimeSettingsTarget is the operator gate's statement of what an agent_settings change will do: its keys are the settings resolveSettingsEffect reports `from` and `target` for, and the ones revalidation checks before applying. It named only the keys the action named, but two paths change a setting nobody named: - **updateRuntimeSettings:** a non-immediate decrease with no pace set (hot `transitionPaceTokens` undefined) installs and persists `DEFAULT_TRANSITION_PACE_TOKENS`. - **resetRuntimeSettings:** a budget reset below the live budget (a descent) with no pace set installs the same default, even when `transitionPaceTokens` isn't among the reset keys. So an operator approved `contextBudgetTokens: 100000 → 60000` and got a 16000 pace as well (probe: the staged change showed `{contextBudgetTokens}` only, the snapshot after apply carried `transitionPaceTokens: 16000`). A pace set between staging and applying went unnoticed by revalidation, because the approval never touched it. The preview now mirrors both: an update's paced decrease (`contextBudgetTokens < live`, not `immediate`, no explicit pace) and a budget reset that starts a descent, each with a live strategy and no pace set, report `transitionPaceTokens` at the default. An explicit pace, an immediate decrease, an increase, and the existing pace-reset rule (which governs whenever the pace is itself reset, reset-all included) are unchanged. resolveSettingsEffect then reports the pace in `from` (null when unset) and `target`, so revalidation refuses the change as stale once a pace is set meanwhile. The doc comment states the rule. Tests: - agent-runtime-settings: one test walks eight actions (a paced decrease with no pace set, reset-all, an immediate decrease, an increase, a budget-only reset starting a descent, a decrease with an explicit pace, a paced decrease with a pace already set, cancel) and asserts the preview names every setting whose reported value changes, at the value getRuntimeSettings reports after. - operator-change-gate: a staged unpaced decrease carries `{transitionPaceTokens: null} → 16000` and is refused as stale once a pace is set. A staged budget-only reset carries the same, and applies exactly its approved target. Mutants caught: the fix removed (both tests); the reset branch alone removed (both tests). agent-runtime-settings and operator-change-gate: 116/116. Co-Authored-By: Petra (Claude Opus 5.5) --- src/agent.ts | 21 +++++++-- test/agent-runtime-settings.test.ts | 71 +++++++++++++++++++++++++++++ test/operator-change-gate.test.ts | 60 ++++++++++++++++++++++++ 3 files changed, 147 insertions(+), 5 deletions(-) diff --git a/src/agent.ts b/src/agent.ts index a32f1aa39..a6c57515d 100644 --- a/src/agent.ts +++ b/src/agent.ts @@ -518,6 +518,12 @@ export class Agent { * default pace. * - cancel: a converging budget stops at the live budget; nothing else * changes. + * + * A setting the action changes without naming it is reported too: a paced + * descent (an update's non-immediate decrease, or a budget reset below the + * live budget) with no pace set installs the default pace, exactly as + * updateRuntimeSettings and resetRuntimeSettings do, so transitionPaceTokens + * is reported at that default. */ previewRuntimeSettingsTarget( change: @@ -526,26 +532,29 @@ export class Agent { | { action: 'cancel' }, ): Partial> { const target: Partial> = {}; + const hot = this.getHotContextSettings(); + const live = this.contextBudgetTokens ?? DEFAULT_CONTEXT_BUDGET_TOKENS; + // A paced descent with no pace set installs the default pace. + const unpaced = hot !== null && hot.transitionPaceTokens === undefined; if (change.action === 'update') { const p = change.patch; if (p.contextBudgetTokens !== undefined) target.contextBudgetTokens = p.contextBudgetTokens; if (p.tailTokens !== undefined) target.tailTokens = p.tailTokens; if (p.transitionPaceTokens !== undefined) target.transitionPaceTokens = p.transitionPaceTokens; + else if (p.contextBudgetTokens !== undefined && p.contextBudgetTokens < live && !p.immediate && unpaced) { + target.transitionPaceTokens = DEFAULT_TRANSITION_PACE_TOKENS; + } if (p.sameRoundThinkTextPolicy !== undefined) target.sameRoundThinkTextPolicy = p.sameRoundThinkTextPolicy; return target; } if (change.action === 'cancel') { - if (this.contextBudgetTargetTokens !== undefined) { - target.contextBudgetTokens = this.contextBudgetTokens ?? DEFAULT_CONTEXT_BUDGET_TOKENS; - } + if (this.contextBudgetTargetTokens !== undefined) target.contextBudgetTokens = live; return target; } const keys = new Set(change.keys ?? ['contextBudgetTokens', 'tailTokens', 'transitionPaceTokens', 'sameRoundThinkTextPolicy']); - const hot = this.getHotContextSettings(); let startsDescent = false; if (keys.has('contextBudgetTokens')) { const configured = this.configuredContextBudgetTokens ?? DEFAULT_CONTEXT_BUDGET_TOKENS; - const live = this.contextBudgetTokens ?? DEFAULT_CONTEXT_BUDGET_TOKENS; startsDescent = configured < live; target.contextBudgetTokens = configured; } @@ -557,6 +566,8 @@ export class Agent { target.transitionPaceTokens = descending && this.configuredTransitionPaceTokens === undefined ? DEFAULT_TRANSITION_PACE_TOKENS : this.configuredTransitionPaceTokens ?? null; + } else if (startsDescent && unpaced) { + target.transitionPaceTokens = DEFAULT_TRANSITION_PACE_TOKENS; } if (keys.has('sameRoundThinkTextPolicy')) { target.sameRoundThinkTextPolicy = this.configuredSameRoundThinkTextPolicy ?? DEFAULT_SAME_ROUND_THINK_TEXT_POLICY; diff --git a/test/agent-runtime-settings.test.ts b/test/agent-runtime-settings.test.ts index 90c482f84..8b22b4bd4 100644 --- a/test/agent-runtime-settings.test.ts +++ b/test/agent-runtime-settings.test.ts @@ -526,3 +526,74 @@ it('preflight: no-op and increasing patches never throw on an already-over-floor ); }); }); + +it('previewRuntimeSettingsTarget names every setting an action changes, at the value it is reported at after, the default pace of a paced descent included', async () => { + const dir = mkdtempSync(join(tmpdir(), 'agent-settings-preview-')); + const framework = await AgentFramework.create({ + storePath: join(dir, 'store'), + membrane, + agents: [{ + name: 'agent', + model: 'test-model', + systemPrompt: 'test', + // No kvStableReachTokens: no transition pace is set, so a paced + // descent installs the default one. + strategy: new AutobiographicalStrategy({ adaptiveResolution: true, foldingStrategy: 'kv-stable', recentWindowTokens: 30_000 }), + contextBudgetTokens: 100_000, + maxTokens: 10_000, + }], + modules: [], + }); + try { + const agent = framework.getAgent('agent')!; + const keys = ['contextBudgetTokens', 'tailTokens', 'transitionPaceTokens', 'sameRoundThinkTextPolicy'] as const; + const reported = () => { + const snapshot = framework.getAgentRuntimeSettings('agent') as unknown as Record; + return Object.fromEntries(keys.map((key) => [key, snapshot[key] ?? null])) as Record; + }; + type Action = Parameters[0]; + // The preview must name exactly what the action changes, at the values + // getRuntimeSettings reports once it applied. + const check = (label: string, action: Action, apply: () => void) => { + const before = reported(); + const target = agent.previewRuntimeSettingsTarget(action) as Record; + apply(); + const after = reported(); + for (const key of keys) { + if (after[key] !== before[key]) { + assert.ok(key in target, `${label}: ${key} changes from ${String(before[key])} to ${String(after[key])}, but the preview omits it`); + } + } + for (const [key, value] of Object.entries(target)) { + assert.equal(value, after[key], `${label}: previewed ${key}=${String(value)}, reported ${String(after[key])}`); + } + }; + + assert.equal(reported().transitionPaceTokens, null, 'no pace is set to begin with'); + check('a paced decrease with no pace set', + { action: 'update', patch: { contextBudgetTokens: 60_000 } }, + () => framework.updateAgentRuntimeSettings('agent', { contextBudgetTokens: 60_000 })); + assert.equal(reported().transitionPaceTokens, 16_000, 'the default pace was installed'); + check('reset-all', { action: 'reset' }, () => framework.resetAgentRuntimeSettings('agent')); + check('an immediate decrease installs no pace', + { action: 'update', patch: { contextBudgetTokens: 80_000, immediate: true } }, + () => framework.updateAgentRuntimeSettings('agent', { contextBudgetTokens: 80_000, immediate: true })); + check('an increase', { action: 'update', patch: { contextBudgetTokens: 150_000 } }, + () => framework.updateAgentRuntimeSettings('agent', { contextBudgetTokens: 150_000 })); + assert.equal(reported().transitionPaceTokens, null); + check('a budget-only reset that starts a descent with no pace set', + { action: 'reset', keys: ['contextBudgetTokens'] }, + () => framework.resetAgentRuntimeSettings('agent', ['contextBudgetTokens'])); + assert.equal(reported().transitionPaceTokens, 16_000, 'the reset installed the default pace'); + check('a decrease with an explicit pace', + { action: 'update', patch: { contextBudgetTokens: 50_000, transitionPaceTokens: 2_000 } }, + () => framework.updateAgentRuntimeSettings('agent', { contextBudgetTokens: 50_000, transitionPaceTokens: 2_000 })); + check('a paced decrease with a pace already set leaves it alone', + { action: 'update', patch: { contextBudgetTokens: 40_000 } }, + () => framework.updateAgentRuntimeSettings('agent', { contextBudgetTokens: 40_000 })); + check('cancel', { action: 'cancel' }, () => framework.cancelAgentRuntimeSettingsTransition('agent')); + } finally { + await framework.stop(); + rmSync(dir, { recursive: true, force: true }); + } +}); diff --git a/test/operator-change-gate.test.ts b/test/operator-change-gate.test.ts index 9a547a796..ff286c8e6 100644 --- a/test/operator-change-gate.test.ts +++ b/test/operator-change-gate.test.ts @@ -376,6 +376,66 @@ describe('operator-change gate: self-change tools', () => { }); }); +describe('operator-change gate: settings an action changes without naming them', () => { + it('stages the default pace a paced descent installs, for an update and a budget reset, and refuses it as stale once a pace is set', async () => { + const dir = mkdtempSync(join(tmpdir(), 'operator-gate-pace-')); + const asked: ResolvedOperatorChange[] = []; + const framework = await AgentFramework.create({ + storePath: join(dir, 'store'), + membrane: new MockMembrane().asMembrane(), + agents: [{ + name: 'ada', model: 'test', systemPrompt: '.', maxTokens: 10_000, contextBudgetTokens: 100_000, + // No kvStableReachTokens: no pace is set until a descent installs one. + strategy: new AutobiographicalStrategy({ adaptiveResolution: true, foldingStrategy: 'kv-stable', recentWindowTokens: 30_000 }), + }], + modules: [], + operatorChangeGate: async (change) => { + asked.push(structuredClone(change)); + return { id: `rev-${asked.length}`, text: 'staged' }; + }, + }); + const quiet = console.log; + console.log = () => {}; + try { + const apply = (change: ResolvedOperatorChange) => framework.runAtSafeBoundary({ verb: 'apply' }, (lease) => + framework.applyResolvedOperatorChange(change, { lease, admission: { id: change.id } })); + const settings = () => framework.getAgentRuntimeSettings('ada'); + + await framework.puppetToolCall('ada', 'agent_settings', { action: 'update', context_budget_tokens: 60_000 }); + const decrease = asked[0]!; + assert.ok(decrease.kind === 'agent-settings'); + assert.deepEqual([decrease.from, decrease.target], [ + { contextBudgetTokens: 100_000, transitionPaceTokens: null }, + { contextBudgetTokens: 60_000, transitionPaceTokens: 16_000 }, + ], 'the approval shows the pace the descent installs'); + framework.updateAgentRuntimeSettings('ada', { transitionPaceTokens: 4_000 }); // a pace set meanwhile + await assert.rejects(apply(decrease), (e: Error & { code?: string }) => e.code === 'stale'); + assert.equal(settings().contextBudgetTokens, 100_000, 'nothing applied'); + + // A budget-only reset that starts a descent, with no pace set. + framework.updateAgentRuntimeSettings('ada', { contextBudgetTokens: 150_000 }); + framework.resetAgentRuntimeSettings('ada', ['transitionPaceTokens']); + await framework.puppetToolCall('ada', 'agent_settings', { action: 'reset', settings: ['context_budget_tokens'] }); + const reset = asked[1]!; + assert.ok(reset.kind === 'agent-settings'); + assert.deepEqual([reset.from, reset.target], [ + { contextBudgetTokens: 150_000, transitionPaceTokens: null }, + { contextBudgetTokens: 100_000, transitionPaceTokens: 16_000 }, + ]); + await apply(reset); + assert.deepEqual( + { contextBudgetTokens: settings().contextBudgetTokens, transitionPaceTokens: settings().transitionPaceTokens }, + reset.target, + 'applied exactly what was approved', + ); + } finally { + console.log = quiet; + await framework.stop(); + rmSync(dir, { recursive: true, force: true }); + } + }); +}); + describe('operator-change gate: tool presentation', () => { it('stages an operator\'s presentation edit with its from-state, and applies it later under a lease', async () => { const dir = mkdtempSync(join(tmpdir(), 'operator-gate-presentation-')); From 8e891242c105a009d62a14919a84fc4d16916e5b Mon Sep 17 00:00:00 2001 From: Petra-1779 Date: Wed, 7 Oct 2026 20:05:51 -0400 Subject: [PATCH 55/73] fix(framework,presentation): a staged tool-presentation edit records what it leaves, so a reset can't expose unapproved wording (Greptile #255 P1-3) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Greptile's third finding on #255 (discussion_r4213085194). A staged `set_tool_description {description: null}` recorded only `from`, the entry as it reads now. A personal description masks the wording a reset exposes, which ToolPresentation.resolve takes from the component default and else the installed description. If either moved while the personal description stayed, revalidation compared `from` (unchanged) and passed, and removing the override exposed wording nobody resolved at staging. The settings changes already have the matching guard: `target`, recomputed and compared before applying. - **ToolPresentation.previewEdit(tool, input, tools, sources)** returns the snapshot an edit would leave, writing nothing. It uses edit()'s own argument checks and its one document change, both now shared (checkEdit, applyEdit), including the result checks edit() made (a hidden recovery tool, the size cap). It resolves through the same code as resolve(): component defaults, installed descriptions, and the catalogue signpost on the editing tools. resolve() is split around that shared resolveDocument(); its output is unchanged. - **ResolvedPresentationChange.target** is the targeted entry as the edit leaves it, resolved at staging. For a reset, that's the exposed wording. - **Revalidation** recomputes it and refuses as stale ("it would now leave X as …, not the approved …") when it differs, or when the edit can no longer be made. Visibility edits and explicit descriptions get the same check. Their targets follow from the input, so for them it only catches a changed configuration. - **Staging refuses an edit the presentation would refuse**, with the edit's own reason ("Operator change refused: it couldn't be resolved (Error: Invalid description)"), instead of staging a change that could only fail at application. - framework: toolPresentationInputs() gives inspectToolPresentation and the new presentationEditTarget() the same tools and sources. `target` is a new required field. Nothing persists a ResolvedPresentationChange in this repository; a host-held one staged before this change has no target and is refused as stale, so it can be restaged. Tests: - component-defaults: previewEdit equals what the edit then leaves, for five edits (three resets: component default, installed, editing tool with signpost; an explicit description; a visibility edit). It writes nothing, and refuses three invalid edits with exactly edit()'s error. - operator-change-gate: a staged reset under a personal description carries the default wording A it exposes. When the component default changes to B under the unchanged personal description, application is refused as stale naming B, and nothing is applied. Restaged, it carries B and applies exactly B. An invalid description is refused at staging, never staged. Mutants caught: target comparison removed (gate test: missing rejection); previewEdit ignoring the edit (both tests). operator-change-gate, tool-presentation, component-defaults and agent-runtime-settings: 127/127. Co-Authored-By: Petra (Claude Opus 5.5) --- src/framework.ts | 33 +++++++++++++++- src/operator-change.ts | 6 +++ src/tool-presentation.ts | 62 +++++++++++++++++++++++-------- test/component-defaults.test.ts | 42 +++++++++++++++++++++ test/operator-change-gate.test.ts | 62 +++++++++++++++++++++++++++++++ 5 files changed, 188 insertions(+), 17 deletions(-) diff --git a/src/framework.ts b/src/framework.ts index 124a7c22b..edb332d62 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -3204,6 +3204,12 @@ export class AgentFramework { } inspectToolPresentation(agentName: string, snapshot?: InferenceToolSnapshot): PresentationSnapshot | null { + const inputs = this.toolPresentationInputs(agentName, snapshot); + return inputs ? inputs.presentation.resolve(inputs.tools, inputs.sources) : null; + } + + /** An agent's presentation, with the tools and sources it resolves. */ + private toolPresentationInputs(agentName: string, snapshot?: InferenceToolSnapshot) { const presentation = this.toolPresentations.get(agentName); if (!presentation) return null; const tools = this.availableToolsForPresentation(agentName, snapshot); @@ -3220,7 +3226,19 @@ export class AgentFramework { } if (!sources.has(tool.name)) sources.set(tool.name, 'Framework'); } - return presentation.resolve(tools, sources); + return { presentation, tools, sources }; + } + + /** The targeted entry as a presentation edit would leave it, written + * nowhere (ToolPresentation.previewEdit). Throws when the edit couldn't + * be made, with the reason the edit itself would give. */ + private presentationEditTarget(agentName: string, tool: string, input: Record): ResolvedPresentationChange['target'] { + const inputs = this.toolPresentationInputs(agentName); + if (!inputs) throw new Error(`Tool presentation is not enabled for ${agentName}`); + const entry = inputs.presentation.previewEdit(tool, input, inputs.tools, inputs.sources) + .entries.find((e) => e.name === input.name); + if (!entry) throw new Error(`Tool presentation has no entry ${JSON.stringify(input.name)} for ${agentName}`); + return { name: entry.name, visible: entry.visible, description: entry.description }; } private advertisedToolsForAgent(agentName: string, snapshot?: InferenceToolSnapshot) { @@ -6975,6 +6993,7 @@ export class AgentFramework { tool: tool as ResolvedPresentationChange['tool'], input: structuredClone(input), from: { name: entry.name, visible: entry.visible, description: entry.description }, + target: this.presentationEditTarget(agentName, tool, input), }; } @@ -7014,6 +7033,18 @@ export class AgentFramework { if (!sameValue(now, change.from)) { stale(`the presentation of ${change.from.name} moved from ${JSON.stringify(change.from)} to ${JSON.stringify(now)}`); } + // The from-state can hold while what the edit exposes moves: a personal + // description masks the component default and installed wording that a + // reset would reveal. + let target: ResolvedPresentationChange['target']; + try { + target = this.presentationEditTarget(change.agent, change.tool, change.input); + } catch (error) { + stale(`it can no longer be resolved (${error instanceof Error ? error.message : String(error)})`); + } + if (!sameValue(target!, change.target)) { + stale(`it would now leave ${change.from.name} as ${JSON.stringify(target!)}, not the approved ${JSON.stringify(change.target)}`); + } } /** diff --git a/src/operator-change.ts b/src/operator-change.ts index ebac6e1c1..34fe08545 100644 --- a/src/operator-change.ts +++ b/src/operator-change.ts @@ -67,6 +67,12 @@ export interface ResolvedPresentationChange extends ResolvedOperatorChangeBase { input: Record; /** The targeted tool's presentation entry when resolved. */ from: { name: string; visible: boolean; description: string }; + /** The entry as the edit leaves it, resolved at staging exactly as the + * presentation resolves (ToolPresentation.previewEdit): a description + * reset shows the wording it exposes, the component default or else the + * installed description. Application applies the edit only if it still + * leaves exactly this. */ + target: { name: string; visible: boolean; description: string }; } /** host/command undo by turns: the exact turn checkpoints it undoes. */ diff --git a/src/tool-presentation.ts b/src/tool-presentation.ts index 8fd9eef42..3d07d4a07 100644 --- a/src/tool-presentation.ts +++ b/src/tool-presentation.ts @@ -51,6 +51,34 @@ export function presentationTools(path: string): ToolDefinition[] { ] as unknown as ToolDefinition[]; } +/** One checked edit: the tool it names, the field it sets, and the value (null resets a description). */ +interface PresentationEdit { name: string; field: 'visible' | 'description'; value: unknown } + +/** edit()'s argument checks, shared with previewEdit; throws what edit() reports as its error. */ +function checkEdit(tool: string, input: unknown, available: ToolDefinition[]): PresentationEdit { + if (!isPresentationTool(tool)) throw new Error('Unknown editing tool'); + const value = input as Record; + const field = tool === 'set_tool_visibility' ? 'visible' : 'description'; + if (!value || typeof value !== 'object' || Array.isArray(value) || typeof value.name !== 'string' + || Object.keys(value).some(k=> !['name',field].includes(k)) || !Object.hasOwn(value,field)) throw new Error('Invalid editing arguments'); + if (!available.some(t=>t.name===value.name)) throw new Error('Tool is not currently available to this agent'); + if (field === 'visible' ? typeof value.visible !== 'boolean' : value.description !== null && typeof value.description !== 'string') throw new Error(`Invalid ${field}`); + return {name:value.name as string,field,value:value[field]}; +} + +/** Make a checked edit to `doc` in place and return the file it makes; throws if that file would be refused. */ +function applyEdit(doc: PresentationDocument, edit: PresentationEdit): string { + const old = Object.hasOwn(doc.tools,edit.name) ? doc.tools[edit.name] : {}; + const updated = {...old}; + if (edit.field === 'description' && edit.value === null) delete updated.description; + else Object.assign(updated,{[edit.field]:edit.value}); + Object.defineProperty(doc.tools,edit.name,{value:updated,enumerable:true,writable:true,configurable:true}); + const next = JSON.stringify(doc,null,2)+'\n'; + parsePresentation(next); + if (Buffer.byteLength(next)>MAX_BYTES) throw new Error('Presentation exceeds 256 KiB'); + return next; +} + /** Per-agent presentation; files are the sole persistent source, independent of editor identity. */ export class ToolPresentation { constructor(readonly config: ToolPresentationConfig) {} @@ -72,6 +100,21 @@ export class ToolPresentation { let raw = '', doc: PresentationDocument = {version:1,tools:{}}, diagnostics: string[] = []; try { raw = this.read(); doc = parsePresentation(raw); } catch (error) { diagnostics.push(`Overrides not applied: ${String(error)}. Default visibility and component descriptions retained.`); } + return this.resolveDocument(raw, doc, diagnostics, tools, sources); + } + /** + * The snapshot an edit would leave, written nowhere: edit()'s own checks + * and change, resolved exactly as resolve() resolves the file (component + * defaults, installed descriptions, catalogue signposts). A description + * reset therefore shows the wording it would expose. Throws what edit() + * would report as its error. + */ + previewEdit(tool: string, input: unknown, tools: ToolDefinition[], sources: ReadonlyMap = new Map()): PresentationSnapshot { + const edit = checkEdit(tool, input, tools); + const doc = parsePresentation(this.read()); + return this.resolveDocument(applyEdit(doc, edit), doc, [], tools, sources); + } + private resolveDocument(raw: string, doc: PresentationDocument, diagnostics: string[], tools: ToolDefinition[], sources: ReadonlyMap): PresentationSnapshot { const defaults = new Map(); const configuredSources = new Set(); for (const profile of this.config.defaults ?? []) { @@ -112,26 +155,13 @@ export class ToolPresentation { const lock = this.config.path + '.lock'; let fd: number | undefined, tempFd: number | undefined, temp: string | undefined; try { - if (!isPresentationTool(tool)) throw new Error('Unknown editing tool'); - const value = input as Record; - const field = tool === 'set_tool_visibility' ? 'visible' : 'description'; - if (!value || typeof value !== 'object' || Array.isArray(value) || typeof value.name !== 'string' - || Object.keys(value).some(k=> !['name',field].includes(k)) || !Object.hasOwn(value,field)) throw new Error('Invalid editing arguments'); - if (!available.some(t=>t.name===value.name)) throw new Error('Tool is not currently available to this agent'); - if (field === 'visible' ? typeof value.visible !== 'boolean' : value.description !== null && typeof value.description !== 'string') throw new Error(`Invalid ${field}`); + const edit = checkEdit(tool, input, available); fd = openSync(lock, 'wx', 0o600); let mode = 0o600; try { const stat = lstatSync(this.config.path); if (stat.isSymbolicLink()) throw new Error('Edit the target file directly; tool editing refuses symlinks'); mode = stat.mode & 0o777; } catch (error) { if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error; } const before = this.read(), doc = parsePresentation(before); - const old = Object.hasOwn(doc.tools,value.name) ? doc.tools[value.name] : {}; - const updated = {...old}; - if (field === 'description' && value.description === null) delete updated.description; - else Object.assign(updated,{[field]:value[field]}); - Object.defineProperty(doc.tools,value.name,{value:updated,enumerable:true,writable:true,configurable:true}); - const next = JSON.stringify(doc,null,2)+'\n'; - parsePresentation(next); - if (Buffer.byteLength(next)>MAX_BYTES) throw new Error('Presentation exceeds 256 KiB'); + const next = applyEdit(doc, edit); temp = this.config.path + '.' + randomUUID() + '.tmp'; tempFd = openSync(temp,'wx',0o600); writeFileSync(tempFd,next); @@ -141,7 +171,7 @@ export class ToolPresentation { if (!named.isFile() || named.dev !== opened.dev || named.ino !== opened.ino) throw new Error('Temporary file changed concurrently; retry after checking the directory'); renameSync(temp,this.config.path); temp=undefined; - return {success:true,data:{name:value.name,[field]:value[field],effective:'next newly compiled request',catalogue:this.config.cataloguePath}}; + return {success:true,data:{name:edit.name,[edit.field]:edit.value,effective:'next newly compiled request',catalogue:this.config.cataloguePath}}; } catch (error) { return {success:false,isError:true,error:String(error)}; } finally { if(tempFd!==undefined) closeSync(tempFd); if(temp) try{unlinkSync(temp);}catch{} if(fd!==undefined){closeSync(fd);unlinkSync(lock);} } } diff --git a/test/component-defaults.test.ts b/test/component-defaults.test.ts index 398d564b0..93c62d798 100644 --- a/test/component-defaults.test.ts +++ b/test/component-defaults.test.ts @@ -41,3 +41,45 @@ test('component and resident descriptions both retain catalogue discovery on edi const own=p.resolve(tools,sources).advertised[0];assert.match(own.description,/mine/);assert.match(own.description,/workspace--read.*board\/recovery.md/); } finally {rmSync(dir,{recursive:true,force:true});} }); + +test('previewEdit resolves what an edit would leave, a reset exposing the default it reveals, and writes nothing',()=>{ + const dir=mkdtempSync(join(tmpdir(),'component-preview-')); + try { + const path=join(dir,'resident.json'),profile=join(dir,'framework.json'); + writeFileSync(profile,JSON.stringify({version:1,tools:{send:{description:'clear'},set_tool_visibility:{description:'profile wording'}}})); + const p=new ToolPresentation({path,cataloguePath:'board/catalogue.md',defaults:[{source:'Framework',path:profile}]}); + const tools=['send','set_tool_visibility','plain'].map(name=>({name,description:'installed '+name,inputSchema:{type:'object' as const}})); + const sources=new Map(tools.map(t=>[t.name,'Framework'])); + writeFileSync(path,JSON.stringify({version:1,tools:{send:{description:'mine'},set_tool_visibility:{description:'mine too'},plain:{description:'mine as well'}}})); + const before=readFileSync(path,'utf8'); + const entry=(snapshot: ReturnType,name: string)=>snapshot.entries.find(e=>e.name===name)!; + // Each preview equals what the edit itself then leaves. + for (const [tool,input] of [ + ['set_tool_description',{name:'send',description:null}], + ['set_tool_description',{name:'set_tool_visibility',description:null}], + ['set_tool_description',{name:'plain',description:null}], + ['set_tool_description',{name:'send',description:'newer'}], + ['set_tool_visibility',{name:'plain',visible:false}], + ] as const) { + const preview=entry(p.previewEdit(tool,input,tools,sources),input.name); + assert.equal(readFileSync(path,'utf8'),before,'a preview writes nothing'); + assert.equal(p.edit(tool,input,tools).success,true); + const after=entry(p.resolve(tools,sources),input.name); + assert.deepEqual({visible:preview.visible,description:preview.description},{visible:after.visible,description:after.description},`${tool} ${JSON.stringify(input)}`); + writeFileSync(path,before); + } + assert.equal(entry(p.previewEdit('set_tool_description',{name:'send',description:null},tools,sources),'send').description,'clear','the component default'); + assert.equal(entry(p.previewEdit('set_tool_description',{name:'plain',description:null},tools,sources),'plain').description,'installed plain','else the installed wording'); + assert.match(entry(p.previewEdit('set_tool_description',{name:'set_tool_visibility',description:null},tools,sources),'set_tool_visibility').description,/^profile wording\nCatalogue: workspace--read/,'with the editing tools\' signpost'); + // A preview refuses what the edit would refuse, with the same reason. + for (const [tool,input] of [ + ['set_tool_visibility',{name:'workspace--read',visible:false}], + ['set_tool_description',{name:'unknown',description:null}], + ['set_tool_description',{name:'send',description:7}], + ] as const) { + const refused=p.edit(tool,input,[...tools,{name:'workspace--read',description:'read',inputSchema:{type:'object' as const}}]); + assert.equal(refused.success,false); + assert.throws(()=>p.previewEdit(tool,input,[...tools,{name:'workspace--read',description:'read',inputSchema:{type:'object' as const}}],sources),(e: Error)=>String(e)===refused.error); + } + } finally {rmSync(dir,{recursive:true,force:true});} +}); diff --git a/test/operator-change-gate.test.ts b/test/operator-change-gate.test.ts index ff286c8e6..ebbc6b22c 100644 --- a/test/operator-change-gate.test.ts +++ b/test/operator-change-gate.test.ts @@ -486,6 +486,68 @@ describe('operator-change gate: tool presentation', () => { rmSync(dir, { recursive: true, force: true }); } }); + + it('stages a description reset with the wording it exposes, and refuses it as stale when that wording moves under an unchanged personal description', async () => { + const dir = mkdtempSync(join(tmpdir(), 'operator-gate-reset-target-')); + const profile = join(dir, 'workspace-defaults.json'); + const personal = join(dir, 'tools.json'); + writeFileSync(profile, JSON.stringify({ version: 1, tools: { 'workspace--glob': { description: 'default wording A' } } })); + writeFileSync(personal, JSON.stringify({ version: 1, tools: { 'workspace--glob': { description: 'mine' } } })); + const workspace = new WorkspaceModule({ mounts: [{ name: 'board', path: dir, mode: 'read-write', watch: 'never' }] }); + const asked: ResolvedOperatorChange[] = []; + const framework = await AgentFramework.create({ + storePath: join(dir, 'store'), + membrane: new MockMembrane().asMembrane(), + agents: [{ + name: 'ada', model: 'test', systemPrompt: '.', + strategy: new AutobiographicalStrategy({ adaptiveResolution: true, foldingStrategy: 'kv-stable', recentWindowTokens: 30000, kvStableReachTokens: 8000 }), + toolPresentation: { path: personal, cataloguePath: 'board/tools.md', defaults: [{ source: 'Module: workspace', path: profile }] }, + }], + modules: [workspace], + operatorChangeGate: async (change) => { + asked.push(structuredClone(change)); + return { id: `rev-${asked.length}`, text: 'staged' }; + }, + }); + const quiet = console.log; + console.log = () => {}; + try { + const shown = () => framework.inspectToolPresentation('ada')!.entries.find((e) => e.name === 'workspace--glob')!.description; + const apply = (change: ResolvedOperatorChange) => framework.runAtSafeBoundary({ verb: 'apply' }, (lease) => + framework.applyResolvedOperatorChange(change, { lease, admission: { id: change.id } })); + const reset = { name: 'workspace--glob', description: null }; + + await framework.puppetToolCall('ada', 'set_tool_description', reset); + const first = asked[0]!; + assert.ok(first.kind === 'tool-presentation'); + assert.deepEqual([first.from.description, first.target.description], ['mine', 'default wording A'], + 'the approval shows the wording the reset exposes'); + + // The default changes while the personal description, all `from` sees, stays. + writeFileSync(profile, JSON.stringify({ version: 1, tools: { 'workspace--glob': { description: 'default wording B' } } })); + assert.equal(shown(), 'mine'); + await assert.rejects(apply(first), (e: Error & { code?: string }) => + e.code === 'stale' && /would now leave workspace--glob/.test(e.message) && /default wording B/.test(e.message)); + assert.equal(shown(), 'mine', 'nothing applied'); + + await framework.puppetToolCall('ada', 'set_tool_description', reset); + const second = asked[1]!; + assert.ok(second.kind === 'tool-presentation'); + assert.equal(second.target.description, 'default wording B'); + await apply(second); + assert.equal(shown(), 'default wording B', 'exactly the approved wording'); + + // An edit the presentation would refuse is refused at staging, never staged. + const refused = await framework.puppetToolCall('ada', 'set_tool_description', { name: 'workspace--glob', description: 7 }); + assert.equal(refused.staged, undefined); + assert.match(String(refused.result.error), /couldn't be resolved .*Invalid description/); + assert.equal(asked.length, 2); + } finally { + console.log = quiet; + await framework.stop(); + rmSync(dir, { recursive: true, force: true }); + } + }); }); describe('operator-change gate: host/command undo by turns', () => { From e64e6e6dd6113ceea789b53dabf7150052e56515 Mon Sep 17 00:00:00 2001 From: Petra-1779 Date: Wed, 7 Oct 2026 20:16:12 -0400 Subject: [PATCH 56/73] fix(framework): resume() refuses while the active body is an unresolved cut's destination, force or not (Greptile #255 P1-1) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Greptile's first finding on #255 (discussion_r4213085182). Startup boots quiesced when an unresolved cut (no recorded switch, failure or operator verdict) left its destination as the active body: "no turn starts on a body nobody can vouch for" (June's #48836). But resume() checked only budget feasibility before clearing `quiesced`, so host.resume, /resume or host/command resume could open traffic on that body without anyone settling it. The contract Kit-1008, June-1016, Agnes-1009 and Theo-1778 agreed in room-222 (#50544, #50546, #50555, #50548): an ordinary resume isn't an attestation about an uncertain body, `force` stays a feasibility override, and a historical unresolved attempt whose source is active doesn't block. - **resume()** reads the unresolved cuts on the active body (unresolvedActiveBodies: unresolvedAttempt, target === active branch, the same condition startup quiesces on). It does so after every await in resume (the maintenance wait and preview retries), with none between that read and reopening, so a settlement that landed meanwhile counts. If any are found it throws ResumeBlockedError, whatever `force` says. - **ResumeBlockedError** gains `unresolved: UnresolvedActiveBody[]` (changeId, agent, kind, attempt, target) beside `verdicts`. `verdicts` carries the feasibility failures only when they block (no `force`). With both conditions there's one error carrying both lists, its message naming each. The message names both settlement routes: live `resolveOperatorChange("", , 'committed' | 'not-committed', { lease, reason })`, and offline `agent-framework-recover --store --operator-change resolve --attempt --verdict … --reason …` with the host stopped. A feasibility-only refusal reads exactly as before. - **Ingresses:** HTTP /resume already answered ResumeBlockedError with 409, and its body now carries `unresolved`. host/command resume already answered `ok:false`, and now carries `unresolved` too. UnresolvedActiveBody is exported. - Doc comments on resume(), ResumeBlockedError and reconcileOperatorChanges state the rule. For the host side (Agnes, room-222 #50639): a `startQuiesced` boot's coordinator surfaces a ResumeBlockedError with `unresolved` as "this framework stays quiesced" and doesn't retry until a resolution is recorded. Tests (operator-change-gate): - An unresolved attempt left its destination active (switch record lost), then restart. resume() and resume({force: true}) are each refused, with `unresolved` naming the change, attempt and destination, empty `verdicts`, and both routes in the message. It stays quiesced. host/command resume with force answers `ok:false` with the same `unresolved`. After resolveOperatorChange(…, 'committed'), resume() proceeds. - An unresolved attempt whose source is active (failure record lost): quiesce then resume proceeds. Mutants caught, each by exactly its own test: no check; `force` overriding it; any unresolved attempt blocking whatever its target. operator-change-gate, host-quiesce, api-admin-gate, overbudget-drain-breaker: 149/149. Co-Authored-By: Petra (Claude Opus 5.5) --- src/api/server.ts | 1 + src/framework.ts | 113 ++++++++++++++++++++++++------ src/index.ts | 2 +- test/operator-change-gate.test.ts | 55 ++++++++++++++- 4 files changed, 146 insertions(+), 25 deletions(-) diff --git a/src/api/server.ts b/src/api/server.ts index 51b7f0d9d..4a9c11f3b 100644 --- a/src/api/server.ts +++ b/src/api/server.ts @@ -1006,6 +1006,7 @@ export class ApiServer { respond(409, { error: error.message, verdicts: error.verdicts, + ...(error.unresolved.length > 0 ? { unresolved: error.unresolved } : {}), hostMode: this.framework.getHostModeStatus(), }); } else { diff --git a/src/framework.ts b/src/framework.ts index edb332d62..01a12f778 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -860,16 +860,35 @@ export interface SafeBoundaryOptions { signal?: AbortSignal; } +/** A cut whose latest attempt is unresolved (neither its switch nor its + * failure recorded, and no operator verdict) and whose destination is the + * active body: nobody can vouch for that body until an operator settles the + * attempt (resolveOperatorChange, or agent-framework-recover + * --operator-change resolve with the host stopped). */ +export interface UnresolvedActiveBody { + changeId: string; + agent: string; + kind: 'undo-turns' | 'undo-messages'; + attempt: number; + /** The attempt's destination, which is the active branch. */ + target: string; +} + /** - * Thrown by `resume()` when the current runtime settings do not compile for - * one or more agents — returning to service would OverBudget-wedge them on - * the first wake. Drain quarantine / advance merges to lower the floor, or - * pass `{ force: true }` after deciding the verdicts are acceptable. + * Thrown by `resume()` when returning to service isn't safe: + * - `verdicts`: the current runtime settings do not compile for one or more + * agents, so returning would OverBudget-wedge them on the first wake. Drain + * quarantine / advance merges to lower the floor, or pass `{ force: true }` + * after deciding the verdicts are acceptable. + * - `unresolved`: the active body is the destination of a cut whose attempt + * is unresolved. Resuming isn't an attestation and `force` doesn't + * override it: settle each attempt first. */ export class ResumeBlockedError extends Error { constructor( message: string, readonly verdicts: Array<{ agentName: string; preview: RuntimeSettingsPreview }>, + readonly unresolved: UnresolvedActiveBody[] = [], ) { super(message); this.name = 'ResumeBlockedError'; @@ -3626,6 +3645,13 @@ export class AgentFramework { * Throws ResumeBlockedError (with all failing verdicts) unless `force`. * Unavailable previews warn and pass — refusing to resume because the * strategy cannot preview would hold hosts hostage to a diagnostic. + * + * Also refuses, whatever `force` says, while the active body is the + * destination of a cut whose attempt is unresolved (the condition startup + * boots quiesced on): a resume isn't an operator's attestation about that + * body. The error's `unresolved` names each change and attempt; settle them + * with resolveOperatorChange, or offline. An unresolved attempt whose + * destination isn't the active body doesn't block. */ async resume(opts?: { force?: boolean }): Promise { if (!this.quiesced) return this.getHostModeStatus(); @@ -3655,20 +3681,37 @@ export class AgentFramework { ); } } - if (failing.length > 0 && !opts?.force) { - const detail = failing.map(({ agentName, preview }) => - `${agentName}: folded floor ${preview.effective!.finalTokens} > hard budget ` + - `${preview.effective!.budgetTokens}` + - (preview.transition === 'blocked' - ? ` (transition blocked: ${preview.transitionReason ?? 'unknown'})` - : ''), - ).join('; '); - throw new ResumeBlockedError( - `resume refused: returning to service would OverBudget-wedge — ${detail}. ` + - `Drain compression quarantine / advance the merge ladder to lower the floor ` + - `(maintenanceTick()), or resume({ force: true }).`, - failing, - ); + // Read after every await above, with none between here and reopening: a + // settlement that landed meanwhile counts, and nothing lands after. + const unresolved = this.unresolvedActiveBodies(); + const overBudget = failing.length > 0 && !opts?.force; + if (unresolved.length > 0 || overBudget) { + const reasons: string[] = []; + for (const u of unresolved) { + reasons.push( + `operator change ${u.changeId} (${u.kind} for ${u.agent}) is unresolved: its attempt ${u.attempt} left ` + + `${u.target} active with neither its switch nor its failure recorded. Resuming isn't an attestation, and ` + + `force doesn't override it: settle it first, live with resolveOperatorChange(${JSON.stringify(u.changeId)}, ` + + `${u.attempt}, 'committed' | 'not-committed', { lease, reason }), or with the host stopped, ` + + `agent-framework-recover --store --operator-change resolve ${u.changeId} --attempt ${u.attempt} ` + + `--verdict --reason `, + ); + } + if (overBudget) { + const detail = failing.map(({ agentName, preview }) => + `${agentName}: folded floor ${preview.effective!.finalTokens} > hard budget ` + + `${preview.effective!.budgetTokens}` + + (preview.transition === 'blocked' + ? ` (transition blocked: ${preview.transitionReason ?? 'unknown'})` + : ''), + ).join('; '); + reasons.push( + `returning to service would OverBudget-wedge — ${detail}. ` + + `Drain compression quarantine / advance the merge ladder to lower the floor ` + + `(maintenanceTick()), or resume({ force: true })`, + ); + } + throw new ResumeBlockedError(`resume refused: ${reasons.join('. Also: ')}.`, overBudget ? failing : [], unresolved); } if (failing.length > 0) { console.warn( @@ -5701,7 +5744,12 @@ export class AgentFramework { return { ok: true, hostMode }; } catch (error) { if (error instanceof ResumeBlockedError) { - return { ok: false, error: error.message, hostMode: this.getHostModeStatus() }; + return { + ok: false, + error: error.message, + ...(error.unresolved.length > 0 ? { unresolved: error.unresolved } : {}), + hostMode: this.getHostModeStatus(), + }; } throw error; } @@ -7303,6 +7351,25 @@ export class AgentFramework { return copy; } + /** The unresolved cuts whose destination is the active body: what startup + * boots quiesced on, and what resume() refuses until it is settled. */ + private unresolvedActiveBodies(): UnresolvedActiveBody[] { + const active = this.store.currentBranch().name; + const found: UnresolvedActiveBody[] = []; + for (const record of this.changesLedger().records.values()) { + const unresolved = this.unresolvedAttempt(record); + if (!unresolved || unresolved.target !== active) continue; + found.push({ + changeId: record.changeId, + agent: record.agent, + kind: record.kind as UnresolvedActiveBody['kind'], + attempt: unresolved.n, + target: unresolved.target, + }); + } + return found; + } + /** A cut whose latest attempt has neither its switch nor its failure * recorded, and wasn't seen by this process: whether it applied is * unknown, whatever its destination's state. */ @@ -8382,10 +8449,10 @@ export class AgentFramework { * cut's destination being active or written to proves nothing: a cut with * no recorded switch, failure or operator verdict is unresolved, never * certified, and if its destination is the active body, startup boots - * quiesced until an operator settles it. This runs before modules, - * connections or traffic, so a committed body never waits on its host to - * settle its marks. Anything else not provably committed is left for the - * host's retry. + * quiesced, and resume() refuses, until an operator settles it. This runs + * before modules, connections or traffic, so a committed body never waits + * on its host to settle its marks. Anything else not provably committed is + * left for the host's retry. */ private async reconcileOperatorChanges(): Promise { // Both journals were first read before the agents (an unreadable one diff --git a/src/index.ts b/src/index.ts index 944862085..f9671ba2b 100644 --- a/src/index.ts +++ b/src/index.ts @@ -3,7 +3,7 @@ export * from './types/index.js'; // Core classes export { AgentFramework, BudgetPreflightError, ResumeBlockedError } from './framework.js'; -export type { RuntimeSettingsPreview, HostModeStatus, SafeBoundaryLease, SafeBoundaryOptions } from './framework.js'; +export type { RuntimeSettingsPreview, HostModeStatus, SafeBoundaryLease, SafeBoundaryOptions, UnresolvedActiveBody } from './framework.js'; export { OperatorJournalUnreadableError, OperatorChangeRecoveryError } from './framework.js'; export type { OperatorChangeReceipt, diff --git a/test/operator-change-gate.test.ts b/test/operator-change-gate.test.ts index ebbc6b22c..196dd0071 100644 --- a/test/operator-change-gate.test.ts +++ b/test/operator-change-gate.test.ts @@ -18,7 +18,7 @@ import { spawnSync } from 'node:child_process'; import { fileURLToPath } from 'node:url'; import { JsStore } from '@animalabs/chronicle'; import { join } from 'node:path'; -import { AgentFramework, AutobiographicalStrategy, WorkspaceModule } from '../src/index.js'; +import { AgentFramework, AutobiographicalStrategy, ResumeBlockedError, WorkspaceModule } from '../src/index.js'; import type { AgentSettingsExtension, InferenceRequest, @@ -1269,6 +1269,59 @@ describe('operator-change gate: host/command undo by turns', () => { assert.deepEqual([record.switched, record.attempts[0]!.resolution!.verdict], [undefined, 'committed'], 'an attestation, not a framework-observed switch'); }); + it('refuses resume(), forced or not and through host/command, while the active body is an unresolved cut\'s destination, until it is settled', async () => { + await say('a0'); await say('a1'); await say('a2'); + await undoWithMarks(2, 'all'); + const change = asked[0]!; + const undoRecord = failJournalOnce('switched'); + try { + await assert.rejects(applyIt(change), /injected switched failure/); + } finally { + undoRecord(); + } + await reopen(); + assert.equal(quiesced(), true); + const held = [{ changeId: change.id, agent: 'scout', kind: 'undo-turns', attempt: 1, target: `undo/scout/op-${change.id}` }]; + for (const opts of [undefined, { force: true }]) { + await assert.rejects(framework.resume(opts), (e: unknown) => { + assert.ok(e instanceof ResumeBlockedError); + assert.deepEqual([e.unresolved, e.verdicts], [held, []]); + assert.match(e.message, new RegExp(`operator change ${change.id} \\(undo-turns for scout\\) is unresolved: its attempt 1`)); + assert.ok(e.message.includes(`resolveOperatorChange("${change.id}", 1, 'committed' | 'not-committed', { lease, reason })`)); + assert.ok(e.message.includes(`--operator-change resolve ${change.id} --attempt 1 --verdict --reason `)); + assert.match(e.message, /force doesn't override it/); + return true; + }); + assert.equal(quiesced(), true, `still quiesced after resume(${JSON.stringify(opts ?? {})})`); + } + const viaCommand = await host().handleHostCommand('discord', { command: 'resume', force: true, requesterName: 'nissa' }); + assert.deepEqual([viaCommand.ok, viaCommand.unresolved], [false, held], 'a refusal the host can read, not a throw'); + assert.equal(quiesced(), true); + + await resolve(change.id, 1, 'committed', 'the switch completed; its record was lost'); + const status = await framework.resume(); + assert.equal(status.quiesced, false, 'settled: resume proceeds'); + }); + + it('lets resume() proceed while an unresolved attempt left its source, not its destination, active', async () => { + await say('m0'); await say('m1'); + await host().handleHostCommand('discord', { command: 'undo', agentName: 'scout', messages: 2, marks: 'all', requesterName: 'nissa' }); + const change = asked[0]!; + const undoCut = failCutAfterInitializerWrite(); + const undoRecord = failJournalOnce('failed'); + try { + await assert.rejects(applyIt(change), /injected strategy initialization failure/); + } finally { + undoCut(); undoRecord(); + } + await reopen(); + assert.equal(branch(), change.sourceBranch); + assert.ok(framework.getOperatorChangeRecord(change.id)!.unresolved, 'unresolved, on a branch that is not active'); + await framework.quiesce({ reason: 'maintenance' }); + const status = await framework.resume(); + assert.equal(status.quiesced, false, 'a historical unresolved attempt does not hold the known body'); + }); + it("restores a recorded failure's source at startup, before any traffic, when the process died before restoring it", async () => { await say('m0'); await say('m1'); await host().handleHostCommand('discord', { command: 'undo', agentName: 'scout', messages: 2, marks: 'none', requesterName: 'nissa' }); From f4716ab79d33bf834bce11b86d6cf57a2abe830c Mon Sep 17 00:00:00 2001 From: Petra-1779 Date: Wed, 7 Oct 2026 20:24:32 -0400 Subject: [PATCH 57/73] fix(framework): the unstick journal reader refuses what its readers can't interpret, in checkpoints, entries and the reduced ledger (Greptile #255 P2-6) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Greptile's sixth finding on #255 (discussion_r4213085212). readUnstickJournal (operator/unstick) checked a checkpoint's operations only for `typeof status === 'string'`. A `completed` attempt without its `outcome` was installed, and rerunUnstick then reported `{status: 'completed', outcome: undefined}` to the host (`existing.outcome!`). Any status string passed. Message ids weren't checked as strings in the checkpoint or in step-intent entries, and entries naming an operation, step or attempt the ledger never held were folded in silently. Theo-1778 asked that the reduced ledger be checked as well as the input entries, so both representations enforce one contract. The reader now mirrors readOperatorChanges (#50095's model, which Kit and June qualified) and validates the ledger as applyUnstickStep and rerunUnstick read it: - **Each operation, whole** (checkpoint records, then every reduced record again after the entries fold in): - operationId matching its key, a non-empty agent; - steps with a positive integer step, unique, status `intent` or `shed`, and non-empty string message ids; - attempts with a unique positive integer step, either `launched` with no outcome, category or error, or `completed` with an outcome of responded, refused or failed and string category and error when present. - **Each entry, its own shape** (positive integer step; step-intent with an agent and string ids; attempt-launched with an agent; attempt-done with a valid outcome and string details) **and its references:** - step-done names a step the ledger holds; - attempt-launched names a shed step, the only one rerunUnstick launches; - attempt-done names an attempt it holds; - an entry's agent matches its operation's. Every legitimate journal satisfies these: records reach the OS in append order before append returns, and each writer appends only after the in-memory ledger shows its predecessor (the intent before its done, a shed step before its launch, a launch before its outcome). A real journal is always a prefix of a valid sequence. Anything else is refused with "malformed operator/unstick …". It reaches the same load-first paths, OperatorJournalUnreadableError at startup and on every use mid-run, as unreadable bytes do. The final whole-ledger pass is defence in depth: with the entry checks in place, no entry sequence reduces to a malformed record, so removing it leaves an equivalent mutant. Tests (operator-change-gate, "operator journals that cannot be read"): - Nine malformed checkpoint operations refused at startup: - completed without outcome (Greptile's case); - an unknown outcome; - an unknown attempt status; - launched carrying an outcome; - a non-string category; - duplicate attempts; - an unknown step status; - non-string ids; - a fractional step. - Six malformed or dangling entries refused: non-string ids, step-done without a step, a launch of an unshed step, attempt-done without an attempt, a non-string error, and an agent differing from its operation's. - A well-formed ledger (checkpoint plus entries) still reads exactly as before. Mutants: the previous reader fails both refusal tests and passes the well-formed one; removing the reference checks is caught. operator-change-gate, inference-exhaustion-rewind, record-journal, recover-cli-operator-log: 125/125. Co-Authored-By: Petra (Claude Opus 5.5) --- src/framework.ts | 77 +++++++++++++++++++++++++------ test/operator-change-gate.test.ts | 63 +++++++++++++++++++++++++ 2 files changed, 125 insertions(+), 15 deletions(-) diff --git a/src/framework.ts b/src/framework.ts index 01a12f778..d7b7b129f 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -613,36 +613,83 @@ type UnstickJournalSnapshot = Record; /** * Read the unstick journal into its operations, refusing anything parseable - * that can't be interpreted as this ledger (a null-snapshot checkpoint, or - * a record or entry missing what it requires), exactly as unreadable bytes. + * that can't be interpreted as this ledger, exactly as unreadable bytes are + * refused: a checkpoint whose snapshot isn't an operation map (a null one + * included), an operation or entry missing or misstating what its kind + * requires, or an entry naming an operation, step or attempt the ledger + * doesn't hold. Every operation is checked whole again once the entries are + * folded in, so the ledger is validated as applyUnstickStep and rerunUnstick + * read it: a step is an intent or shed with its message ids, and an attempt + * is launched with no outcome, or completed with one. */ function readUnstickJournal(load: { snapshot: unknown; entries: Array<{ entry: unknown }>; checkpointed: boolean }): Map { - const malformed = (what: string): never => { throw new Error(`malformed operator/unstick ${what}`); }; + const malformed: (what: string) => never = (what) => { throw new Error(`malformed operator/unstick ${what}`); }; const isObject = (v: unknown): v is Record => typeof v === 'object' && v !== null && !Array.isArray(v); - const isCount = (v: unknown): v is number => typeof v === 'number' && Number.isFinite(v); + const isText = (v: unknown): v is string => typeof v === 'string' && v.length > 0; + const isStep = (v: unknown): v is number => Number.isSafeInteger(v) && (v as number) >= 1; + const isIds = (v: unknown): v is string[] => Array.isArray(v) && v.every(isText); + const isOutcome = (v: unknown): boolean => v === 'responded' || v === 'refused' || v === 'failed'; + const isDetail = (r: Record): boolean => + (r.category === undefined || typeof r.category === 'string') && (r.error === undefined || typeof r.error === 'string'); + const checkOperation: (op: unknown, id: string) => void = (op, id) => { + const where = `operation ${id}`; + if (!isObject(op) || op.operationId !== id || !isText(op.agent) || !Array.isArray(op.steps) || !Array.isArray(op.attempts)) { + malformed(where); + } + const steps = new Set(); + for (const s of op.steps) { + if (!isObject(s) || !isStep(s.step) || steps.has(s.step) || (s.status !== 'intent' && s.status !== 'shed') || !isIds(s.messageIds)) { + malformed(`${where}: a step`); + } + steps.add(s.step); + } + const attempts = new Set(); + for (const a of op.attempts) { + if (!isObject(a) || !isStep(a.step) || attempts.has(a.step)) malformed(`${where}: an attempt`); + attempts.add(a.step); + const ok = a.status === 'launched' + ? a.outcome === undefined && a.category === undefined && a.error === undefined + : a.status === 'completed' && isOutcome(a.outcome) && isDetail(a); + if (!ok) malformed(`${where}: attempt ${a.step} (launched with no outcome, or completed with one)`); + } + }; const ops = new Map(); if (load.checkpointed) { const snapshot = load.snapshot; if (!isObject(snapshot)) throw new Error('malformed operator/unstick checkpoint: its snapshot is not an operation map'); for (const [id, op] of Object.entries(snapshot)) { - if (!isObject(op) || op.operationId !== id || typeof op.agent !== 'string' || !Array.isArray(op.steps) || !Array.isArray(op.attempts) - || !op.steps.every((x) => isObject(x) && isCount(x.step) && typeof x.status === 'string' && Array.isArray(x.messageIds)) - || !op.attempts.every((x) => isObject(x) && isCount(x.step) && typeof x.status === 'string')) { - malformed(`operation ${id}`); - } + checkOperation(op, id); ops.set(id, structuredClone(op as unknown as UnstickOperationRecord)); } } for (const { entry } of load.entries) { const e = entry as Record; - const ok = isObject(e) && typeof e.operationId === 'string' && e.operationId.length > 0 && isCount(e.step) && ( - (e.kind === 'step-intent' && typeof e.agent === 'string' && Array.isArray(e.messageIds)) - || e.kind === 'step-done' - || (e.kind === 'attempt-launched' && typeof e.agent === 'string') - || (e.kind === 'attempt-done' && ['responded', 'refused', 'failed'].includes(e.outcome as string))); - if (!ok) malformed(`entry ${isObject(e) ? String(e.kind) : typeof e}`); + if (!isObject(e) || !isText(e.operationId) || !isStep(e.step)) malformed(`entry ${isObject(e) ? String(e.kind) : typeof e}`); + const where = `${String(e.kind)} entry for ${e.operationId} step ${e.step}`; + const op = ops.get(e.operationId); + switch (e.kind) { + case 'step-intent': + if (!isText(e.agent) || !isIds(e.messageIds)) malformed(where); + break; + case 'step-done': + if (!op?.steps.some((s) => s.step === e.step)) malformed(`${where}: no such step`); + break; + case 'attempt-launched': + if (!isText(e.agent)) malformed(where); + if (!op?.steps.some((s) => s.step === e.step && s.status === 'shed')) malformed(`${where}: its step was never shed`); + break; + case 'attempt-done': + if (!isOutcome(e.outcome) || !isDetail(e)) malformed(where); + if (!op?.attempts.some((a) => a.step === e.step)) malformed(`${where}: no such attempt`); + break; + default: + malformed(where); + } + if ('agent' in e && op && op.agent !== e.agent) malformed(`${where}: its agent differs from its operation's`); reduceUnstickEntry(ops, entry as UnstickJournalEntry); } + // The ledger as it will be read: every operation whole. + for (const [id, op] of ops) checkOperation(op, id); return ops; } diff --git a/test/operator-change-gate.test.ts b/test/operator-change-gate.test.ts index 196dd0071..a81dea60a 100644 --- a/test/operator-change-gate.test.ts +++ b/test/operator-change-gate.test.ts @@ -2074,6 +2074,69 @@ describe('operator journals that cannot be read', () => { await assert.rejects(startOn(store), refusedAs('operator/unstick')); }); + // operator/unstick, read as applyUnstickStep and rerunUnstick read it. + const unstickStore = (snapshot: object | undefined, entries: object[]) => { + const store = JsStore.openOrCreate({ path: join(dir, `unstick-${++ledgers}`) }); + if (snapshot) { + const covered = store.appendJson('operator/unstick', { kind: 'step-intent', operationId: 'covered', agent: 'scout', step: 1, messageIds: ['c'] }); + store.appendJson('operator/unstick/checkpoint', { through: String(covered.id), snapshot }); + } + for (const entry of entries) store.appendJson('operator/unstick', entry); + store.sync(); + return store; + }; + const unstickRefuses = async (snapshot: object | undefined, entries: object[], why: RegExp) => { + const store = unstickStore(snapshot, entries); + try { + await assert.rejects(startOn(store), (e: Error) => refusedAs('operator/unstick')(e) && why.test(e.message)); + } finally { + store.close(); + } + }; + const operation = (attempts: object[], steps: object[] = [{ step: 1, status: 'shed', messageIds: ['m1', 'm2'] }]) => + ({ op: { operationId: 'op', agent: 'scout', steps, attempts } }); + const intent = (step: number, agent = 'scout') => ({ kind: 'step-intent', operationId: 'op', agent, step, messageIds: [`m${step}`] }); + const shed = (step: number) => ({ kind: 'step-done', operationId: 'op', step }); + const launched = (step: number) => ({ kind: 'attempt-launched', operationId: 'op', agent: 'scout', step }); + + it('refuses an unstick checkpoint whose attempt or step misstates what it records, rather than reporting an outcome it lacks', async () => { + await unstickRefuses(operation([{ step: 1, status: 'completed' }]), [], /operation op: attempt 1 \(launched with no outcome, or completed with one\)/); + await unstickRefuses(operation([{ step: 1, status: 'completed', outcome: 'maybe' }]), [], /attempt 1/); + await unstickRefuses(operation([{ step: 1, status: 'done', outcome: 'responded' }]), [], /attempt 1/); + await unstickRefuses(operation([{ step: 1, status: 'launched', outcome: 'refused' }]), [], /attempt 1/); + await unstickRefuses(operation([{ step: 1, status: 'completed', outcome: 'refused', category: 7 }]), [], /attempt 1/); + await unstickRefuses(operation([{ step: 1, status: 'launched' }, { step: 1, status: 'launched' }]), [], /an attempt/); + await unstickRefuses(operation([], [{ step: 1, status: 'finished', messageIds: ['m1'] }]), [], /a step/); + await unstickRefuses(operation([], [{ step: 1, status: 'shed', messageIds: [7] }]), [], /a step/); + await unstickRefuses(operation([], [{ step: 0.5, status: 'shed', messageIds: ['m1'] }]), [], /a step/); + }); + + it('refuses unstick entries that misstate their ids, or name a step or attempt the ledger never held', async () => { + await unstickRefuses(undefined, [{ ...intent(1), messageIds: [1, 2] }], /step-intent entry for op step 1/); + await unstickRefuses(undefined, [shed(1)], /no such step/); + await unstickRefuses(undefined, [intent(1), launched(1)], /its step was never shed/); + await unstickRefuses(undefined, [intent(1), shed(1), { kind: 'attempt-done', operationId: 'op', step: 1, outcome: 'refused' }], /no such attempt/); + await unstickRefuses(undefined, [intent(1), shed(1), launched(1), { kind: 'attempt-done', operationId: 'op', step: 1, outcome: 'refused', error: {} }], /attempt-done entry/); + await unstickRefuses(undefined, [intent(1), intent(2, 'someone-else')], /its agent differs/); + }); + + it('reads a well-formed unstick ledger, its checkpoint and the entries after it, as before', async () => { + const store = unstickStore(operation([{ step: 1, status: 'completed', outcome: 'refused', category: 'safety' }]), + [intent(2), shed(2), launched(2), { kind: 'attempt-done', operationId: 'op', step: 2, outcome: 'responded' }]); + const framework = await startOn(store); + try { + assert.deepEqual(framework.getUnstickOperation('op'), { + operationId: 'op', + agent: 'scout', + steps: [{ step: 1, status: 'shed', messageIds: ['m1', 'm2'] }, { step: 2, status: 'shed', messageIds: ['m2'] }], + attempts: [{ step: 1, status: 'completed', outcome: 'refused', category: 'safety' }, { step: 2, status: 'completed', outcome: 'responded' }], + }); + } finally { + await framework.stop(); + store.close(); + } + }); + it('refuses a malformed ledger offline as well', () => { const path = join(dir, 'offline'); const store = JsStore.openOrCreate({ path }); From 5de55ca74a4cc78a6d71924c3159e140c01c4088 Mon Sep 17 00:00:00 2001 From: Petra-1779 Date: Wed, 7 Oct 2026 20:32:36 -0400 Subject: [PATCH 58/73] fix(framework): an abandoned destination left active is held until its source is restored; resume() refuses it, and restoreOperatorChangeSource restores it live (Greptile #255 P1-1, review) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Theo-1778's finding in review of e64e6e6 (room-272 #50951, room-222 #50954). Startup boots quiesced on an unresolved active destination. A not-committed resolution then records its verdict, and if restoring the source fails, the receipt says so in `remaining` while the abandoned destination stays active. unresolvedAttempt then returned null, so resume() opened traffic on a body nobody approved. He reproduced it from the lost-switch regression with `cm.switchBranch` failing. A second live path never paused at all: when applyCut's attempt fails and restoreSourceBranch fails too, the host is told `failed`, but once the lease released, turns ran on the abandoned destination. Agreed shape in room-222 (#50958 Agnes, #50965/#50996 Theo, #51006 Kit, #50992 the design). **One predicate, two conditions.** activeBodyHold(record): a cut record with no outcome and not dropped, whose latest attempt's destination is the active branch and didn't commit. The disposition is the live framework's own (cutAttemptDisposition with this process's cutKnowledge), the same one boot's restoreAbandonedCutSources uses for abandoned cuts. - **unresolved** (no switch, failure or verdict): an attestation is needed, live or offline. Unchanged from e64e6e6. - **abandoned** (failure recorded or seen, or a not-committed verdict): restoration is needed. The verdict is recorded, so no attestation is asked for. A committed attempt, a known source active, or another branch active clears both. **resume()** refuses either, whatever `force` says. ResumeBlockedError gains `restorationRequired: AbandonedActiveBody[]` (changeId, agent, kind, attempt, target, source) beside `unresolved`. Its message names the routes and never asks for another attestation: restoreOperatorChangeSource(…, { lease }), a restart (startup restores the source before any agent initializes), or moving the host to a known-safe body. HTTP /resume 409 and host/command `ok:false` carry the list. **restoreOperatorChangeSource(changeId, { lease, requester? })**, the live route. Under the held lease: - it refuses (`invalid`, "nothing to restore") unless that change holds the active body as abandoned; - it restores through restoreSourceBranch and the config-mount refresh, as the resolver does; - it returns OperatorChangeRestorationReceipt `{ changeId, attempt, restored? , remaining? }`; - it's logged as `restore-operator-change-source`. It carries out the recorded verdict and attests nothing new, so it takes no reason. **Traffic is held when the framework itself leaves an abandoned destination active** (holdAbandonedBody). This happens when: - a failed attempt's restore fails (applyCut's catch); - a not-committed resolution's restore fails; - the host's retry of an abandoned change finds the restore failing again; - a live restoration fails again. The framework then quiesces in memory as startup holds an unresolved body: wakes park, writes defer, data planes pause, and an ops alert (`operator-change-restoration-required`) names the change. The hold doesn't depend on anyone calling resume(). It is decided from what the caller saw, without rereading the journal, so a journal that just failed can't prevent it. It isn't persisted: the next start restores the source or refuses to start. A host-initiated quiesce keeps its own reason. **The host's retry of an abandoned change** whose restore fails again now refuses `failed` ("restoring its source … failed again; traffic is held until it is restored"). Before, it was a misleading `stale` ("the active branch moved"), which invited restaging against the abandoned body. **Receipts tell only the truth about restoration.** OperatorChangeResolutionReceipt's `restored` is now set only when the source was restored, and then carries its name. A failure is named in `remaining` instead of a failure sentence under a success-shaped field (Theo, #50996). The new restoration receipt follows the same rule. Both receipt types are exported, with AbandonedActiveBody. **Scope.** The predicate and receipts refer to R2's journaled operations. The unjournaled direct surgeries' own "restore also failed" paths have no disposition to consult. This change makes no claim about them (Kit, #51006). Tests (operator-change-gate): - **Theo's state:** lost switch, restart, then a not-committed resolution with every switch failing. - The receipt has no `restored`, and its `remaining` names the source. - resume() and resume({force}) are refused with `restorationRequired` (destination and source), empty `unresolved`, and no attestation route in the message. - host/command answers the same. - restoreOperatorChangeSource restores the source and is logged; a second call is refused as having nothing to restore; resume proceeds. - **The live path:** a failed cut whose restore fails leaves the framework quiesced by itself. - A queued wake makes no provider call after the lease released. - resume({force}) is refused. - The host's retry and a live restoration, with switches still failing, each fail again and keep the hold. - A live restoration then succeeds, resume proceeds, and the parked wake runs on the source body. Mutants caught: resume ignoring restorationRequired; no hold after a failed cut's restore; `restored` carrying the failure; the restore route without its predicate. The hold in resolveOperatorChange is defence in depth: an active unresolved destination is always already held by startup when it is resolved. It also raises the restoration alert. operator-change-gate, host-quiesce, api-admin-gate, overbudget-drain-breaker, recover-cli-operator-log: 157/157. Co-Authored-By: Petra (Claude Opus 5.5) --- src/api/server.ts | 1 + src/framework.ts | 208 ++++++++++++++++++++++++++---- src/index.ts | 4 +- src/operator-change.ts | 15 ++- test/operator-change-gate.test.ts | 109 ++++++++++++++++ 5 files changed, 307 insertions(+), 30 deletions(-) diff --git a/src/api/server.ts b/src/api/server.ts index 4a9c11f3b..be71277cc 100644 --- a/src/api/server.ts +++ b/src/api/server.ts @@ -1007,6 +1007,7 @@ export class ApiServer { error: error.message, verdicts: error.verdicts, ...(error.unresolved.length > 0 ? { unresolved: error.unresolved } : {}), + ...(error.restorationRequired.length > 0 ? { restorationRequired: error.restorationRequired } : {}), hostMode: this.framework.getHostModeStatus(), }); } else { diff --git a/src/framework.ts b/src/framework.ts index d7b7b129f..4f52f2364 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -28,6 +28,7 @@ import { type OperatorChangeEvidence, type OperatorChangeResolution, type OperatorChangeResolutionReceipt, + type OperatorChangeRestorationReceipt, type OperatorChangeOutcome, type OperatorChangeRecord, type ResolvedHideChange, @@ -921,6 +922,24 @@ export interface UnresolvedActiveBody { target: string; } +/** A cut whose latest attempt was abandoned (not committed: its failure + * recorded or seen, or an operator's not-committed verdict) while its + * destination is still the active body, because restoring its source + * failed: a body nobody approved. Its source must be restored + * (restoreOperatorChangeSource, or a restart) or the host moved to a + * known-safe body; the verdict is already recorded, so no attestation is + * needed. */ +export interface AbandonedActiveBody { + changeId: string; + agent: string; + kind: 'undo-turns' | 'undo-messages'; + attempt: number; + /** The abandoned destination, which is the active branch. */ + target: string; + /** The branch to restore. */ + source: string; +} + /** * Thrown by `resume()` when returning to service isn't safe: * - `verdicts`: the current runtime settings do not compile for one or more @@ -930,12 +949,16 @@ export interface UnresolvedActiveBody { * - `unresolved`: the active body is the destination of a cut whose attempt * is unresolved. Resuming isn't an attestation and `force` doesn't * override it: settle each attempt first. + * - `restorationRequired`: the active body is the destination of an + * abandoned attempt whose source couldn't be restored. `force` doesn't + * override it either: restore the source, or move to a known-safe body. */ export class ResumeBlockedError extends Error { constructor( message: string, readonly verdicts: Array<{ agentName: string; preview: RuntimeSettingsPreview }>, readonly unresolved: UnresolvedActiveBody[] = [], + readonly restorationRequired: AbandonedActiveBody[] = [], ) { super(message); this.name = 'ResumeBlockedError'; @@ -3694,11 +3717,15 @@ export class AgentFramework { * strategy cannot preview would hold hosts hostage to a diagnostic. * * Also refuses, whatever `force` says, while the active body is the - * destination of a cut whose attempt is unresolved (the condition startup - * boots quiesced on): a resume isn't an operator's attestation about that - * body. The error's `unresolved` names each change and attempt; settle them - * with resolveOperatorChange, or offline. An unresolved attempt whose - * destination isn't the active body doesn't block. + * destination of a cut attempt that didn't commit: + * - unresolved (the condition startup boots quiesced on): a resume isn't an + * operator's attestation about that body. The error's `unresolved` names + * each change and attempt; settle them with resolveOperatorChange, or + * offline. + * - abandoned, its source not restored (traffic is held when that + * happens): the error's `restorationRequired` names each; restore it with + * restoreOperatorChangeSource, restart, or move to a known-safe body. + * An attempt whose destination isn't the active body doesn't block. */ async resume(opts?: { force?: boolean }): Promise { if (!this.quiesced) return this.getHostModeStatus(); @@ -3729,10 +3756,11 @@ export class AgentFramework { } } // Read after every await above, with none between here and reopening: a - // settlement that landed meanwhile counts, and nothing lands after. - const unresolved = this.unresolvedActiveBodies(); + // settlement or restoration that landed meanwhile counts, and nothing + // lands after. + const { unresolved, restorationRequired } = this.heldActiveBodies(); const overBudget = failing.length > 0 && !opts?.force; - if (unresolved.length > 0 || overBudget) { + if (unresolved.length > 0 || restorationRequired.length > 0 || overBudget) { const reasons: string[] = []; for (const u of unresolved) { reasons.push( @@ -3744,6 +3772,15 @@ export class AgentFramework { `--verdict --reason `, ); } + for (const a of restorationRequired) { + reasons.push( + `operator change ${a.changeId} (${a.kind} for ${a.agent}) was abandoned: its attempt ${a.attempt} didn't ` + + `commit, but its destination ${a.target} is still the active body, and its source ${a.source} must be ` + + `restored before traffic. Its verdict is recorded, and force doesn't override this: restore it live with ` + + `restoreOperatorChangeSource(${JSON.stringify(a.changeId)}, { lease }), restart the host (startup restores ` + + `the source before any agent initializes), or move the host to a known-safe body`, + ); + } if (overBudget) { const detail = failing.map(({ agentName, preview }) => `${agentName}: folded floor ${preview.effective!.finalTokens} > hard budget ` + @@ -3758,7 +3795,12 @@ export class AgentFramework { `(maintenanceTick()), or resume({ force: true })`, ); } - throw new ResumeBlockedError(`resume refused: ${reasons.join('. Also: ')}.`, overBudget ? failing : [], unresolved); + throw new ResumeBlockedError( + `resume refused: ${reasons.join('. Also: ')}.`, + overBudget ? failing : [], + unresolved, + restorationRequired, + ); } if (failing.length > 0) { console.warn( @@ -5795,6 +5837,7 @@ export class AgentFramework { ok: false, error: error.message, ...(error.unresolved.length > 0 ? { unresolved: error.unresolved } : {}), + ...(error.restorationRequired.length > 0 ? { restorationRequired: error.restorationRequired } : {}), hostMode: this.getHostModeStatus(), }; } @@ -7398,23 +7441,113 @@ export class AgentFramework { return copy; } - /** The unresolved cuts whose destination is the active body: what startup - * boots quiesced on, and what resume() refuses until it is settled. */ - private unresolvedActiveBodies(): UnresolvedActiveBody[] { - const active = this.store.currentBranch().name; - const found: UnresolvedActiveBody[] = []; + /** Whether a cut's latest attempt holds the active body: its destination + * is the active branch and the attempt didn't commit, either unresolved + * (nobody can vouch for that body) or abandoned with its source not + * restored (a body nobody approved). Decided as everywhere else, by the + * record and what this process saw (cutAttemptDisposition). */ + private activeBodyHold(record: OperatorChangeRecord): { held: 'unresolved' | 'abandoned'; n: number; target: string; source: string } | null { + if (record.outcome || record.dropped || (record.kind !== 'undo-turns' && record.kind !== 'undo-messages')) return null; + const last = record.attempts[record.attempts.length - 1]; + if (!last || last.evidence.target !== this.store.currentBranch().name) return null; + const disposition = cutAttemptDisposition(record, last, this.cutKnowledge.get(record.changeId)?.get(last.n)); + if (disposition === 'committed') return null; + return { + held: disposition === 'unresolved' ? 'unresolved' : 'abandoned', + n: last.n, + target: last.evidence.target, + source: last.evidence.source, + }; + } + + /** The cuts holding the active body: what resume() refuses, startup boots + * quiesced on (unresolved) or restores before any agent initializes + * (abandoned), and a failed restoration holds traffic on. */ + private heldActiveBodies(): { unresolved: UnresolvedActiveBody[]; restorationRequired: AbandonedActiveBody[] } { + const unresolved: UnresolvedActiveBody[] = []; + const restorationRequired: AbandonedActiveBody[] = []; for (const record of this.changesLedger().records.values()) { - const unresolved = this.unresolvedAttempt(record); - if (!unresolved || unresolved.target !== active) continue; - found.push({ - changeId: record.changeId, - agent: record.agent, - kind: record.kind as UnresolvedActiveBody['kind'], - attempt: unresolved.n, - target: unresolved.target, - }); + const hold = this.activeBodyHold(record); + if (!hold) continue; + const body = { changeId: record.changeId, agent: record.agent, kind: record.kind as UnresolvedActiveBody['kind'], attempt: hold.n, target: hold.target }; + if (hold.held === 'unresolved') unresolved.push(body); + else restorationRequired.push({ ...body, source: hold.source }); + } + return { unresolved, restorationRequired }; + } + + /** + * Hold traffic when restoring an abandoned attempt's source failed and its + * destination is still the active body, a body nobody approved: the + * framework quiesces in memory, as startup holds an unresolved active body + * (wakes park, writes defer, data planes pause), so the hold doesn't + * depend on anyone calling resume(). resume() then refuses until the + * source is restored (restoreOperatorChangeSource, or a restart, which + * restores it before any agent initializes) or the host moves to a + * known-safe body. Not persisted: the next start decides afresh. Decided + * from what the caller saw, without rereading the journal, so a journal + * that just failed can't keep the hold from being taken. + */ + private holdAbandonedBody(abandoned: { changeId: string; agent: string; attempt: number; target: string; source: string }): void { + if (this.store.currentBranch().name !== abandoned.target) return; + const { changeId, agent } = abandoned; + const reason = `operator change ${changeId} was abandoned, but its attempt ${abandoned.attempt} left ${abandoned.target} ` + + `active: its source ${abandoned.source} could not be restored`; + if (!this.quiesced) { + this.quiesced = true; + this.quiesceReason = reason; + this.quiescedAt = Date.now(); + this.eventGate?.setQuiesced(true); + for (const connection of this.mcplServerRegistry?.getAllServers() ?? []) connection.pauseDataPlane(); } - return found; + this.opsAlert( + 'operator-change-restoration-required', + agent, + `${reason}; traffic is held until it is restored (restoreOperatorChangeSource, or a restart) and the host resumes`, + ); + } + + /** + * Restore the source of an abandoned cut whose destination is still the + * active body, under the held lease: its latest attempt didn't commit (its + * failure recorded or seen, or an operator's not-committed verdict), and + * restoring its source failed when it failed or was settled. It carries + * out what is already recorded, so it takes no verdict; a fresh attempt + * stays the host's decision on its next retry. Refuses unless that holds. + * If restoring fails again, traffic stays held and the receipt says so. + */ + async restoreOperatorChangeSource( + changeId: string, + opts: { lease: SafeBoundaryLease; requester?: OperatorRequester }, + ): Promise { + this.heldLeaseTokens(opts.lease, `restore the source of operator change ${changeId}`); + const record = this.changesLedger().records.get(changeId); + const hold = record ? this.activeBodyHold(record) : null; + if (!record || hold?.held !== 'abandoned') { + throw new OperatorActionError( + 'invalid', + `Operator change ${changeId} has no abandoned attempt whose destination is the active body; there is nothing to restore`, + ); + } + const agent = this.agents.get(record.agent); + if (!agent) throw new OperatorActionError('unknown-agent', `Unknown agent: ${record.agent}`); + const outcome = await this.restoreSourceBranch(agent.getContextManager(), hold.source, hold.target, record.agent); + this.materializeConfigMountAfterBranchSwitch(); + const failed = this.store.currentBranch().name === hold.target; + if (failed) this.holdAbandonedBody({ changeId, agent: record.agent, attempt: hold.n, target: hold.target, source: hold.source }); + const receipt: OperatorChangeRestorationReceipt = { + changeId, + attempt: hold.n, + ...(failed ? { remaining: `the source ${hold.source} could not be restored: ${outcome}` } : { restored: hold.source }), + }; + this.recordOperatorAction({ + kind: 'restore-operator-change-source', + agent: record.agent, + ...(opts.requester ? { requester: opts.requester } : {}), + params: { changeId, attempt: hold.n }, + result: { ...(receipt.restored ? { restored: receipt.restored } : {}), ...(receipt.remaining ? { remaining: receipt.remaining } : {}) }, + }); + return receipt; } /** A cut whose latest attempt has neither its switch nor its failure @@ -7433,7 +7566,9 @@ export class AgentFramework { * attempt), distinct from anything the framework observed. * - `not-committed` abandons the attempt and restores the source if its * destination is the active body. A fresh attempt, under the same staging - * authorization, is the host's decision on its next retry. + * authorization, is the host's decision on its next retry. If restoring + * fails, the verdict stands, traffic is held, and the receipt's + * `remaining` says so: restoreOperatorChangeSource retries it. * - `committed` attests that the attempt applied. Its outcome is * established, and its staged marks activated, on the host's next retry * (or at the next start), never by this call. @@ -7463,9 +7598,16 @@ export class AgentFramework { if (verdict === 'not-committed' && this.store.currentBranch().name === target.target) { const agent = this.agents.get(record.agent); if (!agent) throw new OperatorActionError('unknown-agent', `Unknown agent: ${record.agent}`); - restored = await this.restoreSourceBranch(agent.getContextManager(), target.source, target.target, record.agent); + const outcome = await this.restoreSourceBranch(agent.getContextManager(), target.source, target.target, record.agent); this.materializeConfigMountAfterBranchSwitch(); - if (this.store.currentBranch().name === target.target) remaining = `the source ${target.source} could not be restored: ${restored}`; + if (this.store.currentBranch().name === target.target) { + // The verdict stands; carrying it out is what remains. The abandoned + // body is held until its source is restored (restoreOperatorChangeSource). + remaining = `the source ${target.source} could not be restored: ${outcome}`; + this.holdAbandonedBody({ changeId, agent: record.agent, attempt, target: target.target, source: target.source }); + } else { + restored = target.source; + } } const receipt: OperatorChangeResolutionReceipt = { changeId, @@ -8133,8 +8275,16 @@ export class AgentFramework { // Never a commitment, whatever its destination holds. If the // process died before restoring the source, restore it now. if (current === target) { - await steps.restore(target); + const restored = await steps.restore(target); current = this.store.currentBranch().name; + if (current === target) { + this.holdAbandonedBody({ changeId: change.id, agent: change.agent, attempt: last.n, target, source: last.evidence.source }); + throw new OperatorActionError( + 'failed', + `${label} ${change.id}: its abandoned attempt ${last.n} left ${target} active, and restoring its source ` + + `${last.evidence.source} failed again (${restored}); traffic is held until it is restored`, + ); + } } } else { // Neither its switch nor its failure was recorded, and this process @@ -8199,6 +8349,8 @@ export class AgentFramework { recorded = `; its failure could not be recorded (${journalError instanceof Error ? journalError.message : String(journalError)})`; } const restored = await steps.restore(destination); + // An abandoned destination left active holds traffic until restored. + this.holdAbandonedBody({ changeId: change.id, agent: change.agent, attempt: n, target: destination, source: change.sourceBranch }); throw new OperatorActionError('failed', `${label} ${change.id} failed; ${restored}${recorded}: ${detail}`, { cause: error }); } note('committed'); diff --git a/src/index.ts b/src/index.ts index f9671ba2b..7d601ad26 100644 --- a/src/index.ts +++ b/src/index.ts @@ -3,7 +3,7 @@ export * from './types/index.js'; // Core classes export { AgentFramework, BudgetPreflightError, ResumeBlockedError } from './framework.js'; -export type { RuntimeSettingsPreview, HostModeStatus, SafeBoundaryLease, SafeBoundaryOptions, UnresolvedActiveBody } from './framework.js'; +export type { RuntimeSettingsPreview, HostModeStatus, SafeBoundaryLease, SafeBoundaryOptions, UnresolvedActiveBody, AbandonedActiveBody } from './framework.js'; export { OperatorJournalUnreadableError, OperatorChangeRecoveryError } from './framework.js'; export type { OperatorChangeReceipt, @@ -18,6 +18,8 @@ export type { UnstickStepRecord, UnstickAttemptRecord, UnstickAttemptOutcome, + OperatorChangeResolutionReceipt, + OperatorChangeRestorationReceipt, } from './operator-change.js'; export { Agent } from './agent.js'; export type { StartStreamResult } from './agent.js'; diff --git a/src/operator-change.ts b/src/operator-change.ts index 34fe08545..85c300dfb 100644 --- a/src/operator-change.ts +++ b/src/operator-change.ts @@ -404,7 +404,8 @@ export interface OperatorChangeResolutionReceipt { /** The attestation as recorded. */ recorded: OperatorChangeResolution; /** The source restored, when a not-committed attempt's destination was the - * active body. */ + * active body and restoring it succeeded. A failed restoration is named + * in `remaining` instead. */ restored?: string; /** When the change settles from here. */ settlement: string; @@ -412,6 +413,18 @@ export interface OperatorChangeResolutionReceipt { remaining?: string; } +/** What restoreOperatorChangeSource did: the source of an abandoned + * attempt whose destination was still the active body. */ +export interface OperatorChangeRestorationReceipt { + changeId: string; + attempt: number; + /** The source restored, when restoring it succeeded. */ + restored?: string; + /** What still needs repair when it didn't: the destination stays active + * and traffic stays held. */ + remaining?: string; +} + const CHANGE_KINDS = new Set(['agent-settings', 'tool-presentation', 'undo-turns', 'unstick', 'hide', 'undo-messages']); /** The kinds whose application is journaled as attempts: the cuts and hide. */ const CUT_KINDS = new Set(['undo-turns', 'undo-messages']); diff --git a/test/operator-change-gate.test.ts b/test/operator-change-gate.test.ts index a81dea60a..c8d25433e 100644 --- a/test/operator-change-gate.test.ts +++ b/test/operator-change-gate.test.ts @@ -1303,6 +1303,115 @@ describe('operator-change gate: host/command undo by turns', () => { assert.equal(status.quiesced, false, 'settled: resume proceeds'); }); + // Every CM branch switch fails until the returned function is called. + const failEverySwitch = () => { + const cmx = framework.getAgent('scout')!.getContextManager() as unknown as { switchBranch: (name: string) => Promise }; + const real = cmx.switchBranch.bind(cmx); + cmx.switchBranch = async () => { throw new Error('injected restore failure'); }; + return () => { cmx.switchBranch = real; }; + }; + const restoreSource = (changeId: string) => framework.runAtSafeBoundary({ verb: 'restore' }, (lease) => + framework.restoreOperatorChangeSource(changeId, { lease, requester: { via: 'test', name: 'nissa' } })); + + it('holds an abandoned destination a not-committed settlement could not restore: resume() refuses, forced or not, until the source is restored live', async () => { + await say('a0'); await say('a1'); await say('a2'); + await undoWithMarks(2, 'all'); + const change = asked[0]!; + const undoRecord = failJournalOnce('switched'); + try { + await assert.rejects(applyIt(change), /injected switched failure/); + } finally { + undoRecord(); + } + await reopen(); + const destination = `undo/scout/op-${change.id}`; + assert.deepEqual([branch(), quiesced()], [destination, true]); + const undoSwitch = failEverySwitch(); + let settled: Awaited>; + try { + settled = await resolve(change.id, 1, 'not-committed', 'nothing applied'); + } finally { + undoSwitch(); + } + assert.equal(settled.restored, undefined, 'nothing was restored'); + assert.match(settled.remaining!, new RegExp(`the source ${change.sourceBranch} could not be restored: .*injected restore failure`)); + assert.equal(branch(), destination); + assert.equal(framework.getOperatorChangeRecord(change.id)!.unresolved, undefined, 'its verdict is recorded'); + + const abandoned = [{ changeId: change.id, agent: 'scout', kind: 'undo-turns', attempt: 1, target: destination, source: change.sourceBranch }]; + for (const opts of [undefined, { force: true }]) { + await assert.rejects(framework.resume(opts), (e: unknown) => { + assert.ok(e instanceof ResumeBlockedError); + assert.deepEqual([e.restorationRequired, e.unresolved, e.verdicts], [abandoned, [], []]); + assert.ok(e.message.includes(`restoreOperatorChangeSource("${change.id}", { lease })`)); + assert.doesNotMatch(e.message, /resolveOperatorChange|--verdict/, 'no further attestation is asked for'); + return true; + }); + } + const viaCommand = await host().handleHostCommand('discord', { command: 'resume', force: true, requesterName: 'nissa' }); + assert.deepEqual([viaCommand.ok, viaCommand.restorationRequired], [false, abandoned]); + assert.equal(quiesced(), true); + + const restored = await restoreSource(change.id); + assert.deepEqual(restored, { changeId: change.id, attempt: 1, restored: change.sourceBranch }); + assert.equal(branch(), change.sourceBranch); + assert.ok(framework.getOperatorLog().some((e) => e.kind === 'restore-operator-change-source' && (e.params as { changeId?: string }).changeId === change.id)); + await assert.rejects(restoreSource(change.id), (e: Error & { code?: string }) => e.code === 'invalid' && /nothing to restore/.test(e.message)); + assert.equal((await framework.resume()).quiesced, false, 'the known body is back: resume proceeds'); + }); + + it('holds traffic itself when a live attempt fails and its source cannot be restored, until a live restoration, then serves the source', async () => { + await say('m0'); await say('m1'); + await host().handleHostCommand('discord', { command: 'undo', agentName: 'scout', messages: 2, marks: 'none', requesterName: 'nissa' }); + const change = asked[0]!; + const destination = `undo-msgs/scout/op-${change.id}`; + const undoCut = failCutAfterInitializerWrite(); + let undoSwitch = () => {}; + const cmx = framework.getAgent('scout')!.getContextManager() as unknown as { switchBranch: (name: string) => Promise }; + const cutting = cmx.switchBranch; + cmx.switchBranch = async (name) => { // the cut fails as before; then every restore fails + const result = cutting(name); + undoSwitch = failEverySwitch(); + return result; + }; + try { + await assert.rejects(applyIt(change), /injected strategy initialization failure/); + } finally { + undoSwitch(); undoCut(); + } + assert.equal(branch(), destination, 'left on the abandoned destination'); + assert.equal(quiesced(), true, 'the framework held traffic without anyone calling resume()'); + + const providerCalls = membrane.calls.length; + host().pendingRequests.push({ agentName: 'scout', reason: 'mcpl:channel-incoming', source: 'test', timestamp: Date.now() } as InferenceRequest); + await framework.runUntilIdle(); + assert.equal(membrane.calls.length, providerCalls, 'a queued wake does not run on the abandoned body after the lease released'); + await assert.rejects(framework.resume({ force: true }), (e: unknown) => e instanceof ResumeBlockedError && e.restorationRequired[0]?.target === destination); + + // A retry that fails again keeps the hold, and says so: the host's retry + // of the change, and a live restoration. + const failing = failEverySwitch(); + let again: Awaited>; + try { + await assert.rejects(applyIt(change), (e: Error & { code?: string }) => + e.code === 'failed' && /abandoned attempt 1 left .* active, and restoring its source .* failed again/.test(e.message)); + again = await restoreSource(change.id); + } finally { + failing(); + } + assert.equal(again.restored, undefined); + assert.match(again.remaining!, /could not be restored: .*injected restore failure/); + assert.deepEqual([branch(), quiesced()], [destination, true]); + + assert.deepEqual(await restoreSource(change.id), { changeId: change.id, attempt: 1, restored: change.sourceBranch }); + await framework.resume(); + membrane.pushResponse(createMockResponse([{ type: 'text', text: 'back on the source' }])); + await framework.runUntilIdle(); + assert.ok(membrane.calls.length > providerCalls, 'the parked wake ran once resumed'); + assert.equal(branch(), change.sourceBranch, 'on the source body'); + assert.ok(texts().includes('back on the source')); + }); + it('lets resume() proceed while an unresolved attempt left its source, not its destination, active', async () => { await say('m0'); await say('m1'); await host().handleHostCommand('discord', { command: 'undo', agentName: 'scout', messages: 2, marks: 'all', requesterName: 'nissa' }); From f03d223610470c3279a727fec3d9e1a5ad7f6896 Mon Sep 17 00:00:00 2001 From: Petra-1779 Date: Wed, 7 Oct 2026 20:41:08 -0400 Subject: [PATCH 59/73] fix(framework): an unstick re-run whose turn ends before its stream starts is settled and released, unless a retry, requeue or hold carries it on (Greptile #255 P1-2) Greptile's second finding on #255 (discussion_r4213085189). rerunUnstick binds the agent's next turn to the operation and step, and its caller waits until driveStream's teardown settles and releases the attempt. A turn can also end without ever reaching driveStream: - beginAgentTurn's terminal branch (a non-retryable setup failure such as a provider refusing the request) only called settleAgent, which handles ephemeral runs; - an agent replaced while hooks or compile awaited; - a throw before the inner try (turn-start flush, checkpoint, locus), which Theo-1778 pointed out. On these paths the binding was dropped. The caller waited until stop(), and every later re-run for the agent was refused `agent-busy`, because its waiter stayed registered. - **startAgentStream's finally**, the frame that owns unconditional teardown, now settles the attempt `failed` ("the re-run ended before its stream started") and releases it last, after the token and provider gate are cleared, as driveStream's teardown does. It doesn't when a successor carries the binding (unstickRunCarried): the agent's live turn (a retry frame that started its stream), a queued request (a requeue or a budget restart), or a request a provider hold keeps. Another turn is a successor only if it's bound to the same operation and step (Theo's review condition, #50635). - **beginAgentTurn's terminal branch** records the provider's own error first, so the caller hears why; the finally then only releases. - settleUnstickAttempt and releaseUnstickAttempt are idempotent, so nested retry frames, where an inner frame failed terminally or its stream already finished, settle once. Tests (operator-change-gate, host/command unstick): - **A setup failure the error policy won't retry.** The re-run resolves `{completed, failed, error: }` and is journaled, with no waiter left. Another operation's re-run for the same agent then starts and responds (before: `agent-busy`). - **A setup failure retried once.** The re-run reports the retried turn's own outcome, `refused`, not `failed`. - **A setup failure under a host provider hold.** The binding waits out the hold in heldRequests, then reports the turn that ran (`responded`). Mutants, each caught: no fix (the caller is still pending after 3 s); no carried check (retry and hold reported `failed`); the cooldown not a carrier; the live turn not a carrier; the provider's error not recorded. operator-change-gate, inference-exhaustion-rewind, provider-acceleration-cooldown, host-quiesce, safe-boundary-lease: 177/177. Co-Authored-By: Petra (Claude Opus 5.5) --- src/framework.ts | 25 ++++++++++ test/operator-change-gate.test.ts | 80 +++++++++++++++++++++++++++++++ 2 files changed, 105 insertions(+) diff --git a/src/framework.ts b/src/framework.ts index 4f52f2364..f3ff46fea 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -12273,9 +12273,31 @@ export class AgentFramework { this.activeTurnTriggers.delete(agent.name); } if (!tokenHandedOff && ownsProviderGate) this.releasePrimaryProviderGate(agent.name); + // An unstick re-run whose turn ended before any stream started (setup + // failed terminally, the agent was replaced, or something threw before + // the stream) has its outcome now, unless a retry, a requeue or a + // provider hold carries it on. Released last, once the turn is over, + // as driveStream's teardown releases it. + if (!tokenHandedOff && trigger?.unstick && !this.unstickRunCarried(agent.name, trigger.unstick)) { + this.settleUnstickAttempt(trigger.unstick, 'failed', { error: 'the re-run ended before its stream started' }); + this.releaseUnstickAttempt(trigger.unstick); + } } } + /** Whether an unstick re-run's binding still travels with a turn that may + * run: the agent's live turn (a retry frame that started its stream), a + * queued request (a requeue, or a budget restart), or a request a provider + * hold keeps. Another turn carries it only if it is bound to the same + * operation and step. */ + private unstickRunCarried(agentName: string, binding: { operationId: string; step: number }): boolean { + const carries = (r: InferenceRequest | undefined) => + r?.unstick?.operationId === binding.operationId && r.unstick.step === binding.step; + return carries(this.activeTurnTriggers.get(agentName)) + || this.pendingRequests.some((r) => r.agentName === agentName && carries(r)) + || (this.providerAccelerationCooldowns.get(agentName)?.heldRequests.some(carries) ?? false); + } + /** * Body of startAgentStream, split out so the caller's try/finally owns the * turn token unconditionally. Returns true iff the token was handed off to @@ -12610,6 +12632,9 @@ export class AgentFramework { speech: '', error: err.message, }); + // An unstick re-run failed with the provider's error; the caller's + // finally (startAgentStream) releases it once the turn is over. + if (trigger?.unstick) this.settleUnstickAttempt(trigger.unstick, 'failed', { error: err.message }); this.emitTrace({ type: 'inference:exhausted', agentName: agent.name, diff --git a/test/operator-change-gate.test.ts b/test/operator-change-gate.test.ts index c8d25433e..a1bf358ce 100644 --- a/test/operator-change-gate.test.ts +++ b/test/operator-change-gate.test.ts @@ -1760,6 +1760,86 @@ describe('operator-change gate: host/command unstick, planned at staging', () => await assert.rejects(step(change, 2), (e: Error & { code?: string }) => e.code === 'stale' && /active branch moved/.test(e.message)); }); + // A re-run's outcome, or 'pending' if it hasn't settled while the framework + // was driven for `ms`. + const outcomeWithin = async (outcome: Promise, ms = 3_000) => { + let settled: unknown = 'pending'; + void outcome.then((o) => { settled = o; }, (e) => { settled = e; }); + const deadline = Date.now() + ms; + while (settled === 'pending' && Date.now() < deadline) { + await framework.runUntilIdle(); + await new Promise((resolve) => setTimeout(resolve, 20)); + } + return settled; + }; + // The provider refuses to set up the stream `times` times, then serves. + const failSetup = (times: number, message: string) => { + const real = membrane.streamYielding.bind(membrane); + let left = times; + membrane.streamYielding = (...args: Parameters) => { + if (left-- > 0) throw new Error(message); + return real(...args); + }; + return () => { membrane.streamYielding = real; }; + }; + type Policies = { + errorPolicy: { onInferenceError: (e: Error, agent: string, attempt: number) => { retry: boolean; delayMs?: number } }; + providerHoldHook?: (e: Error) => { holdMs: number } | undefined; + }; + + it('settles and releases a re-run whose turn fails before its stream starts: its caller hears, and the agent is not left busy', async () => { + await unstick(3); + const change = asked[0]!; + await step(change, 1); + (framework as unknown as Policies).errorPolicy = { onInferenceError: () => ({ retry: false }) }; + const undoSetup = failSetup(1, 'provider setup refused the request'); + try { + const outcome = await outcomeWithin(framework.rerunUnstick(change as never, { step: 1 })); + assert.deepEqual(outcome, { step: 1, status: 'completed', outcome: 'failed', error: 'provider setup refused the request' }); + } finally { + undoSetup(); + } + assert.equal(host().unstickAttemptWaiters.size, 0, 'released'); + assert.deepEqual(framework.getUnstickOperation(change.id)!.attempts, + [{ step: 1, status: 'completed', outcome: 'failed', error: 'provider setup refused the request' }], 'journaled'); + // The agent isn't held busy: another operation's re-run starts and runs. + await unstick(1); + const other = asked[1]!; + await step(other, 1); + membrane.pushResponse(createMockResponse([{ type: 'text', text: 'answered' }])); + const second = await outcomeWithin(framework.rerunUnstick(other as never, { step: 1 })); + assert.deepEqual(second, { step: 1, status: 'completed', outcome: 'responded' }); + }); + + it('keeps a re-run bound through a setup retry and a provider hold, reporting the turn that finally ran', async () => { + await unstick(3); + const change = asked[0]!; + await step(change, 1); + (framework as unknown as Policies).errorPolicy = { + onInferenceError: (_e, _agent, attempt) => (attempt < 1 ? { retry: true, delayMs: 1 } : { retry: false }), + }; + let undoSetup = failSetup(1, 'transient setup failure'); + try { + membrane.pushResponse(refusal()); + const retried = await outcomeWithin(framework.rerunUnstick(change as never, { step: 1 })); + assert.deepEqual(retried, { step: 1, status: 'completed', outcome: 'refused', category: 'unknown' }, 'the retried turn\'s own outcome'); + } finally { + undoSetup(); + } + await step(change, 2); + let holds = 1; // the host holds once; asked again when the slice expires, it releases + (framework as unknown as Policies).providerHoldHook = (e) => (/quota/.test(e.message) && holds-- > 0 ? { holdMs: 1_000 } : undefined); + undoSetup = failSetup(1, 'quota exhausted for now'); + try { + membrane.pushResponse(createMockResponse([{ type: 'text', text: 'back' }])); + const held = await outcomeWithin(framework.rerunUnstick(change as never, { step: 2 }), 6_000); + assert.deepEqual(held, { step: 2, status: 'completed', outcome: 'responded' }, 'held, then run: not failed by the hold'); + } finally { + undoSetup(); + } + assert.equal(host().unstickAttemptWaiters.size, 0); + }); + it('finishes an interrupted step on its planned exchange, never another', async () => { await unstick(2); const change = asked[0]!; From 5b9b058e03c8558bef17938b193a6dde191f5ed3 Mon Sep 17 00:00:00 2001 From: Petra-1779 Date: Wed, 7 Oct 2026 20:51:18 -0400 Subject: [PATCH 60/73] fix(framework): a restoration counts only once the source's context initialized; the restart advice follows what persisted (Greptile #255 P1-1, review) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Theo-1778's two findings on 5de55ca (room-272 #51161, #51170; room-222 #51171, with Kit #51178 and Agnes #51187 agreeing). **1. Branch position isn't restoration.** ContextManager.switchBranch moves the store, then awaits strategy initialization, which marks the context uninitialized first. When initialization failed after the move, restoreSourceBranch caught it. The resolver and restoreOperatorChangeSource then read only `current !== target`, so they reported `restored: `, and activeBodyHold, keyed on the active branch's name, cleared. resume() opened traffic on a context that never initialized. Theo reproduced it with only the real strategy's `initialize` throwing. - **restoreCutSource** is now R2's one restoration route, for the applyCut failure path, the host's retry, the resolver and the live route. It calls `cm.switchBranch(source)`, and the restoration completed only if that call resolved, initialization included. Otherwise it's kept in memory (`incompleteRestorations`, change → attempt and failure) and traffic is held (holdAbandonedBody, now taken only here). - **Re-run on a store already back on the source,** it initializes the source. - **The undo-turns kind's own store-level `restore` step and the undo-messages kind's restoreSourceBranch step are gone** from applyCut's steps: both kinds restore through restoreCutSource. restoreSourceBranch stays as it was for the unjournaled direct surgeries. - **activeBodyHold** also holds an abandoned attempt whose restoration selected the source without completing (an incomplete restoration of that attempt, with the source active). It reports `active` and the `restorationFailure`. - **The host's retry** of an abandoned change completes such a restoration first. If it fails again, the retry refuses `failed` and no fresh attempt starts on an uninitialized source. - **Receipts:** `restored` only on completion. The resolver's and the live route's `remaining` read "the source … is not restored: RESTORE TO … FAILED (…)". **2. Restart advice follows what persisted.** An attempt reads abandoned from this process's cutKnowledge even when its failure record never landed. After a restart it's unresolved, so "a restart restores the source" was false there. - `AbandonedActiveBody` gains `active`, `recorded` (whether the journal holds the failure or not-committed verdict) and `restorationFailure`. - resume()'s message says which basis applies. The restart route it names fits the case: - recorded with the destination active: startup restores the source; - seen-only with the destination active: startup finds the attempt unresolved and boots quiesced until it is settled not-committed (live or offline); - the source selected but not initialized: the source initializes afresh at startup. - It no longer says "its verdict is recorded" unconditionally. **The limit, stated** (room-222 #51257 Agnes, #51267/#51274 Theo, #51268 Kit; decided #51260). Only the framework's tracked operations are covered. A cut's destination is a branch of its own, so its being active is read as that attempt's doing. A selection made outside them, a host calling the context manager's switchBranch or the store itself (today's host /checkout, /redo, /restore), leaves no record. It must not reselect a retained unresolved or abandoned attempt's destination: that reapplies the attempt's hold, and at the next start its source restoration, even after an earlier settlement or restoration succeeded. This is documented on activeBodyHold and on boot's restoreAbandonedCutSources. The establishment rule for deliberate selections, under which only a completed activation of that branch supersedes an older hold, ships with the companion's switching routes. No head-movement heuristic was added. Tests (operator-change-gate): - **The resolver with the real initializer failing.** The store is back on the source, but `restored` is unset and `remaining` names the failure. resume({force}) is refused with `restorationRequired` (active: source, recorded, restorationFailure) and the advice that the source initializes afresh. restoreOperatorChangeSource then calls the strategy's initialize exactly once (counted) and reports restored; resume proceeds. - **The host's retry from the selected-but-uninitialized state.** It's held. A retry while initialization still fails refuses `failed` and leaves one attempt. Once initialization works, the retry completes the restoration, then attempts afresh (`~2`). - **Seen-only.** With the failure record failing and the restore failing, the entry is `recorded: false`, and the message says a restart finds it unresolved and boots quiesced until settled not-committed. - 5de55ca's tests now assert the new fields and wording. Two existing crash-before-restore tests now stub the new seam (restoreCutSource) instead of restoreSourceBranch, which R2's cut paths no longer call. Mutants caught: completion judged by branch name; the predicate ignoring an incomplete restoration; the host's retry ignoring it; a restoration skipped when the source is already selected. operator-change-gate, host-quiesce, api-admin-gate, overbudget-drain-breaker, recover-cli-operator-log, safe-boundary-lease, operator-surgery, surgery-marks-receipt: 215/215. Co-Authored-By: Petra (Claude Opus 5.5) --- src/framework.ts | 250 ++++++++++++++++++++---------- test/operator-change-gate.test.ts | 138 +++++++++++++++-- 2 files changed, 292 insertions(+), 96 deletions(-) diff --git a/src/framework.ts b/src/framework.ts index f3ff46fea..18aceab1d 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -922,22 +922,35 @@ export interface UnresolvedActiveBody { target: string; } -/** A cut whose latest attempt was abandoned (not committed: its failure - * recorded or seen, or an operator's not-committed verdict) while its - * destination is still the active body, because restoring its source - * failed: a body nobody approved. Its source must be restored - * (restoreOperatorChangeSource, or a restart) or the host moved to a - * known-safe body; the verdict is already recorded, so no attestation is - * needed. */ +/** A cut whose latest attempt was abandoned (it didn't commit: its failure + * recorded or seen, or an operator's not-committed verdict) and whose + * source hasn't been restored: its destination is still the active body, + * or a restoration moved the store back to the source but failed before + * the source's context initialized. Neither is a usable body. Restore it + * (restoreOperatorChangeSource), restart, or move the host to a + * known-safe body; no attestation is needed. */ export interface AbandonedActiveBody { changeId: string; agent: string; kind: 'undo-turns' | 'undo-messages'; attempt: number; - /** The abandoned destination, which is the active branch. */ + /** The abandoned destination. */ target: string; /** The branch to restore. */ source: string; + /** The branch active now: the destination, or the source when a + * restoration selected it but didn't complete. */ + active: string; + /** Whether the journal holds that the attempt didn't commit (its failure, + * or a not-committed verdict). It decides what a restart does while the + * destination is active: when recorded, startup restores the source + * before any agent initializes; when only this process saw the failure, + * startup finds the attempt unresolved and boots quiesced until it is + * settled not-committed. With the source active, a restart initializes + * it afresh either way. */ + recorded: boolean; + /** Why this process's last restoration of it failed, when one was tried. */ + restorationFailure?: string; } /** @@ -1787,6 +1800,11 @@ export class AgentFramework { /** What this process saw of each cut attempt (change id -> attempt -> * result). It binds even when the record of it couldn't be written. */ private cutKnowledge: Map> = new Map(); + /** Restorations of an abandoned cut's source that this process tried and + * that didn't complete (change id -> its attempt, and why): the store may + * already be back on the source, its context never initialized there. In + * memory only: the next start initializes whatever body it boots on. */ + private incompleteRestorations: Map = new Map(); /** The operator/changes journal, loaded on first use (changesLedger). */ private changesJournalState: { journal: RecordJournal; @@ -3717,7 +3735,8 @@ export class AgentFramework { * strategy cannot preview would hold hosts hostage to a diagnostic. * * Also refuses, whatever `force` says, while the active body is the - * destination of a cut attempt that didn't commit: + * destination of a cut attempt whose commitment is unresolved or known + * not to have happened: * - unresolved (the condition startup boots quiesced on): a resume isn't an * operator's attestation about that body. The error's `unresolved` names * each change and attempt; settle them with resolveOperatorChange, or @@ -3773,12 +3792,24 @@ export class AgentFramework { ); } for (const a of restorationRequired) { + const where = a.active === a.target + ? `its destination ${a.target} is still the active body` + : `a restoration selected its source ${a.source} but didn't complete (${a.restorationFailure})`; + const basis = a.recorded + ? 'its failure or not-committed verdict is recorded' + : 'this process saw it fail, but its failure couldn\'t be recorded'; + const restart = a.active !== a.target + ? 'restart the host (the source initializes afresh at startup)' + : a.recorded + ? 'restart the host (startup restores the source before any agent initializes)' + : 'restart the host, which finds the attempt unresolved and boots quiesced until it is settled ' + + 'not-committed (resolveOperatorChange, or agent-framework-recover --operator-change resolve), which restores the source'; reasons.push( `operator change ${a.changeId} (${a.kind} for ${a.agent}) was abandoned: its attempt ${a.attempt} didn't ` + - `commit, but its destination ${a.target} is still the active body, and its source ${a.source} must be ` + - `restored before traffic. Its verdict is recorded, and force doesn't override this: restore it live with ` + - `restoreOperatorChangeSource(${JSON.stringify(a.changeId)}, { lease }), restart the host (startup restores ` + - `the source before any agent initializes), or move the host to a known-safe body`, + `commit (${basis}), but ${where}, and its source ${a.source} must be restored before traffic. No ` + + `attestation is needed, and force doesn't override this: restore it live with ` + + `restoreOperatorChangeSource(${JSON.stringify(a.changeId)}, { lease }), ${restart}, or move the host to a ` + + `known-safe body`, ); } if (overBudget) { @@ -7441,15 +7472,41 @@ export class AgentFramework { return copy; } - /** Whether a cut's latest attempt holds the active body: its destination - * is the active branch and the attempt didn't commit, either unresolved - * (nobody can vouch for that body) or abandoned with its source not - * restored (a body nobody approved). Decided as everywhere else, by the - * record and what this process saw (cutAttemptDisposition). */ - private activeBodyHold(record: OperatorChangeRecord): { held: 'unresolved' | 'abandoned'; n: number; target: string; source: string } | null { + /** Whether a cut's latest attempt holds the active body. Its commitment + * is unresolved or known not to have happened, and either its + * destination is the active branch, or a restoration of its source + * selected the source without completing (the source's context never + * initialized). Unresolved: nobody can vouch for + * that body. Abandoned: a body nobody approved, its source still to + * restore. Decided as everywhere else, by the record and what this + * process saw (cutAttemptDisposition), never by branch position alone. + * + * The limit: only the framework's tracked operations are covered. A + * cut's destination is a branch of its own, so its being active is read + * as that attempt's doing. A selection made outside them (a host calling + * the context manager's switchBranch or the store itself) leaves no + * record and can't be told apart, so it must not reselect the + * destination of a retained unresolved or abandoned attempt: that + * reapplies the attempt's hold, and at the next start its source + * restoration, even after an earlier settlement or restoration + * succeeded. A host that needs such a selection needs tracked + * activation ownership for it, not a past repair. */ + private activeBodyHold(record: OperatorChangeRecord): { + held: 'unresolved' | 'abandoned'; + n: number; + target: string; + source: string; + active: string; + recorded: boolean; + restorationFailure?: string; + } | null { if (record.outcome || record.dropped || (record.kind !== 'undo-turns' && record.kind !== 'undo-messages')) return null; const last = record.attempts[record.attempts.length - 1]; - if (!last || last.evidence.target !== this.store.currentBranch().name) return null; + if (!last) return null; + const active = this.store.currentBranch().name; + const incomplete = this.incompleteRestorations.get(record.changeId); + const pending = incomplete?.attempt === last.n ? incomplete : undefined; + if (active !== last.evidence.target && !(pending && active === last.evidence.source)) return null; const disposition = cutAttemptDisposition(record, last, this.cutKnowledge.get(record.changeId)?.get(last.n)); if (disposition === 'committed') return null; return { @@ -7457,12 +7514,15 @@ export class AgentFramework { n: last.n, target: last.evidence.target, source: last.evidence.source, + active, + recorded: last.failed !== undefined || last.resolution?.verdict === 'not-committed', + ...(pending ? { restorationFailure: pending.failure } : {}), }; } /** The cuts holding the active body: what resume() refuses, startup boots * quiesced on (unresolved) or restores before any agent initializes - * (abandoned), and a failed restoration holds traffic on. */ + * (abandoned and recorded), and an incomplete restoration holds traffic on. */ private heldActiveBodies(): { unresolved: UnresolvedActiveBody[]; restorationRequired: AbandonedActiveBody[] } { const unresolved: UnresolvedActiveBody[] = []; const restorationRequired: AbandonedActiveBody[] = []; @@ -7470,29 +7530,64 @@ export class AgentFramework { const hold = this.activeBodyHold(record); if (!hold) continue; const body = { changeId: record.changeId, agent: record.agent, kind: record.kind as UnresolvedActiveBody['kind'], attempt: hold.n, target: hold.target }; - if (hold.held === 'unresolved') unresolved.push(body); - else restorationRequired.push({ ...body, source: hold.source }); + if (hold.held === 'unresolved') { + unresolved.push(body); + continue; + } + restorationRequired.push({ + ...body, + source: hold.source, + active: hold.active, + recorded: hold.recorded, + ...(hold.restorationFailure !== undefined ? { restorationFailure: hold.restorationFailure } : {}), + }); } return { unresolved, restorationRequired }; } /** - * Hold traffic when restoring an abandoned attempt's source failed and its - * destination is still the active body, a body nobody approved: the - * framework quiesces in memory, as startup holds an unresolved active body - * (wakes park, writes defer, data planes pause), so the hold doesn't - * depend on anyone calling resume(). resume() then refuses until the - * source is restored (restoreOperatorChangeSource, or a restart, which - * restores it before any agent initializes) or the host moves to a - * known-safe body. Not persisted: the next start decides afresh. Decided - * from what the caller saw, without rereading the journal, so a journal - * that just failed can't keep the hold from being taken. + * Restore an abandoned cut attempt's source: switch the agent's context + * back to it, strategy initialization included. It completed only if that + * switch finished. A switch that moved the store back but failed to + * initialize is no restored body, whatever the active branch's name, so an + * incomplete restoration is kept (incompleteRestorations) and holds + * traffic until one completes. Re-run on a store already back on the + * source, it initializes the source. Never throws: it reports what + * happened. + */ + private async restoreCutSource(abandoned: { changeId: string; agent: string; attempt: number; target: string; source: string }): Promise<{ completed: boolean; detail: string }> { + const cm = this.agents.get(abandoned.agent)?.getContextManager(); + try { + if (!cm) throw new Error(`unknown agent ${abandoned.agent}`); + await cm.switchBranch(abandoned.source); + this.materializeConfigMountAfterBranchSwitch(); + this.incompleteRestorations.delete(abandoned.changeId); + return { completed: true, detail: `active branch restored to ${abandoned.source}` }; + } catch (error) { + const failure = error instanceof Error ? error.message : String(error); + this.incompleteRestorations.set(abandoned.changeId, { attempt: abandoned.attempt, failure }); + this.holdAbandonedBody(abandoned); + const status = `RESTORE TO ${abandoned.source} FAILED (${failure}); the active branch is ${this.store.currentBranch().name}`; + console.error(`[operator-changes] agent=${abandoned.agent} ${abandoned.changeId}: ${status}`); + return { completed: false, detail: status }; + } + } + + /** + * Hold traffic while an abandoned attempt's source isn't restored and + * nothing else holds it: the framework quiesces in memory, as startup holds + * an unresolved active body (wakes park, writes defer, data planes pause), + * so the hold doesn't depend on anyone calling resume(). resume() then + * refuses until a restoration completes (restoreOperatorChangeSource, the + * host's retry, or a restart) or the host moves to a known-safe body. Not + * persisted: the next start decides afresh. Taken from what the caller + * saw, without rereading the journal, so a journal that just failed can't + * prevent it. */ private holdAbandonedBody(abandoned: { changeId: string; agent: string; attempt: number; target: string; source: string }): void { - if (this.store.currentBranch().name !== abandoned.target) return; const { changeId, agent } = abandoned; - const reason = `operator change ${changeId} was abandoned, but its attempt ${abandoned.attempt} left ${abandoned.target} ` + - `active: its source ${abandoned.source} could not be restored`; + const reason = `operator change ${changeId} was abandoned, but its source ${abandoned.source} isn't restored ` + + `(its attempt ${abandoned.attempt} cut to ${abandoned.target}; the active branch is ${this.store.currentBranch().name})`; if (!this.quiesced) { this.quiesced = true; this.quiesceReason = reason; @@ -7503,18 +7598,19 @@ export class AgentFramework { this.opsAlert( 'operator-change-restoration-required', agent, - `${reason}; traffic is held until it is restored (restoreOperatorChangeSource, or a restart) and the host resumes`, + `${reason}; traffic is held until a restoration completes (restoreOperatorChangeSource, or a restart) and the host resumes`, ); } /** - * Restore the source of an abandoned cut whose destination is still the - * active body, under the held lease: its latest attempt didn't commit (its - * failure recorded or seen, or an operator's not-committed verdict), and - * restoring its source failed when it failed or was settled. It carries - * out what is already recorded, so it takes no verdict; a fresh attempt - * stays the host's decision on its next retry. Refuses unless that holds. - * If restoring fails again, traffic stays held and the receipt says so. + * Restore the source of an abandoned cut under the held lease: its latest + * attempt didn't commit (its failure recorded or seen, or an operator's + * not-committed verdict), and its destination is still the active body, + * or an earlier restoration selected the source without completing. It + * carries out what is already decided, so it takes no verdict; a fresh + * attempt stays the host's decision on its next retry. Refuses unless + * that holds. Restored only when the source's context finished switching; + * otherwise traffic stays held and the receipt says what remains. */ async restoreOperatorChangeSource( changeId: string, @@ -7526,19 +7622,14 @@ export class AgentFramework { if (!record || hold?.held !== 'abandoned') { throw new OperatorActionError( 'invalid', - `Operator change ${changeId} has no abandoned attempt whose destination is the active body; there is nothing to restore`, + `Operator change ${changeId} has no abandoned attempt whose source is still to restore; there is nothing to restore`, ); } - const agent = this.agents.get(record.agent); - if (!agent) throw new OperatorActionError('unknown-agent', `Unknown agent: ${record.agent}`); - const outcome = await this.restoreSourceBranch(agent.getContextManager(), hold.source, hold.target, record.agent); - this.materializeConfigMountAfterBranchSwitch(); - const failed = this.store.currentBranch().name === hold.target; - if (failed) this.holdAbandonedBody({ changeId, agent: record.agent, attempt: hold.n, target: hold.target, source: hold.source }); + const result = await this.restoreCutSource({ changeId, agent: record.agent, attempt: hold.n, target: hold.target, source: hold.source }); const receipt: OperatorChangeRestorationReceipt = { changeId, attempt: hold.n, - ...(failed ? { remaining: `the source ${hold.source} could not be restored: ${outcome}` } : { restored: hold.source }), + ...(result.completed ? { restored: hold.source } : { remaining: `the source ${hold.source} is not restored: ${result.detail}` }), }; this.recordOperatorAction({ kind: 'restore-operator-change-source', @@ -7596,18 +7687,12 @@ export class AgentFramework { let restored: string | undefined; let remaining: string | undefined; if (verdict === 'not-committed' && this.store.currentBranch().name === target.target) { - const agent = this.agents.get(record.agent); - if (!agent) throw new OperatorActionError('unknown-agent', `Unknown agent: ${record.agent}`); - const outcome = await this.restoreSourceBranch(agent.getContextManager(), target.source, target.target, record.agent); - this.materializeConfigMountAfterBranchSwitch(); - if (this.store.currentBranch().name === target.target) { - // The verdict stands; carrying it out is what remains. The abandoned - // body is held until its source is restored (restoreOperatorChangeSource). - remaining = `the source ${target.source} could not be restored: ${outcome}`; - this.holdAbandonedBody({ changeId, agent: record.agent, attempt, target: target.target, source: target.source }); - } else { - restored = target.source; - } + if (!this.agents.has(record.agent)) throw new OperatorActionError('unknown-agent', `Unknown agent: ${record.agent}`); + const result = await this.restoreCutSource({ changeId, agent: record.agent, attempt, target: target.target, source: target.source }); + // The verdict stands either way. An incomplete restoration is what + // remains, and holds traffic until one completes. + if (result.completed) restored = target.source; + else remaining = `the source ${target.source} is not restored: ${result.detail}`; } const receipt: OperatorChangeResolutionReceipt = { changeId, @@ -8227,8 +8312,6 @@ export class AgentFramework { removal: () => Array<{ metadata?: unknown }>; /** Create `destination` (it never exists yet) and switch to it. */ cut: (destination: string) => Promise; - /** Put the source back if the cut left `destination` active. */ - restore: (destination: string) => Promise; finish: (outcome: OperatorChangeOutcome, target: string) => void; }, ): Promise<{ outcome: OperatorChangeOutcome; alreadyApplied: boolean; target: string }> { @@ -8272,17 +8355,19 @@ export class AgentFramework { return settle(this.establishOutcome(change, last, 'undo'), true); } if (disposition === 'not-committed') { - // Never a commitment, whatever its destination holds. If the - // process died before restoring the source, restore it now. - if (current === target) { - const restored = await steps.restore(target); + // Never a commitment, whatever its destination holds. If its source + // was never restored (the process died first, or a restoration + // didn't complete, even one that selected the source), restore it + // now, and only then make a fresh attempt. + const pending = this.incompleteRestorations.get(change.id)?.attempt === last.n; + if (current === target || (pending && current === last.evidence.source)) { + const result = await this.restoreCutSource({ changeId: change.id, agent: change.agent, attempt: last.n, target, source: last.evidence.source }); current = this.store.currentBranch().name; - if (current === target) { - this.holdAbandonedBody({ changeId: change.id, agent: change.agent, attempt: last.n, target, source: last.evidence.source }); + if (!result.completed) { throw new OperatorActionError( 'failed', - `${label} ${change.id}: its abandoned attempt ${last.n} left ${target} active, and restoring its source ` + - `${last.evidence.source} failed again (${restored}); traffic is held until it is restored`, + `${label} ${change.id}: its abandoned attempt ${last.n} cut to ${target}, and restoring its source ` + + `${last.evidence.source} failed again (${result.detail}); traffic is held until it is restored`, ); } } @@ -8348,9 +8433,11 @@ export class AgentFramework { } catch (journalError) { recorded = `; its failure could not be recorded (${journalError instanceof Error ? journalError.message : String(journalError)})`; } - const restored = await steps.restore(destination); - // An abandoned destination left active holds traffic until restored. - this.holdAbandonedBody({ changeId: change.id, agent: change.agent, attempt: n, target: destination, source: change.sourceBranch }); + // The cut left its destination active: restore the source, completely + // (restoreCutSource holds traffic if it can't). + const restored = this.store.currentBranch().name === destination + ? (await this.restoreCutSource({ changeId: change.id, agent: change.agent, attempt: n, target: destination, source: change.sourceBranch })).detail + : `active branch is ${this.store.currentBranch().name}`; throw new OperatorActionError('failed', `${label} ${change.id} failed; ${restored}${recorded}: ${detail}`, { cause: error }); } note('committed'); @@ -8391,12 +8478,6 @@ export class AgentFramework { this.store.switchBranch(destination); this.materializeConfigMountAfterBranchSwitch(); }, - restore: async (destination) => { - if (this.store.currentBranch().name !== destination) return `active branch is ${this.store.currentBranch().name}`; - this.store.switchBranch(change.sourceBranch); - this.materializeConfigMountAfterBranchSwitch(); - return `active branch restored to ${change.sourceBranch}`; - }, finish: (established, destination) => { // The one redo entry back to the source tip, while the destination is // active and no turn has run on it (new work invalidates redo). @@ -8471,7 +8552,6 @@ export class AgentFramework { await cm.switchBranch(cm.branchAt(change.tail.id as MessageId, destination)); this.materializeConfigMountAfterBranchSwitch(); }, - restore: (destination) => this.restoreSourceBranch(cm, change.sourceBranch, destination, change.agent), finish: (established, destination) => { this.recordChangeAction({ kind: 'rollback', @@ -8618,6 +8698,8 @@ export class AgentFramework { * Before any agent initializes on it: when an abandoned cut (its attempt * failed, or an operator settled it as not committed) left its destination * as the active branch, switch the store back to the source and sync it. + * The active branch is read as that attempt's doing (see activeBodyHold + * for the limit of that reading). */ private restoreAbandonedCutSources(): void { this.unstickLedger(); diff --git a/test/operator-change-gate.test.ts b/test/operator-change-gate.test.ts index a1bf358ce..af42aa5c5 100644 --- a/test/operator-change-gate.test.ts +++ b/test/operator-change-gate.test.ts @@ -1139,14 +1139,14 @@ describe('operator-change gate: host/command undo by turns', () => { framework.getStore().setStateJson('probe/initializer', { wrote: true }); throw new Error('injected strategy initialization failure'); }; - const fw = framework as unknown as { restoreSourceBranch: (...args: unknown[]) => Promise }; - const realRestore = fw.restoreSourceBranch.bind(fw); - if (crashBeforeRestore) fw.restoreSourceBranch = async () => { throw new Error('the process died before restoring'); }; + const fw = framework as unknown as { restoreCutSource: (...args: unknown[]) => Promise }; + const realRestore = fw.restoreCutSource.bind(fw); + if (crashBeforeRestore) fw.restoreCutSource = async () => { throw new Error('the process died before restoring'); }; try { await assert.rejects(applyIt(change), crashBeforeRestore ? /died before restoring/ : /injected strategy initialization failure/); } finally { cmx.switchBranch = realSwitch; - fw.restoreSourceBranch = realRestore; + fw.restoreCutSource = realRestore; } assert.equal(branch(), crashBeforeRestore ? destination : change.sourceBranch); assert.equal(framework.getOperatorChangeRecord(change.id)!.attempts[0]!.failed !== undefined, true, 'its failure recorded first'); @@ -1334,11 +1334,14 @@ describe('operator-change gate: host/command undo by turns', () => { undoSwitch(); } assert.equal(settled.restored, undefined, 'nothing was restored'); - assert.match(settled.remaining!, new RegExp(`the source ${change.sourceBranch} could not be restored: .*injected restore failure`)); + assert.match(settled.remaining!, new RegExp(`the source ${change.sourceBranch} is not restored: RESTORE TO ${change.sourceBranch} FAILED \\(injected restore failure\\)`)); assert.equal(branch(), destination); assert.equal(framework.getOperatorChangeRecord(change.id)!.unresolved, undefined, 'its verdict is recorded'); - const abandoned = [{ changeId: change.id, agent: 'scout', kind: 'undo-turns', attempt: 1, target: destination, source: change.sourceBranch }]; + const abandoned = [{ + changeId: change.id, agent: 'scout', kind: 'undo-turns', attempt: 1, target: destination, source: change.sourceBranch, + active: destination, recorded: true, restorationFailure: 'injected restore failure', + }]; for (const opts of [undefined, { force: true }]) { await assert.rejects(framework.resume(opts), (e: unknown) => { assert.ok(e instanceof ResumeBlockedError); @@ -1394,13 +1397,13 @@ describe('operator-change gate: host/command undo by turns', () => { let again: Awaited>; try { await assert.rejects(applyIt(change), (e: Error & { code?: string }) => - e.code === 'failed' && /abandoned attempt 1 left .* active, and restoring its source .* failed again/.test(e.message)); + e.code === 'failed' && /abandoned attempt 1 cut to .*, and restoring its source .* failed again/.test(e.message)); again = await restoreSource(change.id); } finally { failing(); } assert.equal(again.restored, undefined); - assert.match(again.remaining!, /could not be restored: .*injected restore failure/); + assert.match(again.remaining!, /is not restored: RESTORE TO .* FAILED \(injected restore failure\)/); assert.deepEqual([branch(), quiesced()], [destination, true]); assert.deepEqual(await restoreSource(change.id), { changeId: change.id, attempt: 1, restored: change.sourceBranch }); @@ -1412,6 +1415,117 @@ describe('operator-change gate: host/command undo by turns', () => { assert.ok(texts().includes('back on the source')); }); + // The strategy's initializer fails until the returned function is called: + // a real CM switch then selects the branch but never initializes there. + const failInitialization = () => { + const strategy = framework.getAgent('scout')!.getContextManager().getStrategy() as unknown as Record; + const own = Object.prototype.hasOwnProperty.call(strategy, 'initialize'); + const real = strategy.initialize; + strategy.initialize = async () => { throw new Error('injected initializer failure'); }; + return () => { if (own) strategy.initialize = real; else delete strategy.initialize; }; + }; + + it('counts a restoration only once the source\'s context initialized: one that selected the source and failed stays held until a live restoration completes', async () => { + await say('a0'); await say('a1'); await say('a2'); + await undoWithMarks(2, 'all'); + const change = asked[0]!; + const undoRecord = failJournalOnce('switched'); + try { + await assert.rejects(applyIt(change), /injected switched failure/); + } finally { + undoRecord(); + } + await reopen(); + const destination = `undo/scout/op-${change.id}`; + const undoInit = failInitialization(); + let settled: Awaited>; + try { + settled = await resolve(change.id, 1, 'not-committed', 'nothing applied'); + } finally { + undoInit(); + } + assert.equal(branch(), change.sourceBranch, 'the store is back on the source…'); + assert.equal(settled.restored, undefined, '…but nothing was restored'); + assert.match(settled.remaining!, /is not restored: RESTORE TO .* FAILED \(injected initializer failure\)/); + await assert.rejects(framework.resume({ force: true }), (e: unknown) => { + assert.ok(e instanceof ResumeBlockedError); + assert.deepEqual(e.restorationRequired, [{ + changeId: change.id, agent: 'scout', kind: 'undo-turns', attempt: 1, target: destination, source: change.sourceBranch, + active: change.sourceBranch, recorded: true, restorationFailure: 'injected initializer failure', + }]); + assert.match(e.message, /a restoration selected its source .* but didn't complete \(injected initializer failure\)/); + assert.match(e.message, /the source initializes afresh at startup/); + return true; + }); + const strategy = framework.getAgent('scout')!.getContextManager().getStrategy() as unknown as Record; + const own = Object.prototype.hasOwnProperty.call(strategy, 'initialize'); + const initialize = strategy.initialize as ((ctx: unknown) => Promise) | undefined; + let initialized = 0; + strategy.initialize = async (ctx: unknown) => { initialized++; await initialize?.call(strategy, ctx); }; + try { + assert.deepEqual(await restoreSource(change.id), { changeId: change.id, attempt: 1, restored: change.sourceBranch }); + } finally { + if (own) strategy.initialize = initialize; else delete strategy.initialize; + } + assert.equal(initialized, 1, 'the retry initialized the source it found selected'); + assert.equal((await framework.resume()).quiesced, false); + }); + + it("completes a selected-but-uninitialized restoration on the host's retry, and only then attempts afresh", async () => { + await say('m0'); await say('m1'); + await host().handleHostCommand('discord', { command: 'undo', agentName: 'scout', messages: 2, marks: 'none', requesterName: 'nissa' }); + const change = asked[0]!; + const destination = `undo-msgs/scout/op-${change.id}`; + const undoCut = failCutAfterInitializerWrite(); // the cut selects its destination, then fails + const undoInit = failInitialization(); // and restoring selects the source but can't initialize it + try { + await assert.rejects(applyIt(change), /injected strategy initialization failure/); + undoCut(); + assert.deepEqual([branch(), quiesced()], [change.sourceBranch, true], 'selected, not restored: held'); + // The host's retry while the source still can't initialize: refused, and no fresh attempt. + await assert.rejects(applyIt(change), (e: Error & { code?: string }) => e.code === 'failed' && /failed again/.test(e.message)); + assert.equal(framework.getOperatorChangeRecord(change.id)!.attempts.length, 1, 'no fresh attempt on an uninitialized source'); + } finally { + undoInit(); undoCut(); + } + await assert.rejects(framework.resume(), (e: unknown) => e instanceof ResumeBlockedError && e.restorationRequired[0]?.active === change.sourceBranch); + + const applied = await applyIt(change); // the host's retry: completes the restoration, then attempts afresh + assert.equal(applied.kind === 'undo-messages' && applied.toBranch, `${destination}~2`); + assert.deepEqual(texts(), ['m0', 'reply to m0']); + assert.equal((await framework.resume()).quiesced, false); + }); + + it('says a restart finds the attempt unresolved when only this process saw it fail', async () => { + await say('m0'); await say('m1'); + await host().handleHostCommand('discord', { command: 'undo', agentName: 'scout', messages: 2, marks: 'none', requesterName: 'nissa' }); + const change = asked[0]!; + const destination = `undo-msgs/scout/op-${change.id}`; + const undoCut = failCutAfterInitializerWrite(); + const undoRecord = failJournalOnce('failed'); // its failure can't be recorded + let undoSwitch = () => {}; + const cmx = framework.getAgent('scout')!.getContextManager() as unknown as { switchBranch: (name: string) => Promise }; + const cutting = cmx.switchBranch; + cmx.switchBranch = async (name) => { // after the cut, every restore fails + const result = cutting(name); + undoSwitch = failEverySwitch(); + return result; + }; + try { + await assert.rejects(applyIt(change), /injected strategy initialization failure/); + } finally { + undoSwitch(); undoRecord(); undoCut(); + } + assert.equal(branch(), destination); + await assert.rejects(framework.resume(), (e: unknown) => { + assert.ok(e instanceof ResumeBlockedError); + assert.deepEqual([e.restorationRequired[0]?.recorded, e.restorationRequired[0]?.active], [false, destination]); + assert.match(e.message, /this process saw it fail, but its failure couldn't be recorded/); + assert.match(e.message, /restart the host, which finds the attempt unresolved and boots quiesced until it is settled not-committed/); + return true; + }); + }); + it('lets resume() proceed while an unresolved attempt left its source, not its destination, active', async () => { await say('m0'); await say('m1'); await host().handleHostCommand('discord', { command: 'undo', agentName: 'scout', messages: 2, marks: 'all', requesterName: 'nissa' }); @@ -1436,13 +1550,13 @@ describe('operator-change gate: host/command undo by turns', () => { await host().handleHostCommand('discord', { command: 'undo', agentName: 'scout', messages: 2, marks: 'none', requesterName: 'nissa' }); const change = asked[0]!; const undoCut = failCutAfterInitializerWrite(); - const fw = framework as unknown as { restoreSourceBranch: (...args: unknown[]) => Promise }; - const realRestore = fw.restoreSourceBranch.bind(fw); - fw.restoreSourceBranch = async () => { throw new Error('the process died before restoring'); }; + const fw = framework as unknown as { restoreCutSource: (...args: unknown[]) => Promise }; + const realRestore = fw.restoreCutSource.bind(fw); + fw.restoreCutSource = async () => { throw new Error('the process died before restoring'); }; try { await assert.rejects(applyIt(change), /died before restoring/); } finally { - undoCut(); fw.restoreSourceBranch = realRestore; + undoCut(); fw.restoreCutSource = realRestore; } assert.equal(branch(), `undo-msgs/scout/op-${change.id}`, 'left on the failed destination'); await reopen(); From 2206c97ade87039d650506ee4172ea3bde98f0d3 Mon Sep 17 00:00:00 2001 From: Petra-1779 Date: Wed, 7 Oct 2026 20:55:10 -0400 Subject: [PATCH 61/73] fix(framework): stop() closes admission before its first await and lets the store's current users finish before tearing anything down (Greptile #255 P1-4) Greptile's fourth finding on #255 (discussion_r4213085201). stop() never accounted for the safe-boundary lease. A granted callback awaiting host bookkeeping could find its store closed when it resumed, and Theo-1778 reproduced exactly that (`stopDone: true, closed: true` while the callback was still held). Waiting requests were neither rejected nor granted, so their callers waited forever. The contract was agreed in room-222: - Kit-1008 (#50544, #50642): the shared-reservation invariant and a supported two-phase handoff. - Agnes-1009 (#50639, #50644, #50646): the reservation invariant, and turn admission closed too. - Theo (#50638, #50641, #50653): the handoff and the turn-admission requirement. - Decided in #50652: drain the shared store reservation, not just the lease callback. Theo's review (room-272 #51378, #51539) found three more ways into the store: - an ephemeral creation parked behind a lease, which woke at the release, started initializing, and had the store closed under it ("Store has been closed"); - a creation admitted in the same turn stop() began, whose continuation registered its initialization only after stop() had looked; - a finished candidate run after stop() began. **One flag, `stopping`, set synchronously before stop()'s first await, closes admission to everything:** - **Turns and provider calls.** `running = false` and `providerAdmissionClosed = true` move to the top; their meaning is unchanged. - **Leases** (closeStoreAdmission). runAtSafeBoundary refuses a new request, and every request still waiting is rejected. Both get an AbortError, "safe boundary for refused: the framework is stopping", as for a withdrawn request, since nothing was granted. - **Direct surgeries.** reserveStoreForSurgery refuses with `OperatorActionError('invalid', 'Cannot : the framework is stopping')`. - **Ephemeral creations.** - tryAdmitEphemeralCreation refuses ("Ephemeral agent creation refused: the framework is stopping"). - Creations parked behind a lease or surgery are woken at once and refused the same way, before their initializer runs. - A creation admitted just before stop() began re-checks after its admission await, then refuses and releases its ticket. Either its initialization was registered before stop() looked, or it never starts. - **Ephemeral runs.** runEphemeralToCompletion refuses before consuming the candidate, which its caller then cleans up. **Then the store's current users finish:** - **Whoever holds the reservation:** a leased callback, or a direct surgery's awaited CM and journal work. The hold now carries its lifetime as a `released` promise. - **Creations admitted before shutdown that are still opening their context in the store** (`ephemeralInitializations`). A finished candidate waiting for its caller isn't waited for. Nothing new can be admitted after either. The rest of stop() then runs as before: coalescer suspend and flush, unstick waiters, deferred-write ack, queue close, and the existing teardown. queue.close() and module/MCPL shutdown therefore stay after the holder's work, which may push events or call tools. **The supported handoff** (Kit #50642; Agnes's coordinator, #50644/#50646): the host calls `const stopping = fw.stop()` inside its lease callback, returns, and awaits `stopping` outside. Admission is already closed when the callback returns, so nothing is granted, started or created in between. Awaiting stop() inside the callback waits for itself; the doc comments on stop() and runAtSafeBoundary say so. Tests (safe-boundary-lease, "stop() and the store reservation"): - **A held callback.** stop() is still pending after 200 ms. The callback then writes and syncs its store and returns its value, and stop() completes. - **Refusals once stop() begins.** A waiting request is rejected with the stopping AbortError, a new request is refused, a direct surgery is refused `invalid`, and nothing is granted. - **The handoff, Agnes's acceptance case.** A wake is queued inside the callback. stop() is initiated there and not awaited, a second lease request inside is refused, and the callback's own store work succeeds. After return, a scheduler pass starts nothing, no provider call is made, and the shutdown completes. - **A creation parked behind a lease.** It is refused once stop() begins, and its initializer never runs. - **A creation admitted before stop(), with its initializer held.** stop() is pending until the initializer finishes and the candidate is returned. - **A creation admitted in the same turn stop() begins** (Theo's case). It is refused, its initializer never runs, and its ticket is released. - **A finished candidate run after stop() begins.** It is refused without being consumed. - **A direct surgery** (rollbackToMessage) with its CM switch held mid-await. stop() is pending until the switch is released and the surgery completes. Mutants: - The previous stop() fails the four reservation tests. - No drain: caught by three tests. - No lease-admission close: caught by two. - Direct surgeries not refused: caught by one. - Creation admission not refused: caught by one. - Parked creations not woken: caught by one. - Initializations not awaited: caught by one. - No post-admission re-check: caught by one. - Runs not refused: caught, by the refusal test timing out. - Equivalent today: closing turn admission after the drain rather than before it. The holder's release resolves the drain's microtask before any scheduler pass can run; the synchronous close makes that independent of ordering. Suites: - safe-boundary-lease, host-quiesce, operator-surgery, and the suites that create ephemeral agents (code-execution, ephemeral-settle, same-round-think-policy, state-scaling-e2e, tool-result-guard, tool-wrapper-prose-guard-integration): see the batch's full-suite run. - Earlier versions: 195/195 there; operator-change-gate, api-admin-gate, module-targeted-delivery and surgery-marks-receipt 218/218. Co-Authored-By: Petra (Claude Opus 5.5) --- src/framework.ts | 97 +++++++++++++++- test/safe-boundary-lease.test.ts | 194 +++++++++++++++++++++++++++++++ 2 files changed, 286 insertions(+), 5 deletions(-) diff --git a/src/framework.ts b/src/framework.ts index 18aceab1d..b9e20de66 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -790,6 +790,15 @@ export class OperatorChangeRecoveryError extends Error { } } +/** A safe-boundary request refused, or a waiting one rejected, because the + * framework is stopping: an AbortError, like a withdrawn request, since + * nothing was granted. */ +function stoppingRefusal(verb: string): Error { + const error = new Error(`safe boundary for ${verb} refused: the framework is stopping`); + error.name = 'AbortError'; + return error; +} + class DiscordAwarenessAccountingError extends Error { constructor(operation: string, cause: unknown) { const detail = cause instanceof Error ? cause.message : String(cause); @@ -1763,7 +1772,11 @@ export class AgentFramework { * agent admitted mid-switch, a wake for an agent created later — at the * scheduler, ephemeral admission and puppet entry points. */ - private surgeryHold: { verb: string; agentName: string; since: number } | null = null; + private surgeryHold: { verb: string; agentName: string; since: number; released: Promise } | null = null; + /** Set by stop() before its first await: no lease, direct surgery, + * ephemeral creation or ephemeral run enters the store from then on + * (closeStoreAdmission). */ + private stopping = false; /** * Hosts waiting for a safe boundary (`runAtSafeBoundary`), oldest first. * While any wait, new resident turns are held (see processInferenceRequests) @@ -1773,6 +1786,8 @@ export class AgentFramework { verb: string; requester?: OperatorRequester; grant: (held: { lease: SafeBoundaryLease; release: () => void }) => void; + /** Reject the request: stop() refuses every request still waiting. */ + refuse: (error: Error) => void; }> = []; /** The lease currently held, and the turn token it reserved per agent. */ private heldLease: { lease: SafeBoundaryLease; tokens: Map } | null = null; @@ -1785,6 +1800,10 @@ export class AgentFramework { */ private ephemeralPending: Set = new Set(); private ephemeralPendingByAgent: WeakMap = new WeakMap(); + /** Admitted creations still initializing (opening their context in the + * store): stop() lets them finish before closing it. A finished + * candidate waiting for its caller isn't here. */ + private ephemeralInitializations: Set> = new Set(); /** Candidates withdrawn by their cleanup(): they can never run, since the * pending admission that protected their pre-run context is gone. */ private releasedEphemeralCandidates: WeakSet = new WeakSet(); @@ -2390,12 +2409,33 @@ export class AgentFramework { } /** - * Stop the event loop. + * Stop the event loop. Admission closes synchronously, before the first + * await (turns, provider calls, leases, direct surgeries and ephemeral + * creations), then the store reservation's holder and any creation still + * initializing finish, then everything is torn down. Never + * await stop() inside a lease callback: initiate it there, return, and + * await it outside. */ async stop(): Promise { + // Admission closes before the first await: no turn starts and no + // provider call is made, no lease or direct surgery takes the store + // reservation, no ephemeral creation enters the store, and waiting lease + // requests and creations are refused. A host can therefore initiate + // shutdown inside a lease callback, return, and await this outside, with + // nothing admitted in between. + this.running = false; + this.providerAdmissionClosed = true; + this.closeStoreAdmission(); + // Whoever holds the store reservation (a lease's callback, a direct + // surgery's awaited work) finishes before anything it may use is torn + // down. Admission is closed, so no other holder can follow it. + const reservation = this.surgeryHold; + if (reservation) await reservation.released; + // So does a creation admitted before shutdown and still opening its + // context in the store (none is admitted after). + if (this.ephemeralInitializations.size > 0) await Promise.all(this.ephemeralInitializations); this.pushCoalescer?.suspend(); this.flushCoalescingSnapshot(); - this.running = false; // A re-run still waiting has no turn coming: its journal says launched, // which reads interrupted, and so does what its callers are told. for (const [key, waiter] of this.unstickAttemptWaiters) { @@ -2405,7 +2445,6 @@ export class AgentFramework { // Flushed-but-unsynced deferred writes: sync and ack now, while the // store is still open, rather than leaving them to a reboot replay. this.ackDeferredWrites(); - this.providerAdmissionClosed = true; this.queue.close(); this.tuneOutCoordinator?.stop(); @@ -4761,6 +4800,13 @@ export class AgentFramework { // From admission until the candidate runs or is cleaned up, the ticket // is pending and a waiting lease isn't granted. const pending = await this.admitEphemeralCreation(opts?.requestedBy); + // Admitted, but stop() may have begun before this continuation ran, and + // it only waits for initializations registered by then: refuse now, + // before touching the store, releasing the ticket. + if (this.stopping) { + this.ephemeralPending.delete(pending); + throw new Error('Ephemeral agent creation refused: the framework is stopping'); + } // Names are Chronicle namespaces and generation identities. Reserve before // opening the namespace so a second creation cannot append to an earlier // generation before runEphemeralToCompletion has a chance to reject it. @@ -4770,6 +4816,9 @@ export class AgentFramework { throw new Error(`Ephemeral agent name \"${config.name}\" is already registered or has been used in this framework`); } this.usedEphemeralAgentNames.add(config.name); + let initialized!: () => void; + const initializing = new Promise((resolve) => { initialized = resolve; }); + this.ephemeralInitializations.add(initializing); try { const namespace = `subagent/${config.name}`; @@ -4807,6 +4856,9 @@ export class AgentFramework { this.ephemeralPending.delete(pending); this.tryGrantSafeBoundary(); throw error; + } finally { + this.ephemeralInitializations.delete(initializing); + initialized(); } } @@ -4899,6 +4951,11 @@ export class AgentFramework { if (this.releasedEphemeralCandidates.has(agent)) { throw new Error(`Ephemeral agent "${agent.name}" was released by cleanup() and can't run; create a new one`); } + // Shutdown has begun: nothing new enters the store. Refused before the + // candidate is consumed; its caller cleans it up. + if (this.stopping) { + throw new Error(`Ephemeral agent "${agent.name}" refused: the framework is stopping`); + } // Only a fresh object returned by createEphemeralAgent may enter this path. // Never overwrite a resident/conversation owner or a concurrent run: their // cleanup is name-keyed and could cancel/deregister the legitimate owner. @@ -6763,6 +6820,9 @@ export class AgentFramework { } private reserveStoreForSurgery(verb: string, agentName: string): () => void { + if (this.stopping) { + throw new OperatorActionError('invalid', `Cannot ${verb}: the framework is stopping`); + } if (this.surgeryHold) { throw new OperatorActionError( 'agent-busy', @@ -6815,13 +6875,15 @@ export class AgentFramework { } // The token snapshot only covers agents that exist now; the hold covers // arrivals during the awaited switch (see surgeryHold). - const hold = { verb, agentName, since: Date.now() }; + let markReleased!: () => void; + const hold = { verb, agentName, since: Date.now(), released: new Promise((resolve) => { markReleased = resolve; }) }; this.surgeryHold = hold; let released = false; const release = () => { if (released) return; released = true; if (this.surgeryHold === hold) this.surgeryHold = null; + markReleased(); // Creations held behind this hold may proceed (unless a lease waits). queueMicrotask(() => this.notifyBoundaryCleared()); for (const [name, token] of reserved) { @@ -6854,6 +6916,18 @@ export class AgentFramework { return { tokens: reserved, release }; } + /** stop()'s first step for the store: no lease, direct surgery or + * ephemeral creation is admitted from now on, and every lease request or + * creation still waiting is refused, so nothing enters the store once + * shutdown has begun. */ + private closeStoreAdmission(): void { + this.stopping = true; + for (const waiter of this.boundaryWaiters?.splice(0) ?? []) waiter.refuse(stoppingRefusal(waiter.verb)); + // Creations parked behind a lease or surgery wake now and are refused + // (tryAdmitEphemeralCreation): none enters the store once stopping. + for (const resume of this.boundaryClearedWaiters?.splice(0) ?? []) resume(); + } + /** * Run `fn` at a safe boundary, holding the store reservation across it. * @@ -6875,6 +6949,13 @@ export class AgentFramework { * Operator methods that would refuse the hold take the lease instead: * `puppetToolCall(…, { lease })` runs under the lease's token for that * agent. + * + * Once stop() has begun, a request is refused, and one still waiting is + * rejected, with an AbortError saying the framework is stopping; nothing + * is granted after. A granted `fn` runs to its end before stop() tears + * down anything it may use. To shut down with no grant in between, a host + * calls stop() inside `fn`, returns, and awaits the shutdown outside: + * awaiting stop() inside `fn` waits for itself. */ async runAtSafeBoundary(opts: SafeBoundaryOptions, fn: (lease: SafeBoundaryLease) => Promise): Promise { const withdrawn = () => { @@ -6883,6 +6964,7 @@ export class AgentFramework { return error; }; if (opts.signal?.aborted) throw withdrawn(); + if (this.stopping) throw stoppingRefusal(opts.verb); const requested = Date.now(); const held = await new Promise<{ lease: SafeBoundaryLease; release: () => void }>((resolve, reject) => { let onAbort: (() => void) | undefined; @@ -6893,6 +6975,10 @@ export class AgentFramework { if (onAbort) opts.signal?.removeEventListener('abort', onAbort); resolve(granted); }, + refuse: (error: Error) => { + if (onAbort) opts.signal?.removeEventListener('abort', onAbort); + reject(error); + }, }; if (opts.signal) { onAbort = () => { @@ -6979,6 +7065,7 @@ export class AgentFramework { * to wait. The store hold (surgeryHold) covers a held lease and a direct * rollback or suppress alike: nothing initializes under either. */ private tryAdmitEphemeralCreation(requestedBy?: string): object | null { + if (this.stopping) throw new Error('Ephemeral agent creation refused: the framework is stopping'); const admitted = this.surgeryHold ? false : !this.boundaryWaiters?.length diff --git a/test/safe-boundary-lease.test.ts b/test/safe-boundary-lease.test.ts index 08c1f2fb0..a77af38b2 100644 --- a/test/safe-boundary-lease.test.ts +++ b/test/safe-boundary-lease.test.ts @@ -446,3 +446,197 @@ describe('runAtSafeBoundary', () => { } }); }); + +describe('stop() and the store reservation', () => { + let tempDir: string; + let membrane: MockMembrane; + let framework: AgentFramework; + let i: Internals; + let stopped: boolean; + let quiet: { log: typeof console.log; error: typeof console.error }; + + beforeEach(async () => { + tempDir = mkdtempSync(join(tmpdir(), 'safe-boundary-stop-')); + membrane = new MockMembrane(); + framework = await AgentFramework.create({ + storePath: join(tempDir, 'test.chronicle'), + membrane: membrane.asMembrane(), + agents: [{ name: 'scout', model: 'test-model', systemPrompt: 'You are scout.' }], + modules: [], + }); + i = framework as unknown as Internals; + stopped = false; + quiet = { log: console.log, error: console.error }; + console.log = () => {}; + console.error = () => {}; + }); + afterEach(async () => { + console.log = quiet.log; + console.error = quiet.error; + if (!stopped) await framework.stop(); + rmSync(tempDir, { recursive: true, force: true }); + }); + + // What a promise settled to within `ms`, or 'pending'. + const within = async (promise: Promise, ms = 200): Promise => { + let timer: ReturnType | undefined; + try { + return await Promise.race([ + promise.catch((e: Error) => e), + new Promise<'pending'>((resolve) => { timer = setTimeout(() => resolve('pending'), ms); }), + ]); + } finally { + clearTimeout(timer); + } + }; + const held = () => { + let release!: () => void; + const gate = new Promise((resolve) => { release = resolve; }); + let entered!: () => void; + const inside = new Promise((resolve) => { entered = resolve; }); + return { gate, release, inside, entered }; + }; + + it('waits for a granted callback before tearing down anything it uses, and the callback still has its store', async () => { + const lease = held(); + const leased = framework.runAtSafeBoundary({ verb: 'held' }, async () => { + lease.entered(); + await lease.gate; + framework.getStore().setStateJson('probe/after-stop', { ok: true }); // host bookkeeping after stop() began + framework.getStore().sync(); + return 'done'; + }); + await lease.inside; + stopped = true; + const stopping = framework.stop(); + assert.equal(await within(stopping), 'pending', 'stop() waits for the granted callback'); + lease.release(); + assert.equal(await leased, 'done', 'the callback finished against an open store'); + await stopping; + }); + + it('refuses a waiting request and any new one, and takes no direct surgery, once stop() begins', async () => { + const first = held(); + const firstDone = framework.runAtSafeBoundary({ verb: 'first' }, async () => { first.entered(); await first.gate; }); + await first.inside; + let secondRan = false; + const second = framework.runAtSafeBoundary({ verb: 'second' }, async () => { secondRan = true; }); + await tick(); + stopped = true; + const stopping = framework.stop(); + const refused = await within(second); + assert.ok(refused instanceof Error && refused.name === 'AbortError' + && /safe boundary for second refused: the framework is stopping/.test(refused.message), String(refused)); + const late = await within(framework.runAtSafeBoundary({ verb: 'late' }, async () => {})); + assert.ok(late instanceof Error && /late refused: the framework is stopping/.test(late.message), String(late)); + assert.throws(() => i.reserveStoreForSurgery('rollback', 'scout'), + (e: Error & { code?: string }) => e.code === 'invalid' && /Cannot rollback: the framework is stopping/.test(e.message)); + first.release(); + await firstDone; + await stopping; + assert.equal(secondRan, false, 'nothing was granted after stop() began'); + }); + + it('closes admission at once when stop() is initiated inside a callback: a queued wake never runs, and shutdown completes once it returns', async () => { + const calls = membrane.calls.length; + let stopping!: Promise; + stopped = true; + await framework.runAtSafeBoundary({ verb: 'handoff' }, async () => { + i.pendingRequests.push(wake('scout')); // a wake already queued + stopping = framework.stop(); // initiated, not awaited + const intruder = await within(framework.runAtSafeBoundary({ verb: 'intruder' }, async () => {})); + assert.ok(intruder instanceof Error && /intruder refused: the framework is stopping/.test(intruder.message), String(intruder)); + framework.getStore().setStateJson('probe/handoff', { applying: true }); // the callback's own work, store open + framework.getStore().sync(); + }); + // The lease is released: a scheduler pass now starts nothing. + await i.processInferenceRequests(); + assert.equal(await within(stopping, 5_000), undefined, 'shutdown completed'); + assert.equal(membrane.calls.length, calls, 'no provider call after the callback returned'); + }); + + // A strategy whose initializer (run as an ephemeral creation opens its + // context in the store) blocks until released, counting its calls. + const gatedStrategy = () => { + const gate = held(); + let calls = 0; + const strategy = new PassthroughStrategy(); + (strategy as unknown as { initialize: () => Promise }).initialize = async () => { calls++; gate.entered(); await gate.gate; }; + return { strategy, gate, calls: () => calls }; + }; + + it('refuses a creation parked behind a lease once stop() begins, before it enters the store', async () => { + const lease = held(); + const leased = framework.runAtSafeBoundary({ verb: 'held' }, async () => { lease.entered(); await lease.gate; }); + await lease.inside; + const init = gatedStrategy(); + const creating = framework.createEphemeralAgent({ name: 'parked', model: 'test-model', systemPrompt: 'test', strategy: init.strategy }); + await tick(); + stopped = true; + const stopping = framework.stop(); + const refused = await within(creating); + assert.ok(refused instanceof Error && /creation refused: the framework is stopping/.test(refused.message), String(refused)); + lease.release(); + await leased; + await stopping; + assert.equal(init.calls(), 0, 'its initializer never ran'); + }); + + it('lets a creation admitted before stop() finish initializing before the store closes', async () => { + const init = gatedStrategy(); + const creating = framework.createEphemeralAgent({ name: 'admitted', model: 'test-model', systemPrompt: 'test', strategy: init.strategy }); + await init.gate.inside; + stopped = true; + const stopping = framework.stop(); + assert.equal(await within(stopping), 'pending', 'stop() waits for the initializer'); + init.gate.release(); + const created = await creating; // no "Store has been closed" + created.cleanup(); + await stopping; + assert.equal(init.calls(), 1); + }); + + it('refuses a creation admitted in the same turn stop() begins, before its initializer runs', async () => { + const init = gatedStrategy(); + const creating = framework.createEphemeralAgent({ name: 'raced', model: 'test-model', systemPrompt: 'test', strategy: init.strategy }); + stopped = true; + const stopping = framework.stop(); // admitted above, its continuation not yet run + const refused = await within(creating); + assert.ok(refused instanceof Error && /creation refused: the framework is stopping/.test(refused.message), String(refused)); + await stopping; + assert.equal(init.calls(), 0, 'its initializer never ran'); + assert.equal(i.ephemeralPending.size, 0, 'its ticket was released'); + }); + + it('refuses to run a finished candidate once stop() begins, without consuming it', async () => { + const created = await framework.createEphemeralAgent({ name: 'finished', model: 'test-model', systemPrompt: 'test' }); + stopped = true; + const stopping = framework.stop(); + await assert.rejects( + framework.runEphemeralToCompletion(created.agent, created.contextManager), + /Ephemeral agent "finished" refused: the framework is stopping/, + ); + assert.equal(i.ephemeralCandidates.get(created.agent), created.contextManager, 'refused before consuming the candidate'); + created.cleanup(); + await stopping; + }); + + it("waits for a direct surgery's awaited switch before tearing down", async () => { + for (const text of ['one', 'two', 'three']) i.addMessage('user', [{ type: 'text', text }]); + const cm = framework.getAgent('scout')!.getContextManager(); + const anchor = String(cm.getAllMessages()[0]!.id); + const cmx = cm as unknown as { switchBranch: (name: string) => Promise }; + const real = cmx.switchBranch.bind(cm); + const switching = held(); + cmx.switchBranch = async (name) => { switching.entered(); await switching.gate; return real(name); }; + const rollback = framework.rollbackToMessage('scout', { messageId: anchor }); + await switching.inside; + stopped = true; + const stopping = framework.stop(); + assert.equal(await within(stopping), 'pending', 'stop() waits for the surgery holding the store'); + switching.release(); + const done = await rollback; + assert.ok(done, 'the surgery completed against an open store'); + await stopping; + }); +}); From 1f6b78c6c34ead32992cd7156ed291ae23d0e518 Mon Sep 17 00:00:00 2001 From: Alfred-1883 Date: Thu, 8 Oct 2026 16:46:02 -0400 Subject: [PATCH 62/73] fix(framework): a caller-named origin is normalized, so a falsy one is the host's, never the agent's own (#255 review P6) slimepriestess's review of #255 (pullrequestreview-5462207876), finding P6. ModuleContext.callTool read `call.origin ?? ambient`, which keeps `''`, `0`, `false` or `NaN`, and the later `...(origin ? { origin } : {})` dropped it. The call then carried no origin, which gatedSelfChange reads as the agent's own model call, so an untyped module's agent_settings applied with the gate never asked (her probe: budget 100000 -> 136000). The types forbid those values, but a JS module or host can pass anything. `normalizeOrigin` (call-provenance.ts) is now the one reading of a caller-supplied origin: undefined or null stays absent, 'puppet' stays the operator's, and any other value is the host's. It applies wherever a caller names one: - ModuleContext.callTool, before falling back to the ambient initiator; - the public executeToolCall, which otherwise passes `call.origin` through; - ModuleRegistry.handleToolCall, for the call a module handler receives and for the provenance its delegated work inherits. Tests (tool-call-origin, a new block with a gate configured: "the agent's own, and nothing else, skips the gate"): - `''`, `0`, `false`, `NaN`, `'agent'`, `'PUPPET'` and `{}` from a module's callTool each reach the gate as the host's; `'puppet'` as the puppet's; null and undefined with no call behind them as the host's; and `''` through the public executeToolCall as the host's. The budget is unchanged throughout. Mutant, checked on this batch's final tree: treating any falsy origin as absent fails that test. Co-Authored-By: Alfred (Claude Opus 5.5) --- src/call-provenance.ts | 11 +++ src/framework.ts | 12 ++-- src/module-registry.ts | 7 +- test/tool-call-origin.test.ts | 129 +++++++++++++++++++++++++++++++++- 4 files changed, 151 insertions(+), 8 deletions(-) diff --git a/src/call-provenance.ts b/src/call-provenance.ts index 7d208e821..4fc1976a4 100644 --- a/src/call-provenance.ts +++ b/src/call-provenance.ts @@ -17,3 +17,14 @@ import type { ToolCall } from './types/index.js'; export type CallProvenance = Pick; export const callProvenance = new AsyncLocalStorage(); + +/** + * A caller-supplied origin as the framework reads it: absent (undefined or + * null) stays absent, 'puppet' stays the operator's, and any other value is + * the host's. Types forbid other values, but an untyped module or host can + * pass anything, and a falsy one must never read as the agent's own. + */ +export function normalizeOrigin(origin: unknown): ToolCall['origin'] { + if (origin === undefined || origin === null) return undefined; + return origin === 'puppet' ? 'puppet' : 'host'; +} diff --git a/src/framework.ts b/src/framework.ts index b9e20de66..50940adae 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -9,7 +9,7 @@ import { MembraneError } from '@animalabs/membrane'; import { ContextManager, PassthroughStrategy, WindowedPassthroughStrategy, OverBudgetError, UncoveredDropError } from '@animalabs/context-manager'; import type { CacheWireReceipt } from './kv-unified-wire.js'; import { SUBCONSCIOUS_TOOLS, SUBCONSCIOUS_TOOL_NAMES, type SubconsciousConfig } from './tune-out/tools.js'; -import { callProvenance } from './call-provenance.js'; +import { callProvenance, normalizeOrigin } from './call-provenance.js'; import { selfChangeKind, sameValue, @@ -1911,7 +1911,7 @@ export class AgentFramework { // call behind it (a module's own timer or event) the host. callTool: (call) => { const ambient = callProvenance.getStore(); - const origin = call.origin ?? (ambient ? ambient.origin : 'host'); + const origin = normalizeOrigin(call.origin) ?? (ambient ? ambient.origin : 'host'); const admission = call.admission ?? ambient?.admission; const { origin: _o, admission: _a, ...rest } = call; return this.executeToolCallFrom( @@ -14342,10 +14342,14 @@ export class AgentFramework { * never confer module trust, whatever the input claims. Module origin * exists only through the private closure handed to ModuleRegistry * (ctx.callTool → executeToolCallFrom with {kind:'module'}). Callers that - * don't identify themselves fail safe to agent semantics too. + * don't identify themselves fail safe to agent semantics too. An origin the + * caller names is normalized (normalizeOrigin): a value other than + * 'puppet' or 'host' is the host's, never the agent's own. */ async executeToolCall(call: ToolCall): Promise { - return this.executeToolCallFrom(call, { + const { origin: named, ...rest } = call; + const origin = normalizeOrigin(named); + return this.executeToolCallFrom({ ...rest, ...(origin ? { origin } : {}) }, { kind: 'agent', agentName: call.callerAgentName ?? '__ephemeral__', }); diff --git a/src/module-registry.ts b/src/module-registry.ts index f02733ab7..11488cad3 100644 --- a/src/module-registry.ts +++ b/src/module-registry.ts @@ -25,7 +25,7 @@ import type { TraceEventListener, } from './types/index.js'; import type { Agent } from './agent.js'; -import { callProvenance } from './call-provenance.js'; +import { callProvenance, normalizeOrigin } from './call-provenance.js'; const MODULE_STATE_PREFIX = 'modules/'; @@ -301,12 +301,13 @@ export class ModuleRegistry { // Create a call with the un-prefixed name (preserving caller identity, // who initiated it, and any operator admission it carries) + const origin = normalizeOrigin(call.origin); const moduleCall: ToolCall = { id: call.id, name: toolName, input: call.input, callerAgentName: call.callerAgentName, - ...(call.origin ? { origin: call.origin } : {}), + ...(origin ? { origin } : {}), ...(call.admission ? { admission: call.admission } : {}), }; @@ -315,7 +316,7 @@ export class ModuleRegistry { // so a tool it delegates to through ctx.callTool keeps that actor. return await callProvenance.run( { - ...(call.origin ? { origin: call.origin } : {}), + ...(origin ? { origin } : {}), ...(call.admission ? { admission: call.admission } : {}), }, () => module.handleToolCall(moduleCall), diff --git a/test/tool-call-origin.test.ts b/test/tool-call-origin.test.ts index 5e6016606..7e59a44fb 100644 --- a/test/tool-call-origin.test.ts +++ b/test/tool-call-origin.test.ts @@ -14,7 +14,7 @@ import { tmpdir } from 'node:os'; import { mkdtempSync, rmSync } from 'node:fs'; import { join } from 'node:path'; import { AgentFramework } from '../src/index.js'; -import type { InferenceRequest, Module, ModuleContext, ToolCall, ToolDefinition, ToolResult } from '../src/index.js'; +import type { InferenceRequest, Module, ModuleContext, ResolvedOperatorChange, ToolCall, ToolDefinition, ToolResult } from '../src/index.js'; import { MockMembrane, createMockResponse } from './helpers/mock-membrane.js'; class ProbeModule implements Module { @@ -173,3 +173,130 @@ describe('tool-call origin', () => { ]); }); }); + +/** + * The gate's boundary is call origin: only the agent's own model call, or + * work that call started for that same agent, applies a self-change + * directly. Each path below once labelled an imposed change as the agent's + * own (found in #255's review); each must reach the gate now, as the actor + * who imposed it, with nothing applied. + */ +class Delegator implements Module { + readonly name = 'dlg'; + ctx!: ModuleContext; + last?: Promise; + async start(ctx: ModuleContext): Promise { this.ctx = ctx; } + async stop(): Promise {} + getTools(): ToolDefinition[] { + return ['retarget', 'self_set', 'notify'].map((name) => ({ name, description: name, inputSchema: { type: 'object', properties: {} } })); + } + async handleToolCall(call: ToolCall): Promise { + const input = (call.input ?? {}) as { target?: string; tokens?: number }; + if (call.name === 'retarget') { + return this.ctx.callTool({ id: 'r', name: 'agent_settings', input: budget(input.tokens ?? 131_000), callerAgentName: input.target }); + } + if (call.name === 'self_set') { + this.last = this.ctx.callTool({ id: 's', name: 'agent_settings', input: budget(input.tokens ?? 132_000), callerAgentName: call.callerAgentName }); + return this.last; + } + // notify: pushes an ordinary event and nothing more. + this.ctx.pushEvent({ type: 'custom', name: 'dlg-notify', data: {} } as never); + return { success: true, data: 'notified' }; + } + async onProcess(): Promise> { return {}; } +} + +/** A separate module that reacts to traces on its own: the host's actor. */ +class Governor implements Module { + readonly name = 'gov'; + ctx!: ModuleContext; + now?: Promise; + later?: Promise; + async start(ctx: ModuleContext): Promise { + this.ctx = ctx; + ctx.onTrace((ev) => { + const event = ev as { type: string; processEvent?: { name?: string } }; + if (event.type !== 'process:received' || event.processEvent?.name !== 'dlg-notify' || this.now) return; + this.now = ctx.callTool({ id: 'g1', name: 'agent_settings', input: budget(133_000), callerAgentName: 'scout' }); + this.later = new Promise((resolve) => setTimeout(() => resolve( + ctx.callTool({ id: 'g2', name: 'agent_settings', input: budget(134_000), callerAgentName: 'scout' }), + ), 5)); + }); + } + async stop(): Promise {} + getTools(): ToolDefinition[] { return []; } + async handleToolCall(): Promise { return { success: true }; } + async onProcess(): Promise> { return {}; } +} + +const budget = (tokens: number) => ({ action: 'update', context_budget_tokens: tokens }); + +describe("tool-call origin: the agent's own, and nothing else, skips the gate", () => { + let tempDir: string; + let membrane: MockMembrane; + let dlg: Delegator; + let gov: Governor; + let framework: AgentFramework; + let asked: ResolvedOperatorChange[]; + let quiet: { log: typeof console.log; error: typeof console.error }; + + beforeEach(async () => { + tempDir = mkdtempSync(join(tmpdir(), 'tool-call-origin-gate-')); + membrane = new MockMembrane(); + dlg = new Delegator(); + gov = new Governor(); + asked = []; + framework = await AgentFramework.create({ + storePath: join(tempDir, 'test.chronicle'), + membrane: membrane.asMembrane(), + agents: [ + { name: 'scout', model: 'test-model', systemPrompt: 'You are scout.', maxTokens: 1000 }, + { name: 'other', model: 'test-model', systemPrompt: 'You are other.', maxTokens: 1000 }, + ], + modules: [dlg, gov], + codeExecution: { enabled: true }, + operatorChangeGate: async (change) => { + asked.push(structuredClone(change)); + return { id: `rev-${asked.length}`, text: `staged ${change.kind}` }; + }, + }); + quiet = { log: console.log, error: console.error }; + console.log = () => {}; + console.error = () => {}; + }); + afterEach(async () => { + console.log = quiet.log; + console.error = quiet.error; + await framework.stop(); + rmSync(tempDir, { recursive: true, force: true }); + }); + + const tokens = (agent = 'scout') => framework.getAgentRuntimeSettings(agent).contextBudgetTokens; + const modelCalls = async (name: string, input: Record) => { + membrane.pushResponse(createMockResponse([{ type: 'tool_use', id: `toolu_${name.replace(/\W/g, '')}`, name, input }], 'tool_use')); + membrane.pushResponse(createMockResponse([{ type: 'text', text: 'done' }])); + (framework as unknown as Internals).pendingRequests.push( + { agentName: 'scout', reason: 'mcpl:channel-incoming', source: 'test', timestamp: Date.now() } as InferenceRequest, + ); + await framework.runUntilIdle(); + }; + const surfaces = () => asked.map((c) => `${c.agent}:${c.surface}`); + + it("reads an origin a module names as the host's unless it's 'puppet', never as the agent's own", async () => { + for (const origin of ['', 0, false, Number.NaN, 'agent', 'PUPPET', {}]) { + await dlg.ctx.callTool({ id: 'n', name: 'agent_settings', input: budget(136_000), callerAgentName: 'scout', origin } as unknown as ToolCall); + } + await dlg.ctx.callTool({ id: 'p', name: 'agent_settings', input: budget(137_000), callerAgentName: 'scout', origin: 'puppet' }); + for (const origin of [null, undefined]) { + await dlg.ctx.callTool({ id: 'u', name: 'agent_settings', input: budget(138_000), callerAgentName: 'scout', origin } as unknown as ToolCall); + } + await framework.executeToolCall({ id: 'x', name: 'agent_settings', input: budget(139_000), callerAgentName: 'scout', origin: '' } as unknown as ToolCall); + assert.deepEqual(surfaces(), [ + ...Array(7).fill('scout:host'), + 'scout:puppet', + 'scout:host', 'scout:host', // no origin and no call behind it: the host + 'scout:host', // the public entry normalizes too + ]); + assert.equal(tokens(), 100_000); + }); +}); From b46db529bf741a3682bfe393a664b1b307cf5f1e Mon Sep 17 00:00:00 2001 From: Alfred-1883 Date: Thu, 8 Oct 2026 16:47:41 -0400 Subject: [PATCH 63/73] fix(framework): a delegation inherits the agent's own origin only for that same agent (#255 review P1) slimepriestess's review of #255 (pullrequestreview-5462207876), finding P1. A module tool that scout's model called could delegate `agent_settings` with `callerAgentName: 'other'`. The ambient store recorded the initiating call's origin and admission, but not which agent that call acted for. So "no origin" was read as `other`'s own self-change: one resident's turn changed another resident's body, with the gate never asked (her probe: other's budget 100000 -> 131000). The store now records the agent the initiating call acted for (`CallProvenance.agent`, from its callerAgentName). ModuleRegistry.handleToolCall sets it around every handler, and `delegatedOrigin` (call-provenance.ts) decides who a delegated call acts for: - an origin the module names, normalized (P6); - otherwise the initiating call's origin: an operator's or the host's stays theirs, and the agent's own carries over only to a call for that same agent; - a call for any other agent, or one naming no agent, acts for the host; - with no call behind it, the host, as before. So a delegation on the model's behalf to its own body still applies directly, the documented case that delegation exists for. The store is bound to the initiating agent rather than to the module whose handler runs, so a chain of modules acting on the model's request keeps it. Tests (tool-call-origin, "the agent's own, and nothing else, skips the gate"): - The control: the agent's own agent_settings applies directly, and so does a change a module delegates from the model's call for that same agent. The gate is never asked. - scout's model calls a module tool that delegates a change for `other`. It reaches the gate as `other:host`, and neither budget moves. Mutant, checked on this batch's final tree: letting the agent's own origin carry over to any agent fails the cross-agent test. Co-Authored-By: Alfred (Claude Opus 5.5) --- src/call-provenance.ts | 23 ++++++++++++++++++++++- src/framework.ts | 9 +++++---- src/module-registry.ts | 4 +++- test/tool-call-origin.test.ts | 15 +++++++++++++++ 4 files changed, 45 insertions(+), 6 deletions(-) diff --git a/src/call-provenance.ts b/src/call-provenance.ts index 4fc1976a4..647716133 100644 --- a/src/call-provenance.ts +++ b/src/call-provenance.ts @@ -8,13 +8,18 @@ * the work: the agent's model (no origin), an operator's puppet, or the * host. That includes delegation from a callback the handler scheduled * after returning, because AsyncLocalStorage follows the work it started. + * The agent's own origin passes on only to a call for that same agent: one + * agent's turn never changes another agent's body as its own. * Work that no tool call started has no store: a module's own timer, or its * event handler, acts for the host. */ import { AsyncLocalStorage } from 'node:async_hooks'; import type { ToolCall } from './types/index.js'; -export type CallProvenance = Pick; +export type CallProvenance = Pick & { + /** The agent the initiating call acted for (its callerAgentName). */ + agent?: string; +}; export const callProvenance = new AsyncLocalStorage(); @@ -28,3 +33,19 @@ export function normalizeOrigin(origin: unknown): ToolCall['origin'] { if (origin === undefined || origin === null) return undefined; return origin === 'puppet' ? 'puppet' : 'host'; } + +/** + * Who a module's delegated call (ModuleContext.callTool) acts for: the origin + * the module named, normalized; otherwise the initiating call's origin, the + * agent's own only for that same agent; and with no call behind it, the host. + */ +export function delegatedOrigin( + call: Pick, + ambient: CallProvenance | undefined, +): ToolCall['origin'] { + const named = normalizeOrigin(call.origin); + if (named) return named; + if (!ambient) return 'host'; + if (ambient.origin) return ambient.origin; + return ambient.agent !== undefined && call.callerAgentName === ambient.agent ? undefined : 'host'; +} diff --git a/src/framework.ts b/src/framework.ts index 50940adae..8940ccbe2 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -9,7 +9,7 @@ import { MembraneError } from '@animalabs/membrane'; import { ContextManager, PassthroughStrategy, WindowedPassthroughStrategy, OverBudgetError, UncoveredDropError } from '@animalabs/context-manager'; import type { CacheWireReceipt } from './kv-unified-wire.js'; import { SUBCONSCIOUS_TOOLS, SUBCONSCIOUS_TOOL_NAMES, type SubconsciousConfig } from './tune-out/tools.js'; -import { callProvenance, normalizeOrigin } from './call-provenance.js'; +import { callProvenance, delegatedOrigin, normalizeOrigin } from './call-provenance.js'; import { selfChangeKind, sameValue, @@ -1907,11 +1907,12 @@ export class AgentFramework { // executeToolCall() is shared with model/ephemeral callers and always // stamps agent origin (see there). // Who asked travels with a delegated call: inside a module tool - // handler (or work it started) the initiator of that call; with no - // call behind it (a module's own timer or event) the host. + // handler (or work it started) the initiator of that call, the + // agent's own only for that same agent; with no call behind it (a + // module's own timer or event) the host (delegatedOrigin). callTool: (call) => { const ambient = callProvenance.getStore(); - const origin = normalizeOrigin(call.origin) ?? (ambient ? ambient.origin : 'host'); + const origin = delegatedOrigin(call, ambient); const admission = call.admission ?? ambient?.admission; const { origin: _o, admission: _a, ...rest } = call; return this.executeToolCallFrom( diff --git a/src/module-registry.ts b/src/module-registry.ts index 11488cad3..51129cc0e 100644 --- a/src/module-registry.ts +++ b/src/module-registry.ts @@ -313,11 +313,13 @@ export class ModuleRegistry { try { // The handler, and any work it starts, runs as this call's initiator, - // so a tool it delegates to through ctx.callTool keeps that actor. + // so a tool it delegates to through ctx.callTool keeps that actor (the + // agent's own origin only for this same agent). return await callProvenance.run( { ...(origin ? { origin } : {}), ...(call.admission ? { admission: call.admission } : {}), + ...(call.callerAgentName !== undefined ? { agent: call.callerAgentName } : {}), }, () => module.handleToolCall(moduleCall), ); diff --git a/test/tool-call-origin.test.ts b/test/tool-call-origin.test.ts index 7e59a44fb..ccb4c3fc2 100644 --- a/test/tool-call-origin.test.ts +++ b/test/tool-call-origin.test.ts @@ -282,6 +282,21 @@ describe("tool-call origin: the agent's own, and nothing else, skips the gate", }; const surfaces = () => asked.map((c) => `${c.agent}:${c.surface}`); + it("applies the agent's own change, and one a module delegates for that same agent, directly", async () => { + await modelCalls('agent_settings', budget(125_000)); + assert.equal(tokens(), 125_000); + await modelCalls('dlg--self_set', { tokens: 126_000 }); + await dlg.last; + assert.equal(tokens(), 126_000); + assert.deepEqual(asked, [], 'the gate is never asked'); + }); + + it("stages a change a module delegates for another agent from one agent's own call, as the host's", async () => { + await modelCalls('dlg--retarget', { target: 'other', tokens: 131_000 }); + assert.deepEqual(surfaces(), ['other:host'], "scout's turn can't change other's body as other's own"); + assert.deepEqual([tokens('other'), tokens()], [100_000, 100_000]); + }); + it("reads an origin a module names as the host's unless it's 'puppet', never as the agent's own", async () => { for (const origin of ['', 0, false, Number.NaN, 'agent', 'PUPPET', {}]) { await dlg.ctx.callTool({ id: 'n', name: 'agent_settings', input: budget(136_000), callerAgentName: 'scout', origin } as unknown as ToolCall); From 5e9e0100aa709d4920e8d1a2b41e13d7bdc8205f Mon Sep 17 00:00:00 2001 From: Alfred-1883 Date: Thu, 8 Oct 2026 16:48:07 -0400 Subject: [PATCH 64/73] fix(framework): only the framework's own dispatch gives a tool-call event's call its origin; any other acts for the host (#255 review P2) slimepriestess's review of #255 (pullrequestreview-5462207876), finding P2. A module could push a `tool-call` event of its own, `ctx.pushEvent({ type: 'tool-call', ... })`, with no tool call underway. It reached dispatchToolCallEvent and moduleRegistry.handleToolCall with no origin stamped. So a delegated agent_settings in that handler read as the agent's own, and applied with the gate never asked (her probe: budget 100000 -> 132000). It was an entry point that never stamped the host. The model's tool calls reach modules as tool-call events the framework's own dispatch (dispatchToolCall) creates. It's the only producer in the framework, and nothing re-pushes one. That dispatch now records each event it creates in a private WeakSet, and dispatchToolCallEvent reads a call's origin as given only for those. An event queued by anything else (a module through ctx.pushEvent, ctx.queue or ProcessState, or the host through framework.pushEvent) acts for the host unless it names the operator (normalizeOrigin). The check is made where the event is consumed, so it doesn't depend on every push path stamping. Membership is checked with `has`, not consumed, so a later requeue of the framework's own event keeps its origin. Tests (tool-call-origin, "the agent's own, and nothing else, skips the gate"): - A module pushes a tool-call event for its own delegating tool, then another that names the operator. They reach the gate as `scout:host` and `scout:puppet`, and the budget is unchanged. - The existing "leaves the agent's own model calls unmarked" test still passes: the framework's own events keep the model's origin. Mutant, checked on this batch's final tree: trusting every tool-call event fails the pushed-event test. Co-Authored-By: Alfred (Claude Opus 5.5) --- src/framework.ts | 21 ++++++++++++++++++--- test/tool-call-origin.test.ts | 15 +++++++++++++++ 2 files changed, 33 insertions(+), 3 deletions(-) diff --git a/src/framework.ts b/src/framework.ts index 8940ccbe2..458d82853 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -1701,6 +1701,13 @@ export class AgentFramework { /** Forks whose TTL closure turn has been queued — disposed (removed from * agents/agentConfigs/conversationAgentHomes) when their stream ends. */ private closingConversationAgents: Set = new Set(); + /** + * The tool-call events this framework's own dispatch (dispatchToolCall) + * created. Only these carry their call's origin as given, the agent's own + * included: a tool-call event a module or the host pushed on its own acts + * for the host (dispatchToolCallEvent). + */ + private readonly dispatchedToolCallEvents = new WeakSet(); // Client-side programmatic tool calling (`code_execution`). Null unless // config.codeExecution.enabled. One PyRunner per agent (interpreter state // is per-agent, like a per-agent container); waiters resolve script-inner @@ -15863,19 +15870,27 @@ export class AgentFramework { const moduleName = sepIndex >= 0 ? enrichedCall.name.substring(0, sepIndex) : 'unknown'; const toolName = sepIndex >= 0 ? call.name.substring(sepIndex + 2) : call.name; - this.pushEvent({ + const event: ToolCallEvent = { type: 'tool-call', callId: call.id, agentName, moduleName, toolName, call, - }); + }; + this.dispatchedToolCallEvents.add(event); + this.pushEvent(event); } private dispatchToolCallEvent(event: ToolCallEvent): void { - const { call, agentName, moduleName } = event; + const { agentName, moduleName } = event; + // Only this framework's own dispatch speaks for the call's initiator; a + // tool-call event anything else queued (a module, the host) acts for the + // host unless it names the operator. + const call: ToolCall = this.dispatchedToolCallEvents.has(event) + ? event.call + : { ...event.call, origin: normalizeOrigin(event.call.origin) ?? 'host' }; this.emitTrace({ type: 'tool:started', module: moduleName, diff --git a/test/tool-call-origin.test.ts b/test/tool-call-origin.test.ts index ccb4c3fc2..6500a3319 100644 --- a/test/tool-call-origin.test.ts +++ b/test/tool-call-origin.test.ts @@ -297,6 +297,21 @@ describe("tool-call origin: the agent's own, and nothing else, skips the gate", assert.deepEqual([tokens('other'), tokens()], [100_000, 100_000]); }); + it('stages a delegated change from a tool-call event a module pushed on its own, as the host\'s unless it names the operator', async () => { + const push = (callId: string, origin?: unknown) => dlg.ctx.pushEvent({ + type: 'tool-call', callId, agentName: 'scout', moduleName: 'dlg', toolName: 'self_set', + call: { id: callId, name: 'dlg--self_set', input: {}, ...(origin !== undefined ? { origin } : {}) }, + } as never); + push('pushed-1'); + await framework.runUntilIdle(); + await dlg.last; + push('pushed-2', 'puppet'); + await framework.runUntilIdle(); + await dlg.last; + assert.deepEqual(surfaces(), ['scout:host', 'scout:puppet']); + assert.equal(tokens(), 100_000); + }); + it("reads an origin a module names as the host's unless it's 'puppet', never as the agent's own", async () => { for (const origin of ['', 0, false, Number.NaN, 'agent', 'PUPPET', {}]) { await dlg.ctx.callTool({ id: 'n', name: 'agent_settings', input: budget(136_000), callerAgentName: 'scout', origin } as unknown as ToolCall); From 58139d5b04eca0e3815d0ad247702e6567049ae6 Mon Sep 17 00:00:00 2001 From: Alfred-1883 Date: Thu, 8 Oct 2026 16:48:34 -0400 Subject: [PATCH 65/73] fix(framework): trace listeners are notified as the host's, never as the call that emitted the trace (#255 review P3) slimepriestess's review of #255 (pullrequestreview-5462207876), finding P3. A model-called module tool that only pushed an event made emitTrace('process:received') run every trace listener synchronously inside that tool's `callProvenance.run({})`. So a *separate* module's onTrace listener that delegated agent_settings inherited the model's own origin, and applied with the gate never asked (her probe: budget 100000 -> 133000). Any timer such a listener scheduled inherited it too. That contradicted call-provenance.ts's own rule that a module's event handler acts for the host. emitTrace now notifies its listeners outside the emitting call's provenance. When a store is active, it runs them under HOST_PROVENANCE (`{ origin: 'host' }`), so a listener, and anything it starts, acts for the host. With no store active it calls them directly, as before, so the hot path (streaming traces) is unchanged. Hilda-1882 checked the other Module hooks for the same synchronous reach into a handler's store (room-320 #57952, #58247). gatherContext runs at turn start, and also in previewActivation's opt-in injection preview, which a host can call from anywhere. onAgentSpeech runs from the turn loop. onProcess runs from the event loop, whose `await queue.pop()` continuation doesn't inherit the pusher's store. An agent-settings extension's `update` does run inside the delegating store, but it is part of the same change, so inheriting is right there. Tests (tool-call-origin, "the agent's own, and nothing else, skips the gate"): - The model calls a module tool that only pushes an event. Another module's trace listener delegates a change at once and schedules another. Both reach the gate as `scout:host`, and the budget is unchanged. Mutant, checked on this batch's final tree: notifying listeners inside the emitting call's store fails that test. Co-Authored-By: Alfred (Claude Opus 5.5) --- src/call-provenance.ts | 6 +++++- src/framework.ts | 22 +++++++++++++++------- test/tool-call-origin.test.ts | 9 +++++++++ 3 files changed, 29 insertions(+), 8 deletions(-) diff --git a/src/call-provenance.ts b/src/call-provenance.ts index 647716133..673c1fe74 100644 --- a/src/call-provenance.ts +++ b/src/call-provenance.ts @@ -11,7 +11,8 @@ * The agent's own origin passes on only to a call for that same agent: one * agent's turn never changes another agent's body as its own. * Work that no tool call started has no store: a module's own timer, or its - * event handler, acts for the host. + * event handler, acts for the host. A trace listener is notified as the + * host's too, even when the trace was emitted inside another module's call. */ import { AsyncLocalStorage } from 'node:async_hooks'; import type { ToolCall } from './types/index.js'; @@ -23,6 +24,9 @@ export type CallProvenance = Pick & { export const callProvenance = new AsyncLocalStorage(); +/** The provenance of work the host does on its own, such as notifying a listener. */ +export const HOST_PROVENANCE: CallProvenance = Object.freeze({ origin: 'host' }); + /** * A caller-supplied origin as the framework reads it: absent (undefined or * null) stays absent, 'puppet' stays the operator's, and any other value is diff --git a/src/framework.ts b/src/framework.ts index 458d82853..9a049bd48 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -9,7 +9,7 @@ import { MembraneError } from '@animalabs/membrane'; import { ContextManager, PassthroughStrategy, WindowedPassthroughStrategy, OverBudgetError, UncoveredDropError } from '@animalabs/context-manager'; import type { CacheWireReceipt } from './kv-unified-wire.js'; import { SUBCONSCIOUS_TOOLS, SUBCONSCIOUS_TOOL_NAMES, type SubconsciousConfig } from './tune-out/tools.js'; -import { callProvenance, delegatedOrigin, normalizeOrigin } from './call-provenance.js'; +import { callProvenance, delegatedOrigin, HOST_PROVENANCE, normalizeOrigin } from './call-provenance.js'; import { selfChangeKind, sameValue, @@ -16254,13 +16254,21 @@ export class AgentFramework { timestamp: Date.now(), } as TraceEvent; - for (const listener of this.traceListeners) { - try { - listener(traceEvent); - } catch (error) { - console.error('Trace listener error:', error); + const notify = () => { + for (const listener of this.traceListeners) { + try { + listener(traceEvent); + } catch (error) { + console.error('Trace listener error:', error); + } } - } + }; + // A listener observes the work that emitted the trace; it never acts as + // the call behind it. Inside a module's tool call, listeners (another + // module's included) are notified as the host's, and so is anything they + // start (call-provenance.ts). + if (callProvenance.getStore()) callProvenance.run(HOST_PROVENANCE, notify); + else notify(); } /** diff --git a/test/tool-call-origin.test.ts b/test/tool-call-origin.test.ts index 6500a3319..45a98a69c 100644 --- a/test/tool-call-origin.test.ts +++ b/test/tool-call-origin.test.ts @@ -312,6 +312,15 @@ describe("tool-call origin: the agent's own, and nothing else, skips the gate", assert.equal(tokens(), 100_000); }); + it("stages a change another module's trace listener makes during the agent's own call, and one it schedules, as the host's", async () => { + await modelCalls('dlg--notify', {}); + assert.ok(gov.now, 'the listener fired'); + await gov.now; + await gov.later; + assert.deepEqual(surfaces(), ['scout:host', 'scout:host']); + assert.equal(tokens(), 100_000); + }); + it("reads an origin a module names as the host's unless it's 'puppet', never as the agent's own", async () => { for (const origin of ['', 0, false, Number.NaN, 'agent', 'PUPPET', {}]) { await dlg.ctx.callTool({ id: 'n', name: 'agent_settings', input: budget(136_000), callerAgentName: 'scout', origin } as unknown as ToolCall); From e5cf57ad8f0ef0d44e846fa8622686ff147ec1e7 Mon Sep 17 00:00:00 2001 From: Alfred-1883 Date: Thu, 8 Oct 2026 16:49:36 -0400 Subject: [PATCH 66/73] fix(code-execution): only the running exec's own work is served, as whoever started it; a task an earlier script left running is refused (#255 review P4, P4b) slimepriestess's review of #255 (pullrequestreview-5462207876), findings P4 and P4b, the strongest. - **P4.** A puppet's code_execution script ran `asyncio.ensure_future(_later())`, and `_later` called agent_settings 0.4 s after the exec had returned "scheduled". `scriptCallProvenance` had been cleared in the exec's `finally`, but the leftover task's `tool_call` still reached PyRunner.handleLine, which checked neither `exec_id` nor busy state. So onToolCall read empty provenance, and the operator's change applied as the agent's own (budget 100000 -> 134000). - **P4b.** The same task firing during the agent's own later exec was attributed to that exec, because the runtime's `exec_id` was the global `_current_exec_id` (-> 135000). - **The inverse,** which Hilda-1882 probed: the model's leftover task firing during a puppet's later exec was staged as the puppet's. **Each call reports its own exec.** The Python runtime binds the exec id in a `contextvars.ContextVar` when an exec starts. Every task and callback its script starts copies the context it was created in, so a tool_call or wake reports the exec whose work made it, even after that exec ends or while a later one runs. The global `_current_exec_id` is gone. **Only the running exec is served.** PyRunner carries each exec's provenance (`exec(..., { provenance })`) on its pending exec, and serves a tool_call or wake only when it reports the exec now running. That call is made as whoever started the exec (ScriptToolCallHandler now receives the provenance). Anything else gets an error string: - a task an earlier script left running, whose exec's standing (its turn token, lease or admission) ended with it; - work that can't be traced to the running script, such as a coroutine a script's own `threading.Thread` sends with `run_coroutine_threadsafe` (threads don't inherit contextvars). The framework's per-agent `scriptCallProvenance` map is gone. That removes one consequence of a race: two puppet execs under one lease could pass the runner's busy check together, and the second's provenance replaced the first's, so the first's inner calls read as the agent's own. On this branch PyRunner.exec still claims the runner only after its first await, so the second exec can replace the first's pending entry. Now that can leave a call refused or an exec unsettled, but never mislabelled. `main` already claims the runner before the await (037afd8), which this branch takes up at its rebase. A background script passes its starter's provenance through the same exec option. **A behaviour change, said where the script's author reads it.** A model script's fire-and-forget tool call that starts after its exec has ended used to run, unrecorded, as the agent's own. It is refused now. So when a foreground script ends with tasks it started still running, the runtime adds a note to the exec's stderr: "[code_execution] N task(s) this script started were still running when it ended. Their tool calls are refused from now on: await them before the script ends, or use a background script." It counts only this exec's own tasks: by the context each task runs in where Python exposes it (3.12+), and otherwise by "created during this exec". A background interpreter is disposed with its exec, so its tasks end with it and no note is added. The review's P4/P4b probes expect the leftover call to be staged as the puppet's (`asked === 1`). With this change it is refused instead (`asked === 0`), with the budget unchanged. Refusal was one of the two remedies the review offered. Tests (tool-call-origin, real python3): - **After its exec and during a later one.** A puppet's script leaves a task that calls after the exec ends: the result's stderr carries the note, and the task gets the refusal. A second puppet script leaves a task that waits until the agent's own script is running, then calls: it is refused too. The agent's script then makes its own change, which applies directly. The gate is never asked. - **A thread.** A script's own thread sends a call with `run_coroutine_threadsafe`: it gets the "can't be traced" refusal, while the script's own call reaches the gate as the puppet's. - **No note** for tasks that finished with the script, or for a thread it joined. - **A background script** a puppet starts makes its call as the puppet's. Mutants, checked on this batch's final tree: - Serving a call from any exec fails the leftover test. - Reporting the current exec instead of the task's own (the old global) fails it, at the during-a-later-exec step. - Serving a call with no exec fails the thread test. - Dropping the note fails the leftover test. - Losing a background script's provenance fails the background test. Co-Authored-By: Alfred (Claude Opus 5.5) --- src/code-execution/py-runner.ts | 47 +++++++++++-- src/code-execution/runtime-py.ts | 50 ++++++++++++-- src/framework.ts | 33 +++------ test/tool-call-origin.test.ts | 115 +++++++++++++++++++++++++++++++ 4 files changed, 209 insertions(+), 36 deletions(-) diff --git a/src/code-execution/py-runner.ts b/src/code-execution/py-runner.ts index df70e4852..dae38be4c 100644 --- a/src/code-execution/py-runner.ts +++ b/src/code-execution/py-runner.ts @@ -20,6 +20,7 @@ import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { createInterface, type Interface } from 'node:readline'; import { PYTHON_RUNTIME_SOURCE } from './runtime-py.js'; +import type { ToolCall } from '../types/index.js'; export interface InjectedTool { /** Python identifier the tool is bound to (sanitized, `--` -> `__`). */ @@ -53,10 +54,16 @@ export interface BackgroundExecOptions { lifetimeMs: number; } -/** Resolves inner tool calls. Must never reject — map errors to strings. */ +/** Who started an exec (its origin and admission): the calls its own work + * makes carry it. Absent: the agent's own model. */ +export type ExecProvenance = Pick; + +/** Resolves inner tool calls, each with the provenance of the exec whose + * work made it. Must never reject — map errors to strings. */ export type ScriptToolCallHandler = ( toolName: string, args: Record, + provenance: ExecProvenance | undefined, ) => Promise; export interface PyRunnerOptions { @@ -77,6 +84,17 @@ const DEFAULT_SCRIPT_TIMEOUT_MS = 600_000; const DEFAULT_IDLE_RECLAIM_MS = 300_000; const CANCEL_GRACE_MS = 10_000; +/** What a tool call gets back when it isn't the running exec's own work + * (see handleLine): `execId` is the exec it reported, if any. */ +function refusalFor(execId: unknown): string { + return typeof execId === 'string' + ? 'Error: this tool call came from a task that an earlier code_execution script left running. ' + + 'Tool calls are served only to the running script and the tasks it started.' + : 'Error: this tool call came from work that can\'t be traced to the running code_execution script, ' + + 'such as a coroutine sent from a thread started without the script\'s context. ' + + 'Tool calls are served only to the running script and the tasks it started.'; +} + /** The longest delay Node's timers honour (~24.8 days); a longer one fires after ~1 ms. */ export const MAX_TIMER_MS = 2_147_483_647; @@ -89,6 +107,8 @@ export function formatLimit(ms: number): string { interface PendingExec { id: string; + /** Who started it: its inner calls carry this (ScriptToolCallHandler). */ + provenance: ExecProvenance | undefined; resolve: (result: ExecResult) => void; deadlineTimer: ReturnType | null; killTimer: ReturnType | null; @@ -140,12 +160,14 @@ export class PyRunner { * * `opts.deadlineMs` replaces the runner's scriptTimeoutMs for this exec (a * per-call time limit); a background exec uses its `lifetimeMs` instead. + * `opts.provenance` is who started the exec: the tool calls its own work + * makes carry it, and only while it runs (see handleLine). */ async exec( code: string, tools: InjectedTool[], background?: BackgroundExecOptions, - opts?: { deadlineMs?: number }, + opts?: { deadlineMs?: number; provenance?: ExecProvenance }, ): Promise { if (this.disposed) { return { stdout: '', stderr: 'code_execution runner disposed', returnCode: 1, aborted: true }; @@ -180,6 +202,7 @@ export class PyRunner { const result = await new Promise((resolve) => { const pending: PendingExec = { id: execId, + provenance: opts?.provenance, resolve, deadlineTimer: null, killTimer: null, @@ -334,7 +357,7 @@ export class PyRunner { private readyResolver: (() => void) | null = null; private handleLine(line: string): void { - let msg: { op?: string; id?: string; name?: string; args?: unknown; stdout?: string; stderr?: string; return_code?: number }; + let msg: { op?: string; id?: string; exec_id?: unknown; name?: string; args?: unknown; stdout?: string; stderr?: string; return_code?: number }; try { msg = JSON.parse(line); } catch { @@ -351,11 +374,19 @@ export class PyRunner { const callId = msg.id; const toolName = msg.name; if (!callId || !toolName) return; + // Only the running exec's own work is served, as whoever started that + // exec. A task an earlier exec left running reports that exec, whose + // standing (its turn, lease or admission) ended with it. + const pending = this.servingExec(msg.exec_id); + if (!pending) { + this.send({ op: 'tool_result', id: callId, result: refusalFor(msg.exec_id) }); + return; + } const args = msg.args && typeof msg.args === 'object' && !Array.isArray(msg.args) ? (msg.args as Record) : {}; - this.onToolCall(toolName, args) + this.onToolCall(toolName, args, pending.provenance) .catch((err) => `Error: ${err instanceof Error ? err.message : String(err)}`) .then((result) => { this.send({ op: 'tool_result', id: callId, result }); @@ -370,7 +401,7 @@ export class PyRunner { ? (msg as { line: number }).line : -1; const payload = (msg as { payload?: unknown }).payload; - const handler = this.onWake; + const handler = this.servingExec(msg.exec_id) ? this.onWake : null; const refuse = handler ? handler(line, payload).catch((err: unknown) => `wake handler failed: ${err instanceof Error ? err.message : String(err)}`) @@ -400,6 +431,12 @@ export class PyRunner { } } + /** The running exec, if `execId` names it: the only exec whose work is served. */ + private servingExec(execId: unknown): PendingExec | null { + const pending = this.pending; + return pending && !pending.settled && execId === pending.id ? pending : null; + } + private settlePending(result: ExecResult): void { const pending = this.pending; if (!pending || pending.settled) return; diff --git a/src/code-execution/runtime-py.ts b/src/code-execution/runtime-py.ts index f29c15f85..289745475 100644 --- a/src/code-execution/runtime-py.ts +++ b/src/code-execution/runtime-py.ts @@ -37,6 +37,12 @@ * {op:"wake", id, exec_id, line, payload} * {op:"exec_result", id, stdout, stderr, return_code, tail?} * + * A tool_call's or wake's exec_id is the exec whose work made it: bound in a + * contextvar when the exec starts, so every task and callback its script + * starts (each copies the context it was created in) reports that exec, even + * after it ends or while a later one runs. The host serves only the exec + * that is running. + * * IMPORTANT: the python source below must not contain backticks or the * sequence dollar+brace (TS template literal syntax). String.raw preserves * backslashes, so \n inside python string literals is fine. @@ -45,6 +51,7 @@ export const PYTHON_RUNTIME_SOURCE: string = String.raw` import ast import asyncio +import contextvars import inspect import io import json @@ -166,7 +173,8 @@ _pending_tool_futures = {} _pending_wake_futures = {} _next_tool_call_id = 0 _next_wake_id = 0 -_current_exec_id = None +# The exec the running work belongs to (see the protocol notes above). +_exec_id = contextvars.ContextVar("exec_id", default=None) _current_exec_task = None @@ -187,7 +195,7 @@ def _make_tool_fn(tool_name, py_name): send({ "op": "tool_call", "id": call_id, - "exec_id": _current_exec_id, + "exec_id": _exec_id.get(), "name": tool_name, "args": args, }) @@ -227,7 +235,7 @@ async def wake_agent(payload=None): send({ "op": "wake", "id": wake_id, - "exec_id": _current_exec_id, + "exec_id": _exec_id.get(), "line": line, "payload": payload, }) @@ -279,8 +287,23 @@ def handle_init(msg): SCRIPT_GLOBALS.pop("wake_agent", None) +def _started_by(task, exec_id, started): + # Whether this exec's own work started the task: read from the context + # it runs in where Python exposes it (3.12+). Older interpreters fall + # back to "created during this exec", which also counts a task that a + # task left by an earlier exec started meanwhile. + get_context = getattr(task, "get_context", None) + if get_context is not None: + return get_context().get(_exec_id) == exec_id + return task not in started + + async def _run_script(exec_id, code): - global _current_exec_id, _current_exec_task + global _current_exec_task + # This task runs in its own copy of the context: the binding reaches the + # script and whatever it starts, and nothing else. + _exec_id.set(exec_id) + started = asyncio.all_tasks() if BACKGROUND: out = LogTee(LOG_PATH, TAIL_CHARS) err = out # interleave, terminal-style; tail is shared @@ -304,7 +327,6 @@ async def _run_script(exec_id, code): return_code = 1 finally: sys.stdout, sys.stderr, sys.stdin = old_out, old_err, old_in - _current_exec_id = None _current_exec_task = None for fut in list(_pending_tool_futures.values()): if not fut.done(): @@ -315,6 +337,21 @@ async def _run_script(exec_id, code): fut.cancel() _pending_wake_futures.clear() + # Tasks this script started that are still running outlive its exec, and + # with it their standing: the host refuses their tool calls from now on. + # Say so where the script's author reads (a background interpreter is + # disposed with its exec, so its tasks end with it). + left = [ + t for t in asyncio.all_tasks() + if t is not asyncio.current_task() and not t.done() and _started_by(t, exec_id, started) + ] + if left and not BACKGROUND: + err.write( + "\n[code_execution] " + str(len(left)) + " task(s) this script started were still running " + "when it ended. Their tool calls are refused from now on: await them before the script " + "ends, or use a background script.\n" + ) + if BACKGROUND: out.flush() out.close() @@ -344,7 +381,7 @@ async def _run_script(exec_id, code): async def main(): - global _current_exec_id, _current_exec_task + global _current_exec_task loop = asyncio.get_running_loop() queue = asyncio.Queue() @@ -382,7 +419,6 @@ async def main(): "return_code": 1, }) continue - _current_exec_id = msg.get("id") _current_exec_task = asyncio.ensure_future( _run_script(msg.get("id"), msg.get("code") or "") ) diff --git a/src/framework.ts b/src/framework.ts index 9a049bd48..ee5a9f48f 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -1715,13 +1715,6 @@ export class AgentFramework { private codeExecutionConfig: import('./types/index.js').CodeExecutionConfig | null = null; private codeExecutionRunners: Map = new Map(); private scriptToolWaiters: Map void> = new Map(); - /** - * Who initiated the interactive code_execution call running for each - * agent (its origin and admission), so the script's inner tool calls carry - * them too. The runner allows one interactive script per agent at a time. - * A background script binds its own at start. - */ - private scriptCallProvenance: Map> = new Map(); /** Agents whose running script hit an endTurn-carrying inner result — * deferred and applied to the final code_execution result instead of * cancelling the stream mid-script (which would wedge the turn). */ @@ -14845,19 +14838,12 @@ export class AgentFramework { const runner = this.getOrCreateScriptRunner(agentName); this.scriptDeferredEndTurn.delete(agentName); - // A busy runner refuses this exec, so only an exec that can run sets - // the provenance its inner calls read; it's cleared when that exec ends. - const owned = !runner.busy; - if (owned) { - if (provenance) this.scriptCallProvenance.set(agentName, provenance); - else this.scriptCallProvenance.delete(agentName); - } - let exec: Awaited>; - try { - exec = await runner.exec(input.code, injected, undefined, timeLimitMs !== undefined ? { deadlineMs: timeLimitMs } : undefined); - } finally { - if (owned && this.scriptCallProvenance.get(agentName) === provenance) this.scriptCallProvenance.delete(agentName); - } + // The exec carries who started it: its own inner calls, and only those, + // are made as that actor (PyRunner serves only the running exec's work). + const exec = await runner.exec(input.code, injected, undefined, { + ...(timeLimitMs !== undefined ? { deadlineMs: timeLimitMs } : {}), + ...(provenance ? { provenance } : {}), + }); const endTurn = this.scriptDeferredEndTurn.delete(agentName); return { @@ -14941,7 +14927,7 @@ export class AgentFramework { scriptTimeoutMs: cfg?.scriptTimeoutMs, idleReclaimMs: 0, // dedicated runner; lifetime is the exec deadline label: `${agentName}:${scriptId}`, - onToolCall: (toolName, args) => this.handleScriptToolCall(agentName, toolName, args, provenance), + onToolCall: (toolName, args, execProvenance) => this.handleScriptToolCall(agentName, toolName, args, execProvenance), }); const record: BackgroundScriptRecord = { @@ -14967,7 +14953,7 @@ export class AgentFramework { logPath: logAbsPath, lifetimeMs, onWake: (line, payload) => this.handleScriptWake(record, line, payload), - }) + }, provenance ? { provenance } : undefined) .then((exec) => this.settleBackgroundScript(record, exec)) .catch((err) => { // exec never rejects by contract; this is the belt-and-suspenders. @@ -15265,8 +15251,7 @@ export class AgentFramework { scriptTimeoutMs: cfg?.scriptTimeoutMs, idleReclaimMs: cfg?.idleReclaimMs, label: agentName, - onToolCall: (toolName, args) => - this.handleScriptToolCall(agentName, toolName, args, this.scriptCallProvenance.get(agentName)), + onToolCall: (toolName, args, provenance) => this.handleScriptToolCall(agentName, toolName, args, provenance), }); this.codeExecutionRunners.set(agentName, runner); } diff --git a/test/tool-call-origin.test.ts b/test/tool-call-origin.test.ts index 45a98a69c..96c9ba127 100644 --- a/test/tool-call-origin.test.ts +++ b/test/tool-call-origin.test.ts @@ -338,4 +338,119 @@ describe("tool-call origin: the agent's own, and nothing else, skips the gate", ]); assert.equal(tokens(), 100_000); }); + + it('refuses a tool call from a task a script left running, after its exec and during a later one', async () => { + framework.start(); + const settle = (name: string) => [ + `for _ in range(500):`, + ` if "${name}" in globals(): break`, + ` await asyncio.sleep(0.01)`, + ]; + // The puppet's script leaves a task that calls after its exec has ended. + const first = await framework.puppetToolCall('scout', 'code_execution', { + code: [ + 'import asyncio', + 'async def _after():', + ' await asyncio.sleep(0.3)', + ' globals()["late1"] = await agent_settings({"action": "update", "context_budget_tokens": 134000})', + 'asyncio.ensure_future(_after())', + 'print("scheduled")', + ].join('\n'), + }); + assert.equal(first.result.success, true, String(first.result.error)); + const out = first.result.data as { stdout: string; stderr: string }; + assert.equal(out.stdout.trim(), 'scheduled'); + assert.match(out.stderr, /1 task\(s\) this script started were still running when it ended/); + + // The next one reads what that task got back, and leaves a task that + // calls only once the agent's own script is running. + const second = await framework.puppetToolCall('scout', 'code_execution', { + code: [ + 'import asyncio', + ...settle('late1'), + 'print(late1)', + 'async def _during():', + ' while not globals().get("agent_running"):', + ' await asyncio.sleep(0.01)', + ' globals()["late2"] = await agent_settings({"action": "update", "context_budget_tokens": 135000})', + 'asyncio.ensure_future(_during())', + ].join('\n'), + }); + assert.match((second.result.data as { stdout: string }).stdout, /earlier code_execution script left running/); + + // The agent's own script waits for that call, then makes its own. + await modelCalls('code_execution', { + code: [ + 'import asyncio', + 'agent_running = True', + ...settle('late2'), + 'print(await agent_settings({"action": "update", "context_budget_tokens": 126000}))', + ].join('\n'), + }); + assert.equal(tokens(), 126_000, "the agent's own script applies its own change"); + assert.deepEqual(asked, [], 'neither leftover call reached the gate: both were refused'); + assert.match(await readGlobal(framework, 'late2'), /earlier code_execution script left running/, + "the puppet's task, firing during the agent's own exec, was refused"); + }); + + it("refuses a call from a thread the script started, which can't be traced to it, while serving the script's own", async () => { + framework.start(); + const { result } = await framework.puppetToolCall('scout', 'code_execution', { + code: [ + 'import asyncio, threading', + 'loop = asyncio.get_running_loop()', + 'box = {}', + 'def worker():', + ' box["thread"] = asyncio.run_coroutine_threadsafe(', + ' agent_settings({"action": "update", "context_budget_tokens": 140000}), loop).result(timeout=10)', + 't = threading.Thread(target=worker)', + 't.start()', + 'while t.is_alive():', + ' await asyncio.sleep(0.02)', + 'print(box["thread"])', + 'print(await agent_settings({"action": "update", "context_budget_tokens": 141000}))', + ].join('\n'), + }); + assert.equal(result.success, true, String(result.error)); + assert.match((result.data as { stdout: string }).stdout, /can't be traced to the running code_execution script/); + assert.deepEqual(surfaces(), ['scout:puppet'], "only the script's own call reached the gate, as the puppet's"); + assert.equal(tokens(), 100_000); + }); + + it('says nothing of tasks that finished with the script, or of a thread it joined', async () => { + framework.start(); + const { result } = await framework.puppetToolCall('scout', 'code_execution', { + code: [ + 'import asyncio, threading', + 'await asyncio.gather(*(asyncio.sleep(0.01) for _ in range(3)))', + 't = threading.Thread(target=lambda: None)', + 't.start()', + 't.join()', + 'print("done")', + ].join('\n'), + }); + assert.equal(result.success, true, String(result.error)); + assert.deepEqual(result.data && { stdout: (result.data as { stdout: string }).stdout.trim(), stderr: (result.data as { stderr: string }).stderr }, { stdout: 'done', stderr: '' }); + }); + + it("keeps a background script's calls its starter's for its whole life", async () => { + framework.start(); + const started = await framework.puppetToolCall('scout', 'code_execution', { + code: ['import asyncio', 'await asyncio.sleep(0.2)', 'await agent_settings({"action": "update", "context_budget_tokens": 142000})'].join('\n'), + background: true, + }); + assert.equal(started.result.success, true, String(started.result.error)); + const deadline = Date.now() + 10_000; + while (asked.length === 0 && Date.now() < deadline) await new Promise((r) => setTimeout(r, 50)); + assert.deepEqual(surfaces(), ['scout:puppet']); + assert.equal(tokens(), 100_000); + }); }); + +/** A script global, read by a host script. */ +async function readGlobal(framework: AgentFramework, name: string): Promise { + const read = await framework.executeToolCall({ + id: `read-${name}`, name: 'code_execution', input: { code: `print(${name})` }, callerAgentName: 'scout', origin: 'host', + }); + return (read.data as { stdout?: string } | undefined)?.stdout ?? String(read.error); +} From f8a818c7534d5e8c46f3fd5b482f788577777039 Mon Sep 17 00:00:00 2001 From: Alfred-1883 Date: Thu, 8 Oct 2026 16:51:27 -0400 Subject: [PATCH 67/73] fix(api): while an operator-change gate is configured, the API server refuses the body changes it can't stage, whole and saying why (#255 review P5) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit slimepriestess's review of #255 (pullrequestreview-5462207876), finding P5. With a gate configured, ApiServer's WebSocket `undo` still called framework.undoLastTurn directly. Her probe got `{ undone: true, toBranch: "undo/scout/1-…" }`, the last reply was cut, and the gate was never asked. The config doc scoped the gate to host/command, so this was outside its stated scope. But it is an operator's undo, and the PR's limits named only the host's checkout, redo and restore. `redo` and `branch.switch` were ungated too, and Hilda-1882 confirmed that `branch.switch` to an earlier branch changes the body. shelf-380, the host contract this producer serves, says framework-owned operator paths take part in the imposed-change contract. This PR has no public staging entry and no branch-move change kind. The host companion that builds on it adds both (`stageOperatorChange`, and branch moves). So in this PR the API server refuses what it can't stage. Agnes-1009, who owns the companion, agreed (room-222 #57958). When the companion rebases onto this, it turns these routes into staging: `undo` as undo-turns, `redo` and `branch.switch` as moves to an existing branch, and `branch.create` with `switchTo` as a move that creates its destination. **While a gate is configured, `undo`, `redo`, `branch.switch` and `branch.create` with `switchTo` are refused whole, before anything changes:** - The error says why: " refused: an operator-change gate is configured, and this API can't stage the change; use the host's command for it". - `branch.create` with `switchTo` is refused before the branch is created. Without `switchTo` it still creates the branch, since that changes no body. - AgentFramework gains a read-only `hasOperatorChangeGate()`, which the server asks. - The MCP server wraps these WebSocket commands, so its `branch_switch` and `branch_create` tools follow. Without a gate, every command behaves as before. The host's own direct calls (undoLastTurn, redo, a store branch switch) remain the host's responsibility, as the PR's stated limit already says: the gate never sees them. The operatorChangeGate config doc now says both. Tests (operator-change-gate, "the API server"; a real server on an ephemeral port, over WebSocket): - **With a gate.** `undo`, `branch.create` with `switchTo` and `branch.switch` are each refused with the reason. The body, the active branch and the branch list are unchanged, and no `side` branch exists. After the host's own undoLastTurn (allowed), the API's `redo` of it is refused and the body stays where the host left it. The gate is never asked. `branch.create` without `switchTo` succeeds. - **Without a gate.** `undo` cuts, `redo` restores, and `branch.create` with `switchTo` creates and switches. Mutants, checked on this batch's final tree: - Not refusing under a gate fails the gated test. - Refusing `branch.create`'s switch only after creating the branch fails it, on the branch list. Co-Authored-By: Alfred (Claude Opus 5.5) --- src/api/server.ts | 20 ++++++ src/framework.ts | 8 +++ src/types/framework.ts | 6 ++ test/operator-change-gate.test.ts | 109 +++++++++++++++++++++++++++++- 4 files changed, 142 insertions(+), 1 deletion(-) diff --git a/src/api/server.ts b/src/api/server.ts index be71277cc..ec809497a 100644 --- a/src/api/server.ts +++ b/src/api/server.ts @@ -426,6 +426,21 @@ export class ApiServer { } } + /** + * With an operator-change gate configured (FrameworkConfig.operatorChangeGate), + * a command that would change an agent's body here (undo, redo, a branch + * switch) is refused whole, before anything changes: this API can't stage + * the change, and an imposed change applies only through the gate. Without + * a gate, these commands work as they always have. + */ + private refuseUnderGate(command: string): void { + if (!this.framework.hasOperatorChangeGate()) return; + throw new Error( + `${command} refused: an operator-change gate is configured, and this API can't stage the change; ` + + "use the host's command for it", + ); + } + /** WS-side admin gate for the mutating host verbs: no-op unless an * adminToken is configured, then `params.adminToken` must match. */ private requireAdminToken(params?: Record): void { @@ -512,6 +527,8 @@ export class ApiServer { } private async cmdBranchCreate(params: BranchCreateParams): Promise<{ name: string }> { + // Refused whole: never a created branch without the switch. + if (params.switchTo) this.refuseUnderGate('branch.create with switchTo'); if (!params.name) { throw new Error('name is required'); } @@ -532,6 +549,7 @@ export class ApiServer { } private async cmdBranchSwitch(params: BranchSwitchParams): Promise<{ switched: boolean }> { + this.refuseUnderGate('branch.switch'); if (!params.name) { throw new Error('name is required'); } @@ -588,6 +606,7 @@ export class ApiServer { // ========================================================================== private async cmdUndo(params: { agentName: string }): Promise { + this.refuseUnderGate('undo'); if (!params.agentName) { throw new Error('agentName is required'); } @@ -607,6 +626,7 @@ export class ApiServer { } private async cmdRedo(params: { agentName: string }): Promise { + this.refuseUnderGate('redo'); if (!params.agentName) { throw new Error('agentName is required'); } diff --git a/src/framework.ts b/src/framework.ts index ee5a9f48f..6886703b9 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -7107,6 +7107,14 @@ export class AgentFramework { // Operator-change admission (FrameworkConfig.operatorChangeGate) // --------------------------------------------------------------------------- + /** Whether the host configured an operator-change gate: then an imposed + * body change applies only through it, and an operator surface that can't + * stage a change (the API server's undo, redo and branch switches) + * refuses it. */ + hasOperatorChangeGate(): boolean { + return !!this.operatorChangeGate; + } + /** Hand one resolved change to the host's gate, which stages it and * answers with its receipt. A gate that throws, or answers without a * receipt, refuses the change (fails closed). Both are logged. */ diff --git a/src/types/framework.ts b/src/types/framework.ts index d99c53929..f8f93e92a 100644 --- a/src/types/framework.ts +++ b/src/types/framework.ts @@ -202,6 +202,12 @@ export interface FrameworkConfig { * through applyResolvedOperatorChange under a held safe-boundary lease. A * gate that throws refuses the change. The agent's own model calls never * reach it. Absent: every operator path behaves as it always has. + * + * The API server can't stage a change, so while a gate is configured its + * body-changing commands (undo, redo, branch.switch, and branch.create with + * switchTo) are refused whole, before anything changes. The host's own + * direct calls (undoLastTurn, redo, a store branch switch) stay the host's + * responsibility: the gate never sees them. */ operatorChangeGate?: ( change: import('../operator-change.js').ResolvedOperatorChange, diff --git a/test/operator-change-gate.test.ts b/test/operator-change-gate.test.ts index af42aa5c5..fe74a4b7a 100644 --- a/test/operator-change-gate.test.ts +++ b/test/operator-change-gate.test.ts @@ -18,7 +18,8 @@ import { spawnSync } from 'node:child_process'; import { fileURLToPath } from 'node:url'; import { JsStore } from '@animalabs/chronicle'; import { join } from 'node:path'; -import { AgentFramework, AutobiographicalStrategy, ResumeBlockedError, WorkspaceModule } from '../src/index.js'; +import WebSocket from 'ws'; +import { AgentFramework, ApiServer, AutobiographicalStrategy, ResumeBlockedError, WorkspaceModule } from '../src/index.js'; import type { AgentSettingsExtension, InferenceRequest, @@ -2537,3 +2538,109 @@ describe('a hard kill between a body change and its outcome record', () => { } }); }); + +describe('operator-change gate: the API server', () => { + type Host = { addMessage(participant: string, content: unknown[]): string; pendingRequests: InferenceRequest[] }; + type Response = { success: boolean; data?: unknown; error?: string }; + + /** A framework, with or without a gate, behind an API server on an ephemeral port. */ + async function withApi(gated: boolean, fn: (h: { + framework: AgentFramework; + send: (command: string, params: Record) => Promise; + texts: () => string[]; + asked: ResolvedOperatorChange[]; + }) => Promise): Promise { + const dir = mkdtempSync(join(tmpdir(), 'operator-gate-api-')); + const membrane = new MockMembrane(); + const asked: ResolvedOperatorChange[] = []; + const framework = await AgentFramework.create({ + storePath: join(dir, 'store'), + membrane: membrane.asMembrane(), + agents: [{ name: 'scout', model: 'test-model', systemPrompt: 'test', maxTokens: 1000 }], + modules: [], + ...(gated ? { operatorChangeGate: async (change: ResolvedOperatorChange) => { asked.push(change); return { id: 'rev', text: 'staged' }; } } : {}), + }); + const quiet = { log: console.log, error: console.error }; + console.log = () => {}; + console.error = () => {}; + const server = new ApiServer(framework, { port: 0, host: '127.0.0.1' }); + await server.start(); + const port = (server as unknown as { httpServer: { address(): { port: number } } }).httpServer.address().port; + const ws = new WebSocket(`ws://127.0.0.1:${port}/ws`); + await new Promise((resolve, reject) => { ws.once('open', () => resolve()); ws.once('error', reject); }); + let n = 0; + const send = (command: string, params: Record) => new Promise((resolve) => { + const id = `c${++n}`; + const onMessage = (raw: WebSocket.RawData) => { + const msg = JSON.parse(raw.toString()) as Response & { type: string; id: string }; + if (msg.type === 'response' && msg.id === id) { ws.off('message', onMessage); resolve(msg); } + }; + ws.on('message', onMessage); + ws.send(JSON.stringify({ type: 'request', id, command, params })); + }); + const texts = () => (framework.getAgent('scout')!.getContextManager().getAllMessages() as Array<{ content: Array<{ text?: string }> }>) + .map((m) => m.content[0]?.text ?? ''); + try { + const host = framework as unknown as Host; + for (const t of ['one', 'two']) { + host.addMessage('user', [{ type: 'text', text: t }]); + membrane.pushResponse(createMockResponse([{ type: 'text', text: `reply to ${t}` }])); + host.pendingRequests.push({ agentName: 'scout', reason: 'mcpl:channel-incoming', source: 'test', timestamp: Date.now() } as InferenceRequest); + await framework.runUntilIdle(); + } + await fn({ framework, send, texts, asked }); + } finally { + ws.close(); + console.log = quiet.log; + console.error = quiet.error; + await server.stop(); + await framework.stop(); + rmSync(dir, { recursive: true, force: true }); + } + } + + it("refuses undo, redo and branch switches whole while a gate is configured, saying why; the host's own calls stay its own", async () => { + await withApi(true, async ({ framework, send, texts, asked }) => { + const store = framework.getStore(); + const snapshot = () => ({ texts: texts(), branch: store.currentBranch().name, branches: store.listBranches().map((b) => b.name).sort() }); + const before = snapshot(); + const refusals = [ + await send('undo', { agentName: 'scout' }), + await send('branch.create', { name: 'side', switchTo: true }), + await send('branch.switch', { name: before.branch }), + ]; + for (const r of refusals) { + assert.equal(r.success, false); + assert.match(String(r.error), /refused: an operator-change gate is configured, and this API can't stage the change; use the host's command for it/); + } + assert.deepEqual(snapshot(), before, 'nothing changed, and branch.create made no branch'); + + // The host's own undo is the host's responsibility; the API's redo of it is refused too. + const undone = framework.undoLastTurn('scout'); + assert.equal(undone.undone, true); + const afterHostUndo = snapshot(); + const redo = await send('redo', { agentName: 'scout' }); + assert.equal(redo.success, false); + assert.match(String(redo.error), /^redo refused: an operator-change gate is configured/); + assert.deepEqual(snapshot(), afterHostUndo); + assert.equal(asked.length, 0, 'the API never stages'); + + const created = await send('branch.create', { name: 'side' }); + assert.equal(created.success, true, 'creating a branch without switching changes no body'); + }); + }); + + it('keeps undo, redo and branch switches without a gate', async () => { + await withApi(false, async ({ framework, send, texts }) => { + const undo = await send('undo', { agentName: 'scout' }); + assert.equal(undo.success, true, String(undo.error)); + assert.deepEqual(texts(), ['one', 'reply to one', 'two']); + const redo = await send('redo', { agentName: 'scout' }); + assert.equal(redo.success, true, String(redo.error)); + assert.deepEqual(texts(), ['one', 'reply to one', 'two', 'reply to two']); + const created = await send('branch.create', { name: 'side', switchTo: true }); + assert.equal(created.success, true, String(created.error)); + assert.equal(framework.getStore().currentBranch().name, 'side'); + }); + }); +}); From 97ae95a81022d3a67964818da97cb7de2fd3ff7e Mon Sep 17 00:00:00 2001 From: Alfred-1883 Date: Thu, 8 Oct 2026 16:52:24 -0400 Subject: [PATCH 68/73] feat(operator-changes): undo-turns says what its cut removed, beside what followed the cut when it was staged (#255 review P7) slimepriestess's review of #255 (pullrequestreview-5462207876), P7, a design question rather than a defect. A staged 1-turn undo applied after three new human turns arrived removed all three as well. ResolvedUndoTurnsChange documents that the cut can reach later arrivals, so this is as specified. But the receipt said `{ requested: 1, undone: 1 }` while the body lost seven messages, and other people's later messages left it unmarked. Unstick, by contrast, never chooses anything that arrives later. She asked whether a staged undo should have the same guarantee, or at least a receipt that counts the later removals. **The absolute cut stays.** shelf-380, the host contract this producer serves, says revalidation compares the affected settings and stable branch identity, "not an append cursor that the notice/ready exchange itself advances", and that the staged destination stays fixed unless the change is restaged. The resident's notice and ready answer land on the source branch after staging, so refusing a cut that would remove later arrivals would refuse every asked-for undo, and would starve one in any busy channel. Carrying later arrivals onto the destination would rewrite other participants' messages onto a branch the host delivers its arrival notice to. Telling the resident what the cut removed is that arrival notice's job (the host companion's), and it now has the numbers for it. **Receipt parity with undo-messages,** which already reports `messagesRemoved` and stages `messagesAfter`: - ResolvedUndoTurnsChange gains `messagesAfter`: how many messages followed the cut point when the change was resolved (staged), counted the same way the cut counts what it removes. - The applied undo-turns receipt gains `messagesRemoved`, from the established outcome, which is already journaled. So a retry, and a recovered completion, report the same count. - Both kinds' docs now say what the difference means: how many arrived after staging, less any of those removed in place meanwhile, by a hide or an unstick step for instance. Agnes-1009 asked for that point in time to be stated (room-222 #57958). Her answer was that the two counts are enough from AF, since the applied witness keeps the source branch, where a host can name the later arrivals itself. The awareness marks are unchanged: they cover only the refs frozen at staging, and an addressable later removal is counted in `markers.unmarked`. Tests (operator-change-gate, "host/command undo by turns"): - Two turns, then a staged 1-turn undo: its change says `messagesAfter: 1`, two's reply. After three more turns it applies: `undone: 1`, `messagesRemoved: 7`, its reply and the six later messages. The body is `one, reply to one, two`, and a retry reports `alreadyApplied` with the same 7. Mutant, checked on this batch's final tree: reporting the staging count as `messagesRemoved` fails that test. Co-Authored-By: Alfred (Claude Opus 5.5) --- src/framework.ts | 14 +++++++------- src/operator-change.ts | 18 ++++++++++++++++-- test/operator-change-gate.test.ts | 17 +++++++++++++++++ 3 files changed, 40 insertions(+), 9 deletions(-) diff --git a/src/framework.ts b/src/framework.ts index 6886703b9..4c58cccac 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -8097,14 +8097,11 @@ export class AgentFramework { const checkpoints = this.getTurnCheckpoints(agentName); if (checkpoints.length === 0) return null; const taken = checkpoints.slice(-requested).reverse(); - // Freeze the publication choice to the refs the cut would remove now: - // every message stored after the oldest staged turn began. + // What the cut would remove now: every message stored after the oldest + // staged turn began. The publication choice is frozen to its refs. const cutAt = taken[taken.length - 1]!.sequenceBefore; - const marks = this.freezeMarks( - this.agents.get(agentName)!.getContextManager().getAllMessages().filter((m) => Number(m.sequence) > cutAt), - publication?.marks ?? 'none', - publication?.serverId ?? '', - ); + const after = this.agents.get(agentName)!.getContextManager().getAllMessages().filter((m) => Number(m.sequence) > cutAt); + const marks = this.freezeMarks(after, publication?.marks ?? 'none', publication?.serverId ?? ''); return { id: randomUUID(), agent: agentName, @@ -8116,6 +8113,7 @@ export class AgentFramework { kind: 'undo-turns', requestedTurns: requested, checkpoints: taken.map((c) => ({ turnIndex: c.turnIndex, sequenceBefore: c.sequenceBefore, branchName: c.branchName })), + messagesAfter: after.length, marks, ...(publication?.serverId ? { serverId: publication.serverId } : {}), }; @@ -8550,6 +8548,7 @@ export class AgentFramework { private async applyUndoTurns(change: ResolvedUndoTurnsChange, admission?: OperatorAdmission): Promise<{ requested: number; undone: number; + messagesRemoved: number; fromBranch: string; toBranch: string; alreadyApplied?: true; @@ -8612,6 +8611,7 @@ export class AgentFramework { return { requested: change.requestedTurns, undone: change.checkpoints.length, + messagesRemoved: outcome.removed, fromBranch: change.sourceBranch, toBranch: target, ...(alreadyApplied ? { alreadyApplied: true as const } : {}), diff --git a/src/operator-change.ts b/src/operator-change.ts index 85c300dfb..1f9c16a8b 100644 --- a/src/operator-change.ts +++ b/src/operator-change.ts @@ -83,6 +83,12 @@ export interface ResolvedUndoTurnsChange extends ResolvedOperatorChangeBase { /** The checkpoints it undoes, newest first. Application makes one cut at * the oldest one's sequenceBefore, onto a branch named for this change. */ checkpoints: Array<{ turnIndex: number; sequenceBefore: number; branchName: string }>; + /** How many messages followed the cut point when the change was resolved + * (staged). The cut also removes what arrives later, so the applied + * receipt's `messagesRemoved` minus this is how many arrived after it was + * staged, less any of these removed in place meanwhile (by a hide or an + * unstick step, for instance). */ + messagesAfter: number; /** The operator's awareness-marks choice, frozen at staging to the refs * the cut would have removed then. The cut can also reach messages that * arrive later; application marks only the frozen refs it actually @@ -145,7 +151,11 @@ export interface ResolvedUndoMessagesChange extends ResolvedOperatorChangeBase { /** The message that becomes the tail (the end of its body group), with its * content fingerprint. */ tail: { id: string; fingerprint: string }; - /** How many messages followed it at staging. */ + /** How many messages followed the tail when the change was resolved + * (staged). The cut also removes what arrives later, so the applied + * receipt's `messagesRemoved` minus this is how many arrived after it was + * staged, less any of these removed in place meanwhile (by a hide or an + * unstick step, for instance). */ messagesAfter: number; /** Frozen to the refs among the messages that followed it at staging. */ marks: FrozenMarks; @@ -276,7 +286,8 @@ export type AppliedOperatorChange = | { kind: 'undo-messages'; requested: number; - /** Messages the cut removed, later arrivals included. */ + /** Messages the cut removed, later arrivals included (compare the + * change's `messagesAfter`). */ messagesRemoved: number; fromBranch: string; toBranch: string; @@ -287,6 +298,9 @@ export type AppliedOperatorChange = kind: 'undo-turns'; requested: number; undone: number; + /** Messages the cut removed, later arrivals included (compare the + * change's `messagesAfter`). */ + messagesRemoved: number; fromBranch: string; toBranch: string; /** The cut had already been applied (the active branch is its diff --git a/test/operator-change-gate.test.ts b/test/operator-change-gate.test.ts index fe74a4b7a..bfec499f6 100644 --- a/test/operator-change-gate.test.ts +++ b/test/operator-change-gate.test.ts @@ -1744,6 +1744,23 @@ describe('operator-change gate: host/command undo by turns', () => { assert.deepEqual(applied.kind === 'undo-turns' && [applied.undone, applied.requested], [1, 3]); }); + it('says what the cut removed, later arrivals included, beside what followed its cut at staging', async () => { + await turn('one'); await turn('two'); + await undo(1); + const change = asked[0]!; + assert.equal(change.kind === 'undo-turns' && change.messagesAfter, 1, "at staging the cut would remove two's reply"); + await turn('alice: new 1'); await turn('bob: new 2'); await turn('carol: new 3'); + const apply = () => framework.runAtSafeBoundary({ verb: 'apply' }, (lease) => + framework.applyResolvedOperatorChange(change, { lease, admission: { id: 'rev-u' } })); + const applied = await apply(); + assert.deepEqual(applied.kind === 'undo-turns' && [applied.undone, applied.messagesRemoved], [1, 7], + 'one turn undone; the cut removed its reply and the six messages that arrived after staging'); + assert.deepEqual(texts(), ['one', 'reply to one', 'two']); + const again = await apply(); + assert.deepEqual(again.kind === 'undo-turns' && [again.alreadyApplied, again.messagesRemoved], [true, 7], + 'a retry reports the established count'); + }); + it('refuses when the gate fails, and asks nothing when there is nothing to undo', async () => { const empty = await undo(1); assert.deepEqual([empty.ok, empty.undone], [true, 0]); From 551c36332cf741b812b009f3308e9e96027a63f0 Mon Sep 17 00:00:00 2001 From: Agnes-1009 Date: Fri, 9 Oct 2026 13:18:21 -0400 Subject: [PATCH 69/73] docs(changelog): fragments for #255's own changes, including the script-author note slimepriestess asked for MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit slimepriestess approved #255 at 97ae95a (re-review, 2026-10-09) and asked that script authors learn from the changelog, not from an error, that a task a script leaves running past its exec now has its tool calls refused. Writing that showed #255 had no fragments for its own changes at all. Its diff passed the `changelog` check only because it still carries #250's fragments, which would stop being true once #250 lands and #255 is rebased. The fragments, written from #255's commits (485a3c7..97ae95a) and the code at 97ae95a: - **`255-operator-change-gate.added.md`:** - the gate (`FrameworkConfig.operatorChangeGate`) for the six kinds it covers here (`ResolvedOperatorChange`'s: undo by turns or by messages, hide, unstick, and an operator's or module's agent_settings or tool-presentation call); - cut and hide attempts journaled in `operator/changes`, an unstick in `operator/unstick`, and their recovery (`getOperatorChangeRecord`, `resolveOperatorChange`, `restoreOperatorChangeSource`, `agent-framework-recover`); - the planned unstick; undo-turns' `messagesRemoved` on the applied receipt beside the change's `messagesAfter`; preview-bound surgery; - staged Discord awareness marks, activated once the change's body change is established; - `runAtSafeBoundary`, tool-call origin, `ModuleContext.addMessage` delivery options, and `Module.getSubconsciousTools()`. - **`255-operator-change-gate.breaking.md`,** each bullet naming who must act: - API server and MCP clients (P5, including the MCP server's `branch_switch`/`branch_create`); - script authors (P4/P4b: leftover tasks refused, a foreground script's stderr counting them, thread calls refused as untraceable); - module and host authors (P6, P2, P3, P1: origin normalization, including events the host's own `pushEvent` queues); - hosts (`resume()` refusing on an unresolved or abandoned cut's destination). - **`255-stop-admission.fixed.md`:** `stop()` closes admission before its first await. **Reviewed by Ada-1017** (room-447 #63827) against Alfred-1883's six commits and 97ae95a's source. Her five corrections are applied: the stderr note's count, foreground only; the MCP server's wrapped tools; events the host queues; the journal's scope; and where each undo-turns count lives. The staged-marks bullet describes her own 1ae93fc, so there she checked accuracy rather than reviewing independently, and her wording is used. I confirmed the MCP tool mapping (src/api/mcp-server.ts: branch_create → branch.create, branch_switch → branch.switch) and the note's text (test/tool-call-origin.test.ts) myself. **Checked:** `scripts/release-changelog.mjs` folds a scratch copy (12 fragments). Co-Authored-By: Agnes (Claude Opus 5.5) --- changelog.d/255-operator-change-gate.added.md | 10 ++++++++++ changelog.d/255-operator-change-gate.breaking.md | 9 +++++++++ changelog.d/255-stop-admission.fixed.md | 1 + 3 files changed, 20 insertions(+) create mode 100644 changelog.d/255-operator-change-gate.added.md create mode 100644 changelog.d/255-operator-change-gate.breaking.md create mode 100644 changelog.d/255-stop-admission.fixed.md diff --git a/changelog.d/255-operator-change-gate.added.md b/changelog.d/255-operator-change-gate.added.md new file mode 100644 index 000000000..f2a84419a --- /dev/null +++ b/changelog.d/255-operator-change-gate.added.md @@ -0,0 +1,10 @@ +- **An operator-change gate for imposed body changes** (#255). With `FrameworkConfig.operatorChangeGate`, the framework's own operator mutations are resolved to an absolute change and handed to the host's gate, which stages it and answers with a receipt: accepted and queued, not applied. They are a `host/command` undo (by turns or by messages), hide or unstick, and an operator's or module's mutating `agent_settings` or tool-presentation call. The host applies a staged change only through `applyResolvedOperatorChange`, under a held safe-boundary lease. Without a gate, every operator path behaves as before. + - Each cut and hide attempt is journaled in the store (`operator/changes`), and recovery settles it from that record; an unstick keeps its own journal (`operator/unstick`). `getOperatorChangeRecord` reads a change's record, `resolveOperatorChange` settles an attempt whose outcome can't be told (or `agent-framework-recover`, offline), and `restoreOperatorChangeSource` restores the source of a cut that didn't commit. + - An unstick is a planned, journaled operation: it sheds the newest exchanges a step at a time until a re-run responds. + - An applied undo-turns receipt reports what its cut removed (`messagesRemoved`), and its resolved change records how many messages followed the cut when it was staged (`messagesAfter`), as undo-messages already did. + - `previewSurgeryMarks` returns the store and branch it previewed (`context: {storeId, branch}`), and `getStoreIdentity()` names the store, so a host can bind a staged surgery to where it was previewed. + - A staged change's Discord awareness marks can be chosen at staging (`prepare({ id, staged: true, … })`), and are activated by the change once its body change is established. +- **`runAtSafeBoundary(opts, fn)`** holds a store reservation across a host callback, with no turn running, for work that must not interleave with one (#255). +- **A tool call carries who initiated it** through every redispatch: the agent itself, a puppet, the host or an operator. So a gate can tell an imposed change from the agent's own (#255). +- **`ModuleContext.addMessage` delivery options** (#255): a targeted, receipted delivery, optionally made durable before its receipt says so. +- **`Module.getSubconsciousTools()`** offers a module's tools on the subconscious's surface (#255). diff --git a/changelog.d/255-operator-change-gate.breaking.md b/changelog.d/255-operator-change-gate.breaking.md new file mode 100644 index 000000000..8c54d9b77 --- /dev/null +++ b/changelog.d/255-operator-change-gate.breaking.md @@ -0,0 +1,9 @@ +- **API server and MCP clients:** while an operator-change gate is configured, the API server refuses `undo`, `redo`, `branch.switch` and `branch.create` with `switchTo` whole, saying why, since it can't stage them (#255). The MCP server's `branch_switch` and `branch_create` tools, which wrap those commands, are refused the same way. Use the host's gated command instead. Without a gate, nothing changes. +- **Script authors (`code_execution`):** a script's tool call is served only while the exec that made it is running, as whoever started that exec (#255). + - A task a script leaves running past its exec has its later tool calls refused: its standing ended with its exec. They used to run, unrecorded, as the agent's own. A foreground script's result notes in its stderr how many of the tasks it started were still running when it ended. + - A call sent from a thread the script started (as with `run_coroutine_threadsafe`) is refused as untraceable. +- **Module and host authors:** a caller-named origin is normalized (#255). Anything other than absent or `'puppet'` acts for the host, never as the agent's own. Likewise: + - a tool-call event that anything other than the framework's own dispatch queues (a module or the host) acts for the host, unless it names the operator; + - trace listeners are notified as the host's; + - a delegation to another agent acts for the host. +- **Hosts:** `resume()` refuses, force or not, while the active body is the destination of an unresolved or abandoned cut, until it's resolved or its source restored (#255). diff --git a/changelog.d/255-stop-admission.fixed.md b/changelog.d/255-stop-admission.fixed.md new file mode 100644 index 000000000..6633f20cd --- /dev/null +++ b/changelog.d/255-stop-admission.fixed.md @@ -0,0 +1 @@ +- `stop()` closes admission before its first await, and lets the store's current users finish before tearing anything down (#255). From 3934785027381d54f312f0f75e3ded78acfd510e Mon Sep 17 00:00:00 2001 From: Agnes-1009 Date: Sun, 11 Oct 2026 15:24:43 -0400 Subject: [PATCH 70/73] docs(operator-changes): say what an unnamed call is, what the gate covers, and what each note claims A review of this change since its approval found texts that said more than the code does, or something else. This commit is that text, plus the one stderr wording a note depends on. The code and test fixes follow in their own commits. **Origin:** - executeToolCall's doc said it "always carries agent origin" and that unnamed callers "fail safe to agent semantics". It now says what holds. The entry is shared with model callers, so it never confers module trust, and an unnamed call is the agent's own because its callers here are models: SubagentModule's ephemerals, code_execution scripts, and puppetToolCall, which names 'puppet'. So a host or an operator must name its origin for a gate to see it. - The registry closure's comment says the same. - normalizeOrigin's doc no longer says "a falsy one must never read as the agent's own" beside a null that does. Absent or null is the agent's own, 'puppet' the operator's, and any other value, a falsy one included, the host's. - The code is unchanged, as intended: the approved scope is honest attribution for well-behaved scripts, not a tamper-proof boundary. **What the gate covers:** hasOperatorChangeGate's doc said an imposed change "applies only through it". It now names what's staged and what refuses, and says that the framework's public body-change methods (undoLastTurn, redo, rollbackToMessage, suppressMessages) are the host's own and aren't gated. **Attempts:** CUT_KINDS's comment said "the cuts and hide" over a set of the cuts alone. It now says why: only a cut's attempt can be unresolved and settled by an operator. A hide is journaled as attempts too, but its effect can be read back, so recovery re-runs it, removing only staged messages still present and unchanged. The added note says the same. **The notes:** - added: without a gate, every path behaves as before except that resume() refuses while a cut a gated run left unresolved or abandoned holds the body; messagesAfter is new for both undos, not "as undo-messages already did"; the origins are the agent, the operator through a puppet, or the host, not a separate "operator". - breaking: MCP branch_create is refused only with switchTo; undo can be staged through the host's gated undo, while redo and branch switching have no gated route and are refused; an agent's own delegation to another agent acts for the host, while a puppet's or the host's keeps its origin; the stderr count is exact from Python 3.12. **Docs:** - getAllUtilities and ModuleContext get their own doc comments back, which had drifted above getAllSubconsciousTools and ModuleMessageOptions. - The two "never a recount" docs name the older-build exception that recordedQueued already counts. - The reset preview's doc says a key with nothing configured returns to its default (the budget, the think-text policy) or keeps its value (tailTokens), as the code does. - activateStaged's doc says an unknown or prepared batch is the caller's error and throws. **The stderr note:** before Python 3.12 the count is of tasks created while the script ran, so the note says that there, and "this script started" from 3.12. tool-call-origin's test accepts either wording, whichever the interpreter reaches. **The check, from a clean worktree of this commit as AF's CI runs it (npm install; node 24.19.0, npm 11.17.0):** `tsc --noEmit` and the build are clean, and `npm test` passes 1535/1535 across 140 suites. That's the same count as at 551c363. An earlier run of this same commit reported 1529 with none failing: six tests in two files (channel-registry-routing, mcpl-tool-lifecycle) were never reported. The rerun reported all 1535. Co-Authored-By: Agnes (Claude Opus 5.5) --- changelog.d/255-operator-change-gate.added.md | 8 ++-- .../255-operator-change-gate.breaking.md | 6 +-- src/agent.ts | 7 ++-- src/call-provenance.ts | 7 ++-- src/code-execution/runtime-py.ts | 4 +- src/framework.ts | 38 +++++++++++-------- src/module-registry.ts | 12 +++--- src/operator-change.ts | 5 ++- src/recovery/discord-awareness-outbox.ts | 11 ++++-- src/types/module.ts | 6 +-- test/tool-call-origin.test.ts | 3 +- 11 files changed, 63 insertions(+), 44 deletions(-) diff --git a/changelog.d/255-operator-change-gate.added.md b/changelog.d/255-operator-change-gate.added.md index f2a84419a..bdfe0d1e8 100644 --- a/changelog.d/255-operator-change-gate.added.md +++ b/changelog.d/255-operator-change-gate.added.md @@ -1,10 +1,10 @@ -- **An operator-change gate for imposed body changes** (#255). With `FrameworkConfig.operatorChangeGate`, the framework's own operator mutations are resolved to an absolute change and handed to the host's gate, which stages it and answers with a receipt: accepted and queued, not applied. They are a `host/command` undo (by turns or by messages), hide or unstick, and an operator's or module's mutating `agent_settings` or tool-presentation call. The host applies a staged change only through `applyResolvedOperatorChange`, under a held safe-boundary lease. Without a gate, every operator path behaves as before. - - Each cut and hide attempt is journaled in the store (`operator/changes`), and recovery settles it from that record; an unstick keeps its own journal (`operator/unstick`). `getOperatorChangeRecord` reads a change's record, `resolveOperatorChange` settles an attempt whose outcome can't be told (or `agent-framework-recover`, offline), and `restoreOperatorChangeSource` restores the source of a cut that didn't commit. +- **An operator-change gate for imposed body changes** (#255). With `FrameworkConfig.operatorChangeGate`, the framework's own operator mutations are resolved to an absolute change and handed to the host's gate, which stages it and answers with a receipt: accepted and queued, not applied. They are a `host/command` undo (by turns or by messages), hide or unstick, and an operator's or module's mutating `agent_settings` or tool-presentation call. The host applies a staged change only through `applyResolvedOperatorChange`, under a held safe-boundary lease. Without a gate, every operator path behaves as before, except that `resume()` refuses while a cut a gated run left unresolved or abandoned holds the active body (see Breaking). + - Each cut and hide attempt is journaled in the store (`operator/changes`) before it acts, and recovery settles it from that record: a cut whose outcome can't be told waits for an operator's verdict, and a hide, whose effect can be read back, is re-run. An unstick keeps its own journal (`operator/unstick`). `getOperatorChangeRecord` reads a change's record, `resolveOperatorChange` settles an attempt whose outcome can't be told (or `agent-framework-recover`, offline), and `restoreOperatorChangeSource` restores the source of a cut that didn't commit. - An unstick is a planned, journaled operation: it sheds the newest exchanges a step at a time until a re-run responds. - - An applied undo-turns receipt reports what its cut removed (`messagesRemoved`), and its resolved change records how many messages followed the cut when it was staged (`messagesAfter`), as undo-messages already did. + - An applied undo-turns receipt reports what its cut removed (`messagesRemoved`), and a resolved undo, by turns or by messages, records how many messages followed the cut when it was staged (`messagesAfter`). - `previewSurgeryMarks` returns the store and branch it previewed (`context: {storeId, branch}`), and `getStoreIdentity()` names the store, so a host can bind a staged surgery to where it was previewed. - A staged change's Discord awareness marks can be chosen at staging (`prepare({ id, staged: true, … })`), and are activated by the change once its body change is established. - **`runAtSafeBoundary(opts, fn)`** holds a store reservation across a host callback, with no turn running, for work that must not interleave with one (#255). -- **A tool call carries who initiated it** through every redispatch: the agent itself, a puppet, the host or an operator. So a gate can tell an imposed change from the agent's own (#255). +- **A tool call carries who initiated it** through every redispatch: the agent itself, the operator (through a puppet call), or the host. So a gate can tell an imposed change from the agent's own (#255). - **`ModuleContext.addMessage` delivery options** (#255): a targeted, receipted delivery, optionally made durable before its receipt says so. - **`Module.getSubconsciousTools()`** offers a module's tools on the subconscious's surface (#255). diff --git a/changelog.d/255-operator-change-gate.breaking.md b/changelog.d/255-operator-change-gate.breaking.md index 8c54d9b77..5375d2392 100644 --- a/changelog.d/255-operator-change-gate.breaking.md +++ b/changelog.d/255-operator-change-gate.breaking.md @@ -1,9 +1,9 @@ -- **API server and MCP clients:** while an operator-change gate is configured, the API server refuses `undo`, `redo`, `branch.switch` and `branch.create` with `switchTo` whole, saying why, since it can't stage them (#255). The MCP server's `branch_switch` and `branch_create` tools, which wrap those commands, are refused the same way. Use the host's gated command instead. Without a gate, nothing changes. +- **API server and MCP clients:** while an operator-change gate is configured, the API server refuses `undo`, `redo`, `branch.switch` and `branch.create` with `switchTo` whole, saying why, since it can't stage them (#255). The MCP server's `branch_switch`, and its `branch_create` with `switchTo`, are refused the same way; a `branch_create` without `switchTo` still works. An undo can be staged instead, through the host's gated `host/command` undo; redo and branch switching have no gated route, so under a gate they're refused. Without a gate, nothing changes. - **Script authors (`code_execution`):** a script's tool call is served only while the exec that made it is running, as whoever started that exec (#255). - - A task a script leaves running past its exec has its later tool calls refused: its standing ended with its exec. They used to run, unrecorded, as the agent's own. A foreground script's result notes in its stderr how many of the tasks it started were still running when it ended. + - A task a script leaves running past its exec has its later tool calls refused: its standing ended with its exec. They used to run, unrecorded, as the agent's own. A foreground script's result notes in its stderr how many of the tasks it started were still running when it ended (on Python before 3.12, the tasks created while it ran, which can include ones a leftover task from an earlier exec started). - A call sent from a thread the script started (as with `run_coroutine_threadsafe`) is refused as untraceable. - **Module and host authors:** a caller-named origin is normalized (#255). Anything other than absent or `'puppet'` acts for the host, never as the agent's own. Likewise: - a tool-call event that anything other than the framework's own dispatch queues (a module or the host) acts for the host, unless it names the operator; - trace listeners are notified as the host's; - - a delegation to another agent acts for the host. + - an agent's own call delegating to another agent acts for the host; a puppet's or the host's call keeps its origin through a delegation. - **Hosts:** `resume()` refuses, force or not, while the active body is the destination of an unresolved or abandoned cut, until it's resolved or its source restored (#255). diff --git a/src/agent.ts b/src/agent.ts index a6c57515d..3d14d1db8 100644 --- a/src/agent.ts +++ b/src/agent.ts @@ -513,9 +513,10 @@ export class Agent { * * - update: the patch's values (a decrease reports its target at once). * - reset: the configured value each key returns to. A key with nothing - * configured keeps its current value; transitionPaceTokens with nothing - * configured reports unset (null) unless a descent it starts needs the - * default pace. + * configured returns to its default (contextBudgetTokens and + * sameRoundThinkTextPolicy) or keeps its current value (tailTokens); + * transitionPaceTokens with nothing configured reports unset (null) + * unless a descent it starts needs the default pace. * - cancel: a converging budget stops at the live budget; nothing else * changes. * diff --git a/src/call-provenance.ts b/src/call-provenance.ts index 673c1fe74..c136c6cff 100644 --- a/src/call-provenance.ts +++ b/src/call-provenance.ts @@ -29,9 +29,10 @@ export const HOST_PROVENANCE: CallProvenance = Object.freeze({ origin: 'host' }) /** * A caller-supplied origin as the framework reads it: absent (undefined or - * null) stays absent, 'puppet' stays the operator's, and any other value is - * the host's. Types forbid other values, but an untyped module or host can - * pass anything, and a falsy one must never read as the agent's own. + * null) stays absent, which is the agent's own, 'puppet' stays the + * operator's, and any other value is the host's. Types forbid other values, + * but an untyped module or host can pass anything, and any other value, a + * falsy one such as '' or 0 included, is never the agent's own. */ export function normalizeOrigin(origin: unknown): ToolCall['origin'] { if (origin === undefined || origin === null) return undefined; diff --git a/src/code-execution/runtime-py.ts b/src/code-execution/runtime-py.ts index 289745475..d1bc7110f 100644 --- a/src/code-execution/runtime-py.ts +++ b/src/code-execution/runtime-py.ts @@ -346,8 +346,10 @@ async def _run_script(exec_id, code): if t is not asyncio.current_task() and not t.done() and _started_by(t, exec_id, started) ] if left and not BACKGROUND: + # Before 3.12 the count is of tasks created while it ran (_started_by). + counted = "this script started" if hasattr(asyncio.Task, "get_context") else "created while this script ran" err.write( - "\n[code_execution] " + str(len(left)) + " task(s) this script started were still running " + "\n[code_execution] " + str(len(left)) + " task(s) " + counted + " were still running " "when it ended. Their tool calls are refused from now on: await them before the script " "ends, or use a background script.\n" ) diff --git a/src/framework.ts b/src/framework.ts index 4c58cccac..3053f4bfb 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -1904,8 +1904,9 @@ export class AgentFramework { onTrace: (listener) => this.onTrace(listener), // The ONLY entry that confers module origin — a private closure the // framework hands to its own module registry. The public - // executeToolCall() is shared with model/ephemeral callers and always - // stamps agent origin (see there). + // executeToolCall() is shared with model/ephemeral callers: it never + // confers module trust, and an origin it isn't given is the agent's + // own (see there). // Who asked travels with a delegated call: inside a module tool // handler (or work it started) the initiator of that call, the // agent's own only for that same agent; with no call behind it (a @@ -7107,10 +7108,14 @@ export class AgentFramework { // Operator-change admission (FrameworkConfig.operatorChangeGate) // --------------------------------------------------------------------------- - /** Whether the host configured an operator-change gate: then an imposed - * body change applies only through it, and an operator surface that can't - * stage a change (the API server's undo, redo and branch switches) - * refuses it. */ + /** Whether the host configured an operator-change gate. Then the + * framework's own operator mutations (a host/command undo, hide or + * unstick, and a named operator's or module's self-change) are staged + * through it, and an operator surface that can't stage a change (the API + * server's undo, redo and branch switches) refuses it. The framework's + * public body-change methods (undoLastTurn, redo, rollbackToMessage, + * suppressMessages) are the host's own and aren't gated: a host routes + * its own imposed changes through the gate. */ hasOperatorChangeGate(): boolean { return !!this.operatorChangeGate; } @@ -14343,17 +14348,18 @@ export class AgentFramework { /** * Execute a tool call and return the result. * Routes to the appropriate handler (module registry or MCPL). - * Used by SubagentModule to dispatch tool calls for ephemeral agents. + * Used by SubagentModule to dispatch tool calls for ephemeral agents, by + * code_execution scripts, and by puppetToolCall. * - * This public entry always carries AGENT origin. Ephemeral/promise-based - * callers are models, and provenance must identify which internal path a - * call entered through — a public method shared with model callers can - * never confer module trust, whatever the input claims. Module origin - * exists only through the private closure handed to ModuleRegistry - * (ctx.callTool → executeToolCallFrom with {kind:'module'}). Callers that - * don't identify themselves fail safe to agent semantics too. An origin the - * caller names is normalized (normalizeOrigin): a value other than - * 'puppet' or 'host' is the host's, never the agent's own. + * This public entry is shared with model callers, so it never confers + * module trust, whatever the input claims: module origin exists only + * through the private closure handed to ModuleRegistry (ctx.callTool → + * executeToolCallFrom with {kind:'module'}). A call that names no origin is + * taken as the agent's own, since its callers here are models. A host or an + * operator must name its origin for an operator-change gate to see it: an + * unnamed self-change runs as the agent's own, ungated. An origin the + * caller names is normalized (normalizeOrigin): 'puppet' is the + * operator's, and any other value is the host's, never the agent's own. */ async executeToolCall(call: ToolCall): Promise { const { origin: named, ...rest } = call; diff --git a/src/module-registry.ts b/src/module-registry.ts index 51129cc0e..a8f4c6b30 100644 --- a/src/module-registry.ts +++ b/src/module-registry.ts @@ -243,12 +243,6 @@ export class ModuleRegistry { return tools; } - /** - * All utilities from all modules, namespaced exactly like tools - * (`module--name`). See Module.getUtilities — these are deliberately kept - * out of the model-facing tool list and surfaced through the framework's - * `utils` meta-tool; dispatch goes through the same handleToolCall. - */ /** Every module's getSubconsciousTools(), prefixed `module--name`. */ getAllSubconsciousTools(): ToolDefinition[] { const tools: ToolDefinition[] = []; @@ -260,6 +254,12 @@ export class ModuleRegistry { return tools; } + /** + * All utilities from all modules, namespaced exactly like tools + * (`module--name`). See Module.getUtilities — these are deliberately kept + * out of the model-facing tool list and surfaced through the framework's + * `utils` meta-tool; dispatch goes through the same handleToolCall. + */ getAllUtilities(): ToolDefinition[] { const utilities: ToolDefinition[] = []; for (const module of this.modules.values()) { diff --git a/src/operator-change.ts b/src/operator-change.ts index 1f9c16a8b..ab55e1ac6 100644 --- a/src/operator-change.ts +++ b/src/operator-change.ts @@ -440,7 +440,10 @@ export interface OperatorChangeRestorationReceipt { } const CHANGE_KINDS = new Set(['agent-settings', 'tool-presentation', 'undo-turns', 'unstick', 'hide', 'undo-messages']); -/** The kinds whose application is journaled as attempts: the cuts and hide. */ +/** The cuts: the kinds whose attempt can be unresolved and settled by an + * operator. A hide is journaled as attempts too, but its effect can be read + * back (its staged messages are present or not), so recovery re-runs it, + * removing only what is still present and unchanged. */ const CUT_KINDS = new Set(['undo-turns', 'undo-messages']); const isObject = (v: unknown): v is Record => typeof v === 'object' && v !== null && !Array.isArray(v); const isText = (v: unknown): v is string => typeof v === 'string' && v.length > 0; diff --git a/src/recovery/discord-awareness-outbox.ts b/src/recovery/discord-awareness-outbox.ts index 5bfd873cd..ed9f58aeb 100644 --- a/src/recovery/discord-awareness-outbox.ts +++ b/src/recovery/discord-awareness-outbox.ts @@ -194,7 +194,9 @@ export interface DiscordAwarenessBatch extends DiscordAwarenessBatchRecord { * requests the release queues. */ releaseAuthorization?: number; /** What the activation recorded: when, and how many adds it requested (the - * original scheduling fact a retry replays, never a recount). */ + * original scheduling fact a retry replays, never a recount). An older + * build's activation, recorded without its count, is counted from its + * requests still standing (recordedQueued). */ activation?: { at: number; queued: number }; held?: { reason: string; at: number; releaseActions: DiscordAwarenessReleaseAction[] }; cancelled?: { at: number; by?: string }; @@ -744,9 +746,12 @@ export class DiscordAwarenessOutbox { * synced, holds the change's facts (refs narrowed to the staged choice's; * the publication audience only shrinks), the activation and the adds it * requested, under the staging-time authorization: a key retracted since - * staging is born superseded. Never throws for the batch's own state: + * staging is born superseded. Never throws for a staged batch's own state + * (an unknown id, or a batch prepared rather than staged, is the caller's + * error, and throws): * - `queued`: activated now, or already (a retry), with the facts and - * count its activation recorded, never a recount; + * count its activation recorded, never a recount (an older build's + * activation without its count is counted from its standing requests); * - `not-scheduled`: cancelled while staged, or discarded; nothing written; * - `unresolved`: the record could not be written; the batch stays staged * and a later call activates it. diff --git a/src/types/module.ts b/src/types/module.ts index f859fdaf0..af48c7cc6 100644 --- a/src/types/module.ts +++ b/src/types/module.ts @@ -172,9 +172,6 @@ export interface AgentSettingsExtension { ): Record; } -/** - * Context provided to modules for interacting with the framework. - */ /** Delivery options for ModuleContext.addMessage. */ export interface ModuleMessageOptions { /** @@ -216,6 +213,9 @@ export interface MessagePlacement { durable?: boolean; } +/** + * Context provided to modules for interacting with the framework. + */ export interface ModuleContext { /** * Get persistent state for this module. diff --git a/test/tool-call-origin.test.ts b/test/tool-call-origin.test.ts index 96c9ba127..8918ecf2c 100644 --- a/test/tool-call-origin.test.ts +++ b/test/tool-call-origin.test.ts @@ -360,7 +360,8 @@ describe("tool-call origin: the agent's own, and nothing else, skips the gate", assert.equal(first.result.success, true, String(first.result.error)); const out = first.result.data as { stdout: string; stderr: string }; assert.equal(out.stdout.trim(), 'scheduled'); - assert.match(out.stderr, /1 task\(s\) this script started were still running when it ended/); + // Python 3.12+ reads who started a task; older interpreters count the tasks created while it ran. + assert.match(out.stderr, /1 task\(s\) (this script started|created while this script ran) were still running when it ended/); // The next one reads what that task got back, and leaves a task that // calls only once the agent's own script is running. From 01152c78d4c2045672b1ce5415b151a3af303cc3 Mon Sep 17 00:00:00 2001 From: Agnes-1009 Date: Sun, 11 Oct 2026 15:55:59 -0400 Subject: [PATCH 71/73] fix(operator-changes): crash bookkeeping, liveness, admission and readers The same review's code findings, each fix with a test. The previous commit is its text. **Crash bookkeeping** - Startup recovery skipped any record that had an outcome. So a crash after the outcome but before its operator-log line and completed mark left both to a host retry. Startup now finishes them, marked recovered at startup. - An attested commit recovered at startup was logged like a switch the framework saw. Its log line now carries the attestation (verdict, via, requester), and the console line says it was attested. - An attempt record whose write threw may still have landed, leaving an attempt that read as unresolved after a restart. When it's there, its failure is recorded too, so the restart reads it as not committed. - The drop is journaled before the staged choice is discarded, and it stands if the discard fails. A batch left behind is inert, because a dropped change never applies. - A staged choice whose record may have landed though staging couldn't tell is retired. - Resolve checks that the agent is registered before it journals a verdict that would restore it. A refusal now leaves the attempt unresolved, as it was. **Resolve's receipts** - A committed attestation that the active body contradicts stands, as the operator's verdict. Its receipt, live and offline, now says the active branch isn't the attempt's destination. - A repeated resolve's refusal names the settlement on record (settledState): its verdict, who gave it, how, and when. **Liveness** - A lease waiting behind a direct surgery is granted when the surgery releases. The release used to wake only parked creations, which return early while a lease waits. - A release whose deferred-write flush throws still grants the next waiter and wakes parked creations, and the callback's own result stands. The flush failure is logged, and its writes stay deferred. **Admission** - While a reservation holds the store, a tool_result is deferred like any other write. Its exemption is for a draining turn, and none runs then. - A module's editMessage and removeMessage under a reservation wait for its release (flushDeferredMessageOps), so a body change's evidence can't move underneath it. - A conversation fork enters the store under the creation admission. A waiting lease isn't granted while the fork opens. stop() refuses a fork once it has begun and waits for one already opening. A fork parks only behind a held reservation, never behind a lease that is only waiting: the event loop awaits the fork, and grants that lease. - Branch.delete refuses a branch that an unsettled operator change's attempt names as its source or destination (operatorChangeReadingBranch). Recovery reads those by name. **Readers** - The Discord awareness outbox refuses a checkpoint with no snapshot instead of reading the entries it covers as gone. - The operator/changes reader refuses a drop after attempts and an attempt after a drop. - It refuses a second record of an attempt's number in replay, as it already did in a snapshot. - **Also:** a resolution entry for a hide is refused in replay, as in a snapshot. - The store identity comes from the first identity record that is usable. An unusable oldest record used to mint and append a new identity at every start. **Others** - No external id is mapped to a deferred write's empty id. - The patch check and the reset preview carry updateRuntimeSettings' "cannot prepare a smaller window live" refusal, so the gate never stages what the apply refuses. The reset test now raises the budget, since its strategy has no hot settings. - A replay returns what the first call recorded: not-scheduled for a retired batch, and the recorded count for an active one. - Placement.durable is true only when the caller asked, as its contract says. - The offline operator-change verb opens only an existing store. - A wake that isn't the running exec's own work is refused for that reason, as tool calls are. - A staged choice that can't be activated because this process has no outbox is reported as not scheduled, with an ops alert. - A resolve refused on an unreadable journal is logged, and the usage text lists resolve among the recorded acts. - IsDrainingStream's dead clause is gone. - **Notes:** the breaking note says that branch.delete refuses a branch an unsettled change reads. The stop() note says what stop() waits for and what it cancels. **Tests** - operator-change-gate: - bookkeeping finished at the next start; - a contradicted attestation and a repeated resolve; - the unknown-agent refusal left unresolved; - a landed attempt whose write threw; - the startup log entry's attestation; - a repeated offline resolve; - a drop whose discard fails, and a begun change that can't be dropped; - a staged choice retired after an ambiguous write; - a descent refused at staging; - branch.delete; - the store identity; - and twelve reader refusals, guard by guard. - discord-awareness-outbox: the null-snapshot checkpoint, and the replays. - safe-boundary-lease: a lease behind a surgery, a throwing release flush, a deferred tool_result, and held edits and removals. - module-targeted-delivery: durable as asked, and the external id. - conversation-routing: a fork behind a held lease, and one refused at stop(). - recover-cli-operator-log: a missing store, and a refused resolve logged. - The stale wake's wording and the missing-outbox alert have no test of their own. **The check, from a clean worktree of this commit as AF's CI runs it (npm install; node 24.19.0, npm 11.17.0):** - **The suite:** `tsc --noEmit` and the build are clean. `npm test` reported 1552 of 1552 passing, across 140 suites, with all 22 new tests among them. Five tests went unreported, with none failing: those at the end of channel-registry-routing.test.ts, a file this commit doesn't touch. The same happened in a run of the text-only commit before it. Run alone at this commit three times, that file passes 24/24 each time. - **Red first:** this commit's six test files, built and run on the previous commit's sources, fail 25 tests. They are all 22 new ones, plus three extended ones: the offline committed verdict's attestation, the retired batch's replay, and the repeated offline resolve. Co-Authored-By: Agnes (Claude Opus 5.5) --- .../255-operator-change-gate.breaking.md | 2 +- changelog.d/255-stop-admission.fixed.md | 2 +- src/agent.ts | 9 + src/api/server.ts | 5 + src/code-execution/py-runner.ts | 18 +- src/framework.ts | 289 +++++++++++++++--- src/module-registry.ts | 8 +- src/operator-change.ts | 29 ++ src/recovery/discord-awareness-outbox.ts | 14 +- src/recovery/recover-cli.ts | 71 +++-- test/conversation-routing.test.ts | 49 +++ test/discord-awareness-outbox.test.ts | 18 ++ test/module-targeted-delivery.test.ts | 18 ++ test/operator-change-gate.test.ts | 226 +++++++++++++- test/recover-cli-operator-log.test.ts | 36 +++ test/safe-boundary-lease.test.ts | 60 ++++ 16 files changed, 764 insertions(+), 90 deletions(-) diff --git a/changelog.d/255-operator-change-gate.breaking.md b/changelog.d/255-operator-change-gate.breaking.md index 5375d2392..091da3997 100644 --- a/changelog.d/255-operator-change-gate.breaking.md +++ b/changelog.d/255-operator-change-gate.breaking.md @@ -1,4 +1,4 @@ -- **API server and MCP clients:** while an operator-change gate is configured, the API server refuses `undo`, `redo`, `branch.switch` and `branch.create` with `switchTo` whole, saying why, since it can't stage them (#255). The MCP server's `branch_switch`, and its `branch_create` with `switchTo`, are refused the same way; a `branch_create` without `switchTo` still works. An undo can be staged instead, through the host's gated `host/command` undo; redo and branch switching have no gated route, so under a gate they're refused. Without a gate, nothing changes. +- **API server and MCP clients:** while an operator-change gate is configured, the API server refuses `undo`, `redo`, `branch.switch` and `branch.create` with `switchTo` whole, saying why, since it can't stage them (#255). The MCP server's `branch_switch`, and its `branch_create` with `switchTo`, are refused the same way; a `branch_create` without `switchTo` still works. An undo can be staged instead, through the host's gated `host/command` undo; redo and branch switching have no gated route, so under a gate they're refused. Without a gate, nothing changes. Gate or not, `branch.delete` refuses a branch that an operator change not yet settled names as its attempt's source or destination, since its recovery reads that branch. - **Script authors (`code_execution`):** a script's tool call is served only while the exec that made it is running, as whoever started that exec (#255). - A task a script leaves running past its exec has its later tool calls refused: its standing ended with its exec. They used to run, unrecorded, as the agent's own. A foreground script's result notes in its stderr how many of the tasks it started were still running when it ended (on Python before 3.12, the tasks created while it ran, which can include ones a leftover task from an earlier exec started). - A call sent from a thread the script started (as with `run_coroutine_threadsafe`) is refused as untraceable. diff --git a/changelog.d/255-stop-admission.fixed.md b/changelog.d/255-stop-admission.fixed.md index 6633f20cd..1394e97f2 100644 --- a/changelog.d/255-stop-admission.fixed.md +++ b/changelog.d/255-stop-admission.fixed.md @@ -1 +1 @@ -- `stop()` closes admission before its first await, and lets the store's current users finish before tearing anything down (#255). +- `stop()` closes admission before its first await: no lease, direct surgery, ephemeral agent or conversation fork enters the store once it has begun. Before tearing anything down it waits for the holder of a store reservation and for agents and conversation forks still being created; live agent streams are cancelled, as before (#255). diff --git a/src/agent.ts b/src/agent.ts index 3d14d1db8..c7ac8f7ce 100644 --- a/src/agent.ts +++ b/src/agent.ts @@ -557,6 +557,8 @@ export class Agent { if (keys.has('contextBudgetTokens')) { const configured = this.configuredContextBudgetTokens ?? DEFAULT_CONTEXT_BUDGET_TOKENS; startsDescent = configured < live; + // As resetRuntimeSettings refuses it: a descent needs hot settings. + if (startsDescent && !hot) throw new Error('The active context strategy cannot prepare a smaller window live'); target.contextBudgetTokens = configured; } if (keys.has('tailTokens') && hot) { @@ -605,6 +607,13 @@ export class Agent { !this.getHotContextSettings()) { throw new Error('The active context strategy does not support live tail/transition settings'); } + // A paced decrease prepares a smaller window, which only a strategy with + // hot settings can (updateRuntimeSettings), so a preview refuses it too. + if (patch.contextBudgetTokens !== undefined && !patch.immediate && + patch.contextBudgetTokens < (this.contextBudgetTokens ?? DEFAULT_CONTEXT_BUDGET_TOKENS) && + !this.getHotContextSettings()) { + throw new Error('The active context strategy cannot prepare a smaller window live'); + } } private settleRuntimeSettingsTransition(): void { diff --git a/src/api/server.ts b/src/api/server.ts index ec809497a..f6e7c17ce 100644 --- a/src/api/server.ts +++ b/src/api/server.ts @@ -593,6 +593,11 @@ export class ApiServer { throw new Error('Cannot delete main branch'); } + const reading = this.framework.operatorChangeReadingBranch(params.name); + if (reading) { + throw new Error(`Cannot delete ${params.name}: operator change ${reading} isn't settled yet, and its recovery reads that branch`); + } + const store = this.framework.getStore(); store.deleteBranch(params.name); diff --git a/src/code-execution/py-runner.ts b/src/code-execution/py-runner.ts index dae38be4c..887b93c3f 100644 --- a/src/code-execution/py-runner.ts +++ b/src/code-execution/py-runner.ts @@ -95,6 +95,16 @@ function refusalFor(execId: unknown): string { 'Tool calls are served only to the running script and the tasks it started.'; } +/** A wake's refusal when it isn't the running exec's own work, as + * refusalFor words a tool call's. */ +function wakeRefusalFor(execId: unknown): string { + return typeof execId === 'string' + ? 'this wake came from a task that an earlier code_execution script left running; ' + + 'wakes are served only to the running script and the tasks it started' + : 'this wake came from work that can\'t be traced to the running code_execution script; ' + + 'wakes are served only to the running script and the tasks it started'; +} + /** The longest delay Node's timers honour (~24.8 days); a longer one fires after ~1 ms. */ export const MAX_TIMER_MS = 2_147_483_647; @@ -401,11 +411,15 @@ export class PyRunner { ? (msg as { line: number }).line : -1; const payload = (msg as { payload?: unknown }).payload; - const handler = this.servingExec(msg.exec_id) ? this.onWake : null; + // A wake that isn't the running exec's own work is refused for that + // reason, as its tool calls are (refusalFor), before any rule about + // who may wake. + const serving = this.servingExec(msg.exec_id); + const handler = serving ? this.onWake : null; const refuse = handler ? handler(line, payload).catch((err: unknown) => `wake handler failed: ${err instanceof Error ? err.message : String(err)}`) - : Promise.resolve('this script is not allowed to wake the agent'); + : Promise.resolve(serving ? 'this script is not allowed to wake the agent' : wakeRefusalFor(msg.exec_id)); void refuse.then((error) => { this.send({ op: 'wake_ack', id: wakeId, ...(error ? { error } : {}) }); }); diff --git a/src/framework.ts b/src/framework.ts index 3053f4bfb..c690ad973 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -38,6 +38,7 @@ import { type RuntimeSettingsValues, type UnstickAttemptOutcome, type UnstickOperationRecord, + settledState, } from './operator-change.js'; import { RecordJournal } from './record-journal.js'; import { TuneOutCoordinator, TUNE_OUT_DEFAULTS } from './tune-out/coordinator.js'; @@ -1814,6 +1815,9 @@ export class AgentFramework { /** Admissions applyResolvedOperatorChange is carrying out, by admission id: * each lets exactly its change's tool call through the gate. */ private activeAdmissions: Map = new Map(); + /** Module edits and removals held while a reservation holds the store, + * applied at its release (flushDeferredMessageOps). */ + private deferredMessageOps: Array<{ op: 'edit'; id: MessageId; content: ContentBlock[] } | { op: 'remove'; id: MessageId }> = []; /** This store's identity (storeIdentity), once read or minted. */ private storeIdentityValue: string | null = null; /** What this process saw of each cut attempt (change id -> attempt -> @@ -6886,8 +6890,12 @@ export class AgentFramework { released = true; if (this.surgeryHold === hold) this.surgeryHold = null; markReleased(); - // Creations held behind this hold may proceed (unless a lease waits). - queueMicrotask(() => this.notifyBoundaryCleared()); + // A lease waiting behind this hold is granted now, and creations held + // behind it may proceed (unless a lease still waits). + queueMicrotask(() => { + this.tryGrantSafeBoundary(); + this.notifyBoundaryCleared(); + }); for (const [name, token] of reserved) { if (this.activeTurnTokens.get(name) === token) this.activeTurnTokens.delete(name); } @@ -6914,10 +6922,28 @@ export class AgentFramework { } this.ackDeferredWrites(); } + // Edits and removals a module made meanwhile land now, in order, on + // what the holder left. + this.flushDeferredMessageOps(); }; return { tokens: reserved, release }; } + /** Apply the module edits and removals held while a reservation held the + * store (editMessage, removeMessage). One that no longer applies, such as + * an edit of a message the holder's change removed, is logged. */ + private flushDeferredMessageOps(): void { + // Optional chaining: prototype-built harnesses leave fields undefined. + for (const op of this.deferredMessageOps?.splice(0) ?? []) { + try { + if (op.op === 'edit') this.editMessage(op.id, op.content); + else this.removeMessage(op.id); + } catch (error) { + console.error(`[operator] a module's ${op.op} of message ${op.id}, held during a reservation, failed at its release: ${error instanceof Error ? error.message : String(error)}`); + } + } + } + /** stop()'s first step for the store: no lease, direct surgery or * ephemeral creation is admitted from now on, and every lease request or * creation still waiting is refused, so nothing enters the store once @@ -7003,7 +7029,13 @@ export class AgentFramework { try { return await fn(held.lease); } finally { - held.release(); + try { + held.release(); + } catch (error) { + // The callback's own outcome stands. Writes the flush didn't land + // stay deferred, and land at their agent's next boundary. + console.error(`[safe-boundary] ${held.lease.verb}: its deferred-write flush failed at release:`, error instanceof Error ? error.message : error); + } console.log(`[safe-boundary] ${held.lease.verb}: released after ${Date.now() - held.lease.since}ms`); } } @@ -7033,11 +7065,15 @@ export class AgentFramework { lease, release: () => { if (this.heldLease === entry) this.heldLease = null; - release(); - // The next waiter may be grantable at once; if none is waiting, - // creations held behind the boundary proceed. - this.tryGrantSafeBoundary(); - this.notifyBoundaryCleared(); + try { + release(); + } finally { + // The next waiter may be grantable at once; if none is waiting, + // creations held behind the boundary proceed. Even when the + // release's deferred-write flush threw: its tokens were freed first. + this.tryGrantSafeBoundary(); + this.notifyBoundaryCleared(); + } }, }); } @@ -7063,6 +7099,26 @@ export class AgentFramework { return ticket; } + /** + * Admission for createConversationAgent, which the event loop awaits + * (channel traffic arrives as events). It parks only while a reservation + * holds the store. A lease that is only waiting is granted from that same + * loop, so a fork that waited for it would wait forever; instead its + * pending ticket makes the lease wait for the fork, which finishes on its + * own. + */ + private async admitConversationCreation(): Promise { + for (;;) { + if (this.stopping) throw new Error('Conversation agent creation refused: the framework is stopping'); + if (!this.surgeryHold) { + const ticket = {}; + this.ephemeralPending.add(ticket); + return ticket; + } + await new Promise((resolve) => this.boundaryClearedWaiters.push(resolve)); + } + } + /** Admit a creation now, returning its pending ticket, or null when it has * to wait. The store hold (surgeryHold) covers a held lease and a direct * rollback or suppress alike: nothing initializes under either. */ @@ -7079,11 +7135,10 @@ export class AgentFramework { } /** A stream a waiting lease is draining: a registered ephemeral run, or an - * agent with a real turn alive (not a lease's reservation token). */ + * agent with a turn alive. Asked only while nothing holds the store (no + * lease or surgery reservation), so any turn token is a real turn's. */ private isDrainingStream(name: string): boolean { - if (this.ephemeralRuns.has(name)) return true; - const token = this.activeTurnTokens.get(name); - return token !== undefined && this.heldLease?.tokens.get(name) !== token; + return this.ephemeralRuns.has(name) || this.activeTurnTokens.has(name); } /** Release creations held by admitEphemeralCreation once no lease is @@ -7405,10 +7460,18 @@ export class AgentFramework { const ids = this.store.getRecordIdsByType('framework/store-identity') .map((id) => ({ id, value: BigInt(id) })) .sort((a, b) => (a.value < b.value ? -1 : a.value > b.value ? 1 : 0)); - if (ids.length > 0) { - const record = this.store.getRecord(ids[0]!.id); - const parsed = record ? JSON.parse(Buffer.from(record.payload).toString('utf-8')) as { id?: unknown } : null; - if (typeof parsed?.id === 'string') return (this.storeIdentityValue = parsed.id); + // The oldest record that names an identity. One that can't be read is + // passed over, so the identity minted below is the one every later start + // finds, rather than a new one minted at each. + for (const { id } of ids) { + const record = this.store.getRecord(id); + let parsed: { id?: unknown } | null = null; + try { + parsed = record ? JSON.parse(Buffer.from(record.payload).toString('utf-8')) as { id?: unknown } : null; + } catch { + parsed = null; + } + if (typeof parsed?.id === 'string' && parsed.id.length > 0) return (this.storeIdentityValue = parsed.id); } const minted = randomUUID(); this.store.appendJson('framework/store-identity', { id: minted, at: Date.now() }); @@ -7561,6 +7624,18 @@ export class AgentFramework { } } + /** The operator change whose recovery still reads `branch`, if any: one + * with no established outcome, not dropped, whose attempt names it as its + * source or destination. Deleting such a branch, or reusing its name, + * would change what that recovery decides. */ + operatorChangeReadingBranch(branch: string): string | null { + for (const record of this.changesLedger().records.values()) { + if (record.outcome || record.dropped) continue; + if (record.attempts.some((a) => a.evidence.source === branch || a.evidence.target === branch)) return record.changeId; + } + return null; + } + /** The durable body record of a gated operator change, or null if it never * began to apply and wasn't dropped. A host reconciles its revision with * this after a restart. */ @@ -7776,19 +7851,30 @@ export class AgentFramework { if (!opts.reason?.trim()) throw new OperatorActionError('invalid', 'Settling an operator change needs a reason'); const record = this.changesLedger().records.get(changeId); const unresolved = record ? this.unresolvedAttempt(record) : null; - if (!record || !unresolved) throw new OperatorActionError('invalid', `Operator change ${changeId} has no unresolved attempt to settle`); + if (!record || !unresolved) { + throw new OperatorActionError('invalid', `Operator change ${changeId} has no unresolved attempt to settle: ${settledState(record)}`); + } if (unresolved.n !== attempt) { throw new OperatorActionError('stale', `Operator change ${changeId}: attempt ${attempt} is not its unresolved attempt (${unresolved.n} is)`); } + const target = record.attempts.find((a) => a.n === attempt)!.evidence; + const active = this.store.currentBranch().name; + // Checked before the verdict is journaled: a refusal leaves the attempt + // unresolved, so it can be settled again. + if (verdict === 'not-committed' && active === target.target && !this.agents.has(record.agent)) { + throw new OperatorActionError('unknown-agent', `Unknown agent: ${record.agent}`); + } const resolution: OperatorChangeResolution = { verdict, reason: opts.reason, ...(opts.requester ? { requester: opts.requester } : {}), at: Date.now(), via: 'live', }; this.journalChange({ kind: 'resolved', changeId, n: attempt, resolution }); - const target = record.attempts.find((a) => a.n === attempt)!.evidence; let restored: string | undefined; - let remaining: string | undefined; - if (verdict === 'not-committed' && this.store.currentBranch().name === target.target) { - if (!this.agents.has(record.agent)) throw new OperatorActionError('unknown-agent', `Unknown agent: ${record.agent}`); + // The attestation stands as the operator's, but one the active body + // contradicts is never silent. + let remaining: string | undefined = verdict === 'committed' && active !== target.target + ? `the active branch is ${active}, not this attempt's destination ${target.target}; its outcome will be established as applied all the same` + : undefined; + if (verdict === 'not-committed' && active === target.target) { const result = await this.restoreCutSource({ changeId, agent: record.agent, attempt, target: target.target, source: target.source }); // The verdict stands either way. An incomplete restoration is what // remains, and holds traffic until one completes. @@ -7826,8 +7912,18 @@ export class AgentFramework { if (record?.outcome || record?.switched !== undefined || (record?.attempts.length ?? 0) > 0) { throw new OperatorActionError('invalid', `Operator change ${change.id} has begun to apply; it can't be dropped`); } - if (this.hasStagedMarks(change)) this.discordAwarenessOutbox!.discard(this.changeBatchId(change.id)); + // Dropped first: once that's durable the change never applies, so its + // staged choice can go. A batch a failure leaves staged behind it is + // inert, so the drop stands either way. this.journalChange({ kind: 'dropped', changeId: change.id, changeKind: change.kind, agent: change.agent, at: Date.now() }); + if (this.hasStagedMarks(change)) { + try { + this.discordAwarenessOutbox!.discard(this.changeBatchId(change.id)); + } catch (error) { + console.error(`[operator-change] dropped ${change.kind} ${change.id}, but its staged marks could not be discarded (nothing will activate them):`, + error instanceof Error ? error.message : error); + } + } } /** The durable record of an unstick operation, or null if it never ran a step. */ @@ -8178,6 +8274,13 @@ export class AgentFramework { emoji: this.discordAwarenessEmoji, }); } catch (error) { + // The record may have landed all the same: retire it, so no staged + // choice outlives a change that was never staged. + try { + this.discordAwarenessOutbox!.discard(this.changeBatchId(change.id)); + } catch { + // Nothing landed, or it can't be retired now: either way nothing activates it. + } return { ok: false, error: `Operator change not staged: its marks choice could not be recorded: ${error instanceof Error ? error.message : String(error)}`, @@ -8375,6 +8478,12 @@ export class AgentFramework { markers = { ...facts, status: 'queued', queued: settled.queued, batchId }; if (settled.queued > 0) this.deliverDiscordAwarenessInBackground(verb, change.agent); } + } else if (attempt.evidence.staged) { + // Staged by a process that had an outbox, applied by one that hasn't: + // the choice can't be carried out here, and says so. + const error = 'no Discord awareness outbox is configured, so its staged marks stay staged'; + markers = { ...facts, status: 'not-scheduled', queued: 0, error }; + this.opsAlert('operator-change-marks-unactivated', change.agent, `${change.kind} ${change.id}: ${error}`); } this.journalChange({ kind: 'outcome', @@ -8502,23 +8611,38 @@ export class AgentFramework { // If writing the attempt throws, it may still have landed; either way // this process knows its cut never ran. note('failed'); - this.journalChange({ - kind: 'attempt', - changeId: change.id, - changeKind: change.kind, - agent: change.agent, - n, - at: Date.now(), - evidence: { - target: destination, - source: change.sourceBranch, - sourceHead: this.store.currentBranch().head, - removed: removed.length, - marks: this.removalMarks(change, removed), - staged: this.hasStagedMarks(change), - ...(change.requester ? { requester: change.requester } : {}), - }, - }); + try { + this.journalChange({ + kind: 'attempt', + changeId: change.id, + changeKind: change.kind, + agent: change.agent, + n, + at: Date.now(), + evidence: { + target: destination, + source: change.sourceBranch, + sourceHead: this.store.currentBranch().head, + removed: removed.length, + marks: this.removalMarks(change, removed), + staged: this.hasStagedMarks(change), + ...(change.requester ? { requester: change.requester } : {}), + }, + }); + } catch (error) { + // journalChange reloads the journal when it can't tell whether the + // write landed. If the attempt is there, its failure is recorded too, + // so a restart reads it as not committed rather than unresolved. + if (record()?.attempts.some((a) => a.n === n)) { + try { + const detail = error instanceof Error ? error.message : String(error); + this.journalChange({ kind: 'failed', changeId: change.id, n, error: `its attempt's record failed before the cut ran: ${detail}` }); + } catch { + // Unrecorded: this process's own knowledge still says it never ran. + } + } + throw error; + } try { await steps.cut(destination); } catch (error) { @@ -8840,7 +8964,40 @@ export class AgentFramework { // Both journals were first read before the agents (an unreadable one // stopped startup there, rather than reading as empty). const records = [...this.changesLedger().records.values()]; + // A change's bookkeeping at startup: its operator-log action (recorded + // once, recordChangeAction) and its completed mark. An attested commit + // carries its attestation, which is the operator's, not something the + // framework observed. + const finish = (record: OperatorChangeRecord, attempt: OperatorChangeRecord['attempts'][number] | undefined, outcome: OperatorChangeOutcome) => { + const attestation = attempt?.resolution; + this.recordChangeAction({ + kind: record.kind === 'undo-messages' ? 'rollback' : record.kind, + agent: record.agent, + ...(attempt?.evidence.requester ? { requester: attempt.evidence.requester } : {}), + params: { + changeId: record.changeId, + recovered: 'startup', + ...(attestation ? { attested: { verdict: attestation.verdict, via: attestation.via, ...(attestation.requester ? { requester: attestation.requester } : {}) } } : {}), + }, + result: { target: attempt?.evidence.target, removed: outcome.removed, markers: outcome.markers }, + }); + this.journalChange({ kind: 'completed', changeId: record.changeId }); + }; for (const record of records) { + if (record.outcome && !record.completed && !record.dropped) { + // Its outcome was established, but the process stopped before its + // bookkeeping: finish it now, rather than wait for a host retry. + try { + finish(record, record.attempts.find((a) => a.n === record.outcome!.n), record.outcome); + console.error(`[operator-changes] ${record.kind} ${record.changeId} for ${record.agent}: its outcome was established before a crash; its bookkeeping finished at startup`); + } catch (error) { + console.error( + `[operator-changes] ${record.kind} ${record.changeId} for ${record.agent}: bookkeeping not finished at startup (left for the host's retry):`, + error instanceof Error ? error.message : error, + ); + } + continue; + } if (record.outcome || record.dropped || record.attempts.length === 0) continue; const last = record.attempts[record.attempts.length - 1]!; const disposition = record.kind === 'hide' ? null : cutAttemptDisposition(record, last); @@ -8874,15 +9031,10 @@ export class AgentFramework { last, record.kind === 'hide' ? 'hide' : 'undo', ); - this.recordChangeAction({ - kind: record.kind === 'undo-messages' ? 'rollback' : record.kind, - agent: record.agent, - ...(last.evidence.requester ? { requester: last.evidence.requester } : {}), - params: { changeId: record.changeId, recovered: 'startup' }, - result: { target: last.evidence.target, removed: outcome.removed, markers: outcome.markers }, - }); - this.journalChange({ kind: 'completed', changeId: record.changeId }); - console.error(`[operator-changes] ${record.kind} ${record.changeId} for ${record.agent}: committed before a crash recorded it; outcome established at startup`); + finish(record, last, outcome); + console.error(last.resolution + ? `[operator-changes] ${record.kind} ${record.changeId} for ${record.agent}: attested committed (${last.resolution.via}); outcome established at startup` + : `[operator-changes] ${record.kind} ${record.changeId} for ${record.agent}: committed before a crash recorded it; outcome established at startup`); } catch (error) { console.error( `[operator-changes] ${record.kind} ${record.changeId} for ${record.agent}: not recovered at startup (left for the host's retry):`, @@ -10743,7 +10895,21 @@ export class AgentFramework { if (this.usedEphemeralAgentNames.has(name) || this.agents.has(name)) { throw new Error(`Conversation agent name "${name}" is already registered or reserved`); } + // A fork enters the store under the creation admission (see + // admitConversationCreation): a waiting lease isn't granted while it + // opens, it's refused once stopping, and stop() waits for one already + // opening. Once registered, the next reservation holds its turn token + // like any agent's. + const pending = await this.admitConversationCreation(); + if (this.usedEphemeralAgentNames.has(name) || this.agents.has(name)) { + this.ephemeralPending.delete(pending); + this.tryGrantSafeBoundary(); + throw new Error(`Conversation agent name "${name}" is already registered or reserved`); + } this.usedEphemeralAgentNames.add(name); + let initialized!: () => void; + const initializing = new Promise((resolve) => { initialized = resolve; }); + this.ephemeralInitializations.add(initializing); try { const contextManager = await ContextManager.open({ @@ -10785,6 +10951,11 @@ export class AgentFramework { } catch (error) { this.usedEphemeralAgentNames.delete(name); throw error; + } finally { + this.ephemeralPending.delete(pending); + this.ephemeralInitializations.delete(initializing); + initialized(); + this.tryGrantSafeBoundary(); } } @@ -16035,8 +16206,12 @@ export class AgentFramework { const midToolCycle = opts?.forAgent ? this.pendingAssistantBlocks.has(agent.name) : this.pendingAssistantBlocks.size > 0; + // The tool_result exemption is for a draining turn. While a reservation + // (a lease or a surgery) holds the store, no turn is running, so a + // tool_result waits like any write: the holder's body change must not + // move underneath it. if ( - !hasToolResult && + (!hasToolResult || this.surgeryHold) && opts?.bypassDeferralFor !== agent.name && (this.quiesced || midToolCycle || @@ -16064,7 +16239,10 @@ export class AgentFramework { if (opts?.placement) { opts.placement.agent = agent.name; opts.placement.deferredId = id; - opts.placement.durable = persisted; + // Durable only as asked for (MessagePlacement.durable): a queue + // persisted for another reason (quiesce, an earlier durable write) + // doesn't make a delivery the caller didn't ask about durable. + opts.placement.durable = durable && persisted; } return '' as MessageId; // Deferred — flushed at the target's next boundary } @@ -16165,6 +16343,13 @@ export class AgentFramework { } private editMessage(id: MessageId, content: ContentBlock[]): void { + // While a reservation (a lease or a surgery) holds the store, a module's + // edit waits for its release, so the holder's body change can't move + // underneath it (flushDeferredMessageOps). + if (this.surgeryHold) { + (this.deferredMessageOps ??= []).push({ op: 'edit', id, content }); + return; + } const agent = this.primaryAgentName ? this.agents.get(this.primaryAgentName) : this.agents.values().next().value; @@ -16175,6 +16360,10 @@ export class AgentFramework { } private removeMessage(id: MessageId): void { + if (this.surgeryHold) { + (this.deferredMessageOps ??= []).push({ op: 'remove', id }); + return; + } const agent = this.primaryAgentName ? this.agents.get(this.primaryAgentName) : this.agents.values().next().value; diff --git a/src/module-registry.ts b/src/module-registry.ts index a8f4c6b30..e6315af8a 100644 --- a/src/module-registry.ts +++ b/src/module-registry.ts @@ -625,8 +625,9 @@ class ModuleContextImpl implements ModuleContext { ): MessageId { const id = this.addMessageFn(participant, content, metadata, options); - // Track external ID if provided - if (metadata?.external) { + // Track external ID if provided. A deferred write returns '' (it has no + // id until it lands), so it has nothing to map yet. + if (metadata?.external && id) { const key = `${metadata.external.source}:${metadata.external.id}`; this.externalIdMap.set(key, id); // Persist the mapping @@ -654,7 +655,8 @@ class ModuleContextImpl implements ModuleContext { findMessageByExternalId(source: string, externalId: string): MessageId | null { const key = `${source}:${externalId}`; - return this.externalIdMap.get(key) ?? null; + // `||`: an empty id an earlier build mapped for a deferred write addresses nothing. + return this.externalIdMap.get(key) || null; } getMessage(id: MessageId): StoredMessage | null { diff --git a/src/operator-change.ts b/src/operator-change.ts index ab55e1ac6..1ab00b798 100644 --- a/src/operator-change.ts +++ b/src/operator-change.ts @@ -439,6 +439,24 @@ export interface OperatorChangeRestorationReceipt { remaining?: string; } +/** + * Why a change has no unresolved attempt to settle, for the refusal of a + * second resolve: the settlement on record, so that a repeated resolve + * tells its operator the attempt is settled, and how. + */ +export function settledState(record: OperatorChangeRecord | undefined): string { + if (!record) return 'no such change is recorded'; + const last = record.attempts[record.attempts.length - 1]; + if (last?.resolution) { + const r = last.resolution; + const who = r.requester ? ` by ${r.requester.name ?? r.requester.id ?? r.requester.via}` : ''; + return `its attempt ${last.n} was settled ${r.verdict}${who} (${r.via}) at ${new Date(r.at).toISOString()}`; + } + if (record.outcome) return 'its outcome is established'; + if (record.dropped) return 'it was dropped'; + return last ? `its attempt ${last.n} isn't unresolved` : 'it has no attempt'; +} + const CHANGE_KINDS = new Set(['agent-settings', 'tool-presentation', 'undo-turns', 'unstick', 'hide', 'undo-messages']); /** The cuts: the kinds whose attempt can be unresolved and settled by an * operator. A hide is journaled as attempts too, but its effect can be read @@ -530,6 +548,10 @@ function checkRecord(r: unknown, id: string): asserts r is OperatorChangeRecord } if (r.completed !== undefined && r.completed !== true) malformed(`${where}: completed`); if (r.dropped !== undefined && !(isObject(r.dropped) && isCount(r.dropped.at))) malformed(`${where}: dropped`); + // A change that began to apply is never dropped (dropResolvedOperatorChange + // refuses it), and a dropped one is skipped by recovery, so the two together + // would hide an attempt from it. + if (r.dropped !== undefined && r.attempts.length > 0) malformed(`${where}: dropped after it began to apply`); } /** One entry, checked before it's folded in: its own shape, and that it @@ -543,6 +565,10 @@ function checkEntry(e: unknown, records: Map): ass case 'attempt': if (!(CUT_KINDS.has(e.changeKind as string) || e.changeKind === 'hide') || !isText(e.agent) || !isCount(e.n) || !isCount(e.at)) malformed(where); if (record && record.kind !== e.changeKind) malformed(`${where}: its kind differs from its change's`); + // Each attempt is journaled once; a second record of its number is + // refused, as a snapshot's is (checkRecord), not folded away. + if (hasAttempt(e.n)) malformed(`${where}: attempt ${String(e.n)} is already recorded`); + if (record?.dropped) malformed(`${where}: its change was dropped`); checkEvidence(e.evidence, where, e.changeKind as string); return; case 'switched': @@ -553,6 +579,8 @@ function checkEntry(e: unknown, records: Map): ass return; case 'resolved': if (!isCount(e.n) || !hasAttempt(e.n)) malformed(where); + // As a snapshot's record is checked (checkRecord): a hide is never held. + if (!CUT_KINDS.has(record!.kind)) malformed(`${where}: only a cut is resolved`); checkResolution(e.resolution, where); return; case 'outcome': @@ -565,6 +593,7 @@ function checkEntry(e: unknown, records: Map): ass case 'dropped': if (!CHANGE_KINDS.has(e.changeKind as string) || !isText(e.agent) || !isCount(e.at)) malformed(where); if (record && record.kind !== e.changeKind) malformed(`${where}: its kind differs from its change's`); + if (record && record.attempts.length > 0) malformed(`${where}: its change began to apply`); return; default: malformed(where); diff --git a/src/recovery/discord-awareness-outbox.ts b/src/recovery/discord-awareness-outbox.ts index ed9f58aeb..5e62311fd 100644 --- a/src/recovery/discord-awareness-outbox.ts +++ b/src/recovery/discord-awareness-outbox.ts @@ -665,13 +665,14 @@ export class DiscordAwarenessOutbox { * per ref, durable before this returns. Activation completes the surgery's * earlier choice; it is not a new one, so an add for a reaction that a * later act (a retract, or another batch's request) already decided is - * recorded superseded and never sent. Returns how many were queued. + * recorded superseded and never sent. Returns how many were queued; for a + * batch already active (a retry), how many its activation queued. */ activate(batchId: string): number { const state = this.load(); const batch = state.batches.get(batchId); if (!batch) throw new Error(`Discord awareness batch not found: ${batchId}`); - if (batch.status === 'active') return 0; + if (batch.status === 'active') return this.recordedQueued(batchId); if (batch.status !== 'prepared') { // A staged batch is activated by its change (activateStaged), never here. throw new Error(`Discord awareness batch ${batchId} is ${batch.status}, not prepared`); @@ -713,6 +714,10 @@ export class DiscordAwarenessOutbox { if (before?.status === 'staged') { return { status: 'unresolved', error: `Discord awareness batch ${batchId} is staged: its change activates it (activateStaged)` }; } + // Already retired (a retry of a call that retired it): replay that. + if (before?.status === 'discarded') { + return { status: 'not-scheduled', error: `Discord awareness batch ${batchId} was retired: none of its marks will be sent` }; + } return { status: 'queued', queued: this.activate(batchId) }; } catch (error) { detail = boundDiscordAwarenessText(error instanceof Error ? error.message : String(error)); @@ -1408,7 +1413,10 @@ export class DiscordAwarenessOutbox { */ private load(): JournalState { if (this.state && !this.journal.needsReconcile) return this.state; - const { snapshot, entries } = this.journal.load(); + const { snapshot, entries, checkpointed } = this.journal.load(); + // A checkpoint covers (and skips) the entries before it, so one with no + // snapshot can't be read as an empty history: refused, like a corrupt entry. + if (checkpointed && !snapshot) throw new Error('Corrupt Discord awareness journal: a checkpoint with no snapshot'); const state = snapshot ? restoreState(snapshot) : emptyState(); for (const { id, entry } of entries) { if (!entry || !Array.isArray(entry.records)) { diff --git a/src/recovery/recover-cli.ts b/src/recovery/recover-cli.ts index b411ebb8d..c41a07801 100644 --- a/src/recovery/recover-cli.ts +++ b/src/recovery/recover-cli.ts @@ -1,5 +1,6 @@ #!/usr/bin/env node +import { existsSync } from 'node:fs'; import { userInfo } from 'node:os'; import { JsStore } from '@animalabs/chronicle'; import { createOfflineRecoveryBranch } from './offline-branch.js'; @@ -19,6 +20,7 @@ import { cutAttemptDisposition, OPERATOR_CHANGES_JOURNAL, readOperatorChanges, + settledState, type OperatorChangeRecord, type OperatorChangeResolution, type OperatorChangeResolutionReceipt, @@ -111,10 +113,10 @@ Gated operator changes (host stopped): activates its staged marks before any traffic. Queued work is delivered when the host next connects to the Discord server. -Each recovery, and each cancel, retract or release, is recorded in -/operator-actions.jsonl, the host's operator log, with the OS account -that ran it, whether it was carried out or refused. A dry run and a list -record nothing. +Each recovery, each cancel, retract or release, and each operator-change +resolve is recorded in /operator-actions.jsonl, the host's operator +log, with the OS account that ran it (or the --requester named), whether it +was carried out or refused. A dry run and a list record nothing. The agent host must be stopped before running this command.`; } @@ -266,18 +268,44 @@ function awareness(options: CliOptions): void { * writer, which stands in for the running framework's lease. */ function operatorChange(options: CliOptions): void { - const store = JsStore.openOrCreate({ path: options.storePath! }); + // An existing store only: a mistyped --store path must not create an empty + // one, which would list nothing held. + if (!existsSync(options.storePath!)) throw new Error(`No store at ${options.storePath}`); + const store = JsStore.open({ path: options.storePath! }); try { const journal = new RecordJournal(store, { type: OPERATOR_CHANGES_JOURNAL }); - // A journal that can't be interpreted is refused, never read as empty. - const records = readOperatorChanges(journal.load()); + const command = options.operatorChange!; + // Who settles it: --requester names them; otherwise the CLI's user, as + // this CLI's other acts record theirs. The same requester goes in the + // attestation and in the operator log. + const requester: OperatorRequester = options.requester ? { via: 'cli', name: options.requester } : cliRequester(); + const logBase = (agent: string): OperatorLogInput => ({ + kind: 'resolve-operator-change', + agent, + requester, + params: { + changeId: command.verb === 'resolve' ? command.changeId : '', + ...(options.attempt !== undefined ? { attempt: options.attempt } : {}), + ...(options.verdict ? { verdict: options.verdict } : {}), + ...(options.reason ? { reason: options.reason } : {}), + via: 'offline', + }, + }); + // A journal that can't be interpreted is refused, never read as empty; + // a resolve it refuses is recorded like any other (a list records nothing). + let records: ReturnType; + try { + records = readOperatorChanges(journal.load()); + } catch (error) { + if (command.verb === 'resolve') recordOperatorAction(options.storePath!, { ...logBase('*'), error: errorMessage(error) }); + throw error; + } const active = store.currentBranch().name; const held = (record: OperatorChangeRecord) => { if (record.outcome || record.dropped || (record.kind !== 'undo-turns' && record.kind !== 'undo-messages')) return null; const last = record.attempts[record.attempts.length - 1]; return last && cutAttemptDisposition(record, last) === 'unresolved' ? last : null; }; - const command = options.operatorChange!; if (command.verb === 'list') { const unresolved = [...records.values()].flatMap((record) => { const attempt = held(record); @@ -301,25 +329,10 @@ function operatorChange(options: CliOptions): void { } const record = records.get(command.changeId); const attempt = record ? held(record) : null; - // Who settles it: --requester names them; otherwise the CLI's user, as - // this CLI's other acts record theirs. The same requester goes in the - // attestation and in the operator log. - const requester: OperatorRequester = options.requester ? { via: 'cli', name: options.requester } : cliRequester(); - const logBase: OperatorLogInput = { - kind: 'resolve-operator-change', - agent: record?.agent ?? '*', - requester, - params: { - changeId: command.changeId, - ...(options.attempt !== undefined ? { attempt: options.attempt } : {}), - ...(options.verdict ? { verdict: options.verdict } : {}), - ...(options.reason ? { reason: options.reason } : {}), - via: 'offline', - }, - }; + const logged = logBase(record?.agent ?? '*'); let receipt: OperatorChangeResolutionReceipt; try { - if (!record || !attempt) throw new Error(`Operator change ${command.changeId} has no unresolved attempt to settle`); + if (!record || !attempt) throw new Error(`Operator change ${command.changeId} has no unresolved attempt to settle: ${settledState(record)}`); if (options.attempt !== attempt.n) throw new Error(`--attempt ${options.attempt} is not its unresolved attempt (${attempt.n} is)`); if (!options.verdict) throw new Error('--verdict committed|not-committed is required'); if (!options.reason?.trim()) throw new Error('--reason is required'); @@ -337,7 +350,9 @@ function operatorChange(options: CliOptions): void { const restoresAtStart = options.verdict === 'not-committed' && active === attempt.evidence.target; const remaining = options.verdict === 'not-committed' && !restoresAtStart && active !== attempt.evidence.source ? `the active branch is ${active}, neither the source ${attempt.evidence.source} nor the attempt's destination` - : undefined; + : options.verdict === 'committed' && active !== attempt.evidence.target + ? `the active branch is ${active}, not this attempt's destination ${attempt.evidence.target}; startup will establish its outcome as applied all the same` + : undefined; receipt = { changeId: record.changeId, attempt: attempt.n, @@ -348,11 +363,11 @@ function operatorChange(options: CliOptions): void { ...(remaining ? { remaining } : {}), }; } catch (error) { - recordOperatorAction(options.storePath!, { ...logBase, error: errorMessage(error) }); + recordOperatorAction(options.storePath!, { ...logged, error: errorMessage(error) }); throw error; } recordOperatorAction(options.storePath!, { - ...logBase, + ...logged, result: { settlement: receipt.settlement, ...(receipt.remaining ? { remaining: receipt.remaining } : {}) }, }); console.log(JSON.stringify(receipt, null, 2)); diff --git a/test/conversation-routing.test.ts b/test/conversation-routing.test.ts index cd55d3c4d..de56f93d2 100644 --- a/test/conversation-routing.test.ts +++ b/test/conversation-routing.test.ts @@ -178,6 +178,55 @@ describe('Conversation routing', () => { await framework.stop(); }); + it('opens a fork only once a held lease releases, so its holder never sees the store change under it', async () => { + membrane.pushResponse(createMockResponse([{ type: 'text', text: 'Hi alice!' }])); + const framework = await makeFramework(); + let release: (() => void) | undefined; + let held!: () => void; + const granted = new Promise((resolve) => { held = resolve; }); + const holding = framework.runAtSafeBoundary({ verb: 'hold' }, () => new Promise((resolve) => { release = resolve; held(); })); + try { + await granted; + framework.pushEvent(incomingEvent({ channelId: 'slack:D1', text: 'hello there', channelType: 'im' })); + const routing = framework.runUntilIdle(); + await new Promise((resolve) => setTimeout(resolve, 50)); + assert.equal(framework.getAgent('conversation-slack-D1-g1'), null, 'no fork while the lease is held'); + release!(); + await holding; + await routing; + assert.ok(framework.getAgent('conversation-slack-D1-g1'), 'opened once the lease released'); + } finally { + release?.(); + await holding; + await framework.stop(); + } + }); + + it('refuses a fork waiting behind a lease once stop() begins, before it enters the store', async () => { + const framework = await makeFramework(); + const quiet = console.error; + console.error = () => {}; + try { + let release!: () => void; + let held!: () => void; + const granted = new Promise((resolve) => { held = resolve; }); + const holding = framework.runAtSafeBoundary({ verb: 'hold' }, () => new Promise((resolve) => { release = resolve; held(); })); + await granted; + framework.pushEvent(incomingEvent({ channelId: 'slack:D1', text: 'hello there', channelType: 'im' })); + const routing = framework.runUntilIdle().catch(() => {}); + await new Promise((resolve) => setTimeout(resolve, 50)); + const stopping = framework.stop(); + release(); + await holding; + await routing; + await stopping; + assert.equal(framework.getAgent('conversation-slack-D1-g1'), null, 'never opened'); + assert.equal(framework.getConversationRouter()!.getBinding('slack:D1'), undefined, 'and never bound'); + } finally { + console.error = quiet; + } + }); + it('channel message without mention is dropped; mention spawns', async () => { membrane.pushResponse(createMockResponse([{ type: 'text', text: 'On it.' }])); const framework = await makeFramework(); diff --git a/test/discord-awareness-outbox.test.ts b/test/discord-awareness-outbox.test.ts index 6b0697bbf..bfe52f79c 100644 --- a/test/discord-awareness-outbox.test.ts +++ b/test/discord-awareness-outbox.test.ts @@ -624,6 +624,14 @@ test('settleActivation says what the journal will do when activation cannot be r const settled = outbox.settleActivation(cancelled.id); assert.equal(settled.status, 'not-scheduled'); assert.equal(outbox.batches().find((b) => b.id === cancelled.id)?.status, 'discarded'); + // A retry replays the retirement, as a retry of an activation replays its count. + assert.equal(outbox.settleActivation(cancelled.id).status, 'not-scheduled'); +})); + +test('activating an active batch again returns what its activation queued', withJournal((outbox) => { + const batch = outbox.prepare({ agentName: 'cairn', sourceBranch: 'main', targetBranch: 't', refs: [ref('m1'), ref('m2')], scope: 'all' })!; + assert.equal(outbox.activate(batch.id), 2); + assert.equal(outbox.activate(batch.id), 2, 'the recorded count, not 0'); })); test('outside error text is bounded before it is journaled', withJournal((outbox) => { @@ -644,6 +652,16 @@ test('a corrupt journal entry stops the reader rather than being skipped', withJ assert.throws(() => h.reopen().batches(), /Corrupt Discord awareness journal entry/); })); +test('a checkpoint with no snapshot is refused, never read as an empty history', withJournal((outbox, h) => { + activeBatch(outbox, [ref('m1')]); + // A checkpoint covers the entries before it, so reading past one with no + // snapshot would lose them. + const journal = new RecordJournal<{ records: unknown[] }, unknown>(h.store, { type: DISCORD_AWARENESS_RECORD_TYPE }); + journal.load(); + journal.checkpoint(null); + assert.throws(() => h.reopen().batches(), /Corrupt Discord awareness journal: a checkpoint with no snapshot/); +})); + test('the journal writes no checkpoints: a long history is replayed whole', withJournal((_outbox, h) => { // Every reduction needs the whole history, so a checkpoint would copy all // of it, and copying it every N entries grows the store quadratically. diff --git a/test/module-targeted-delivery.test.ts b/test/module-targeted-delivery.test.ts index 7ecc93f16..b941f7bd8 100644 --- a/test/module-targeted-delivery.test.ts +++ b/test/module-targeted-delivery.test.ts @@ -165,6 +165,24 @@ describe('ModuleContext.addMessage delivery options', () => { assert.deepEqual(texts(framework, 'other'), [], 'lost with the process, as memory-only deferrals always were'); }); + it("reports a deferral durable only when its caller asked, though the queue was persisted for another", async () => { + (framework as unknown as Internals).activeTurnTokens.set('other', 77); + const asked: MessagePlacement = {}; + courier.ctx.addMessage('user', [{ type: 'text', text: 'asked' }], undefined, { forAgent: 'other', placement: asked, durable: true }); + // The queue is persisted now, so the next deferral rides in it, but its + // caller didn't ask for durability, and isn't told it has it. + const plain: MessagePlacement = {}; + courier.ctx.addMessage('user', [{ type: 'text', text: 'plain' }], undefined, { forAgent: 'other', placement: plain }); + assert.deepEqual([asked.durable, plain.durable], [true, false]); + }); + + it('maps no external id to a deferred write, which has no id until it lands', async () => { + (framework as unknown as Internals).activeTurnTokens.set('scout', 77); + const id = courier.ctx.addMessage('user', [{ type: 'text', text: 'held' }], { external: { source: 'discord', id: 'd1' } }); + assert.equal(id, ''); + assert.equal(courier.ctx.findMessageByExternalId('discord', 'd1'), null, 'not found as the empty id'); + }); + it('drops a message for an unknown agent, and the empty placement says so', async () => { const placement: MessagePlacement = {}; const id = courier.ctx.addMessage('user', [{ type: 'text', text: 'to nobody' }], undefined, { forAgent: 'ghost', placement, durable: true }); diff --git a/test/operator-change-gate.test.ts b/test/operator-change-gate.test.ts index bfec499f6..42368e927 100644 --- a/test/operator-change-gate.test.ts +++ b/test/operator-change-gate.test.ts @@ -330,7 +330,10 @@ describe('operator-change gate: self-change tools', () => { }); it('resolves a reset to the concrete values it restores, and refuses if those change', async () => { - framework.updateAgentRuntimeSettings('scout', { contextBudgetTokens: 120_000 }); + // Below the configured budget, so the reset is a raise: this agent's + // strategy has no hot settings, and a reset that starts a descent is + // refused at staging, as it would be at apply. + framework.updateAgentRuntimeSettings('scout', { contextBudgetTokens: 80_000, immediate: true }); await framework.puppetToolCall('scout', 'agent_settings', { action: 'reset', settings: ['context_budget_tokens'] }); const change = asked[0]!; assert.equal(change.kind === 'agent-settings' && change.target.contextBudgetTokens, 100_000, 'the configured default, concretely'); @@ -346,7 +349,21 @@ describe('operator-change gate: self-change tools', () => { } finally { agent.previewRuntimeSettingsTarget = real; } - assert.equal(budget(), 120_000, 'nothing applied'); + assert.equal(budget(), 80_000, 'nothing applied'); + }); + + it("refuses at staging a descent this agent's strategy can't prepare, as applying it would", async () => { + // scout's strategy has no hot settings: a paced decrease, or a reset that + // starts one, would be refused when applied, so it's never staged. + const before = budget(); + const lowered = await framework.puppetToolCall('scout', 'agent_settings', { action: 'update', context_budget_tokens: before - 10_000 }); + assert.match(String(lowered.result.error), /couldn't be resolved \(The active context strategy cannot prepare a smaller window live\)/); + framework.updateAgentRuntimeSettings('scout', { contextBudgetTokens: before + 20_000 }); + const reset = await framework.puppetToolCall('scout', 'agent_settings', { action: 'reset', settings: ['context_budget_tokens'] }); + assert.match(String(reset.result.error), /cannot prepare a smaller window live/); + assert.equal(asked.length, 0, 'neither reached the gate'); + const immediate = await framework.puppetToolCall('scout', 'agent_settings', { action: 'update', context_budget_tokens: before - 10_000, immediate: true }); + assert.match(String(immediate.result.error), /staged agent-settings/, 'an immediate decrease needs no prepared window'); }); it("needs the held lease to apply, whether the change was an operator's or a module's", async () => { @@ -1047,6 +1064,46 @@ describe('operator-change gate: host/command undo by turns', () => { await assert.rejects(applyIt(dropped), (e: Error & { code?: string }) => e.code === 'stale' && /dropped/.test(e.message)); }); + it('keeps a drop when its staged choice cannot be discarded, and never drops a change that began to apply', async () => { + await say('a0'); await say('a1'); await say('a2'); + await undoWithMarks(2, 'all'); + const change = asked[0]!; + const box = outbox() as unknown as { discard: (id: string) => boolean }; + const realDiscard = box.discard.bind(box); + box.discard = () => { throw new Error('injected discard failure'); }; + try { + framework.dropResolvedOperatorChange(change); // the drop is recorded first, and stands + } finally { + box.discard = realDiscard; + } + assert.ok(framework.getOperatorChangeRecord(change.id)!.dropped, 'dropped'); + assert.equal(outbox().batches().find((x) => x.id === `op-${change.id}`)?.status, 'staged', 'its choice is left staged, and inert'); + await assert.rejects(applyIt(change), (e: Error & { code?: string }) => e.code === 'stale' && /dropped/.test(e.message)); + + await undoWithMarks(1, 'all'); + const begun = asked[1]!; + await applyIt(begun); + assert.throws(() => framework.dropResolvedOperatorChange(begun), (e: Error & { code?: string }) => + e.code === 'invalid' && /has begun to apply; it can't be dropped/.test(e.message)); + }); + + it('retires a staged choice whose record landed though staging could not tell', async () => { + await say('a0'); await say('a1'); await say('a2'); + const box = outbox() as unknown as { prepare: (input: unknown) => unknown }; + const realPrepare = box.prepare.bind(box); + box.prepare = (input) => { realPrepare(input); throw new Error('injected: its record landed, but the write threw'); }; + let r: Record; + try { + r = await undoWithMarks(2, 'all'); + } finally { + box.prepare = realPrepare; + } + assert.match(String(r.error), /not staged: its marks choice could not be recorded/); + assert.equal(asked.length, 0, 'the gate was never asked'); + const batches = outbox().batches(); + assert.deepEqual(batches.map((x) => x.status), ['discarded'], 'retired, so no staged choice outlives a change never staged'); + }); + it('stages a hide with its exact messages and frozen refs, applies it under the lease, and reports one receipt', async () => { await say('h0'); await say('h1'); await say('h2'); const r = await host().handleHostCommand('discord', { command: 'hide', agentName: 'scout', fromMessageId: 'h1', toMessageId: 'h2', marks: 'all', requesterName: 'nissa' }); @@ -1270,6 +1327,95 @@ describe('operator-change gate: host/command undo by turns', () => { assert.deepEqual([record.switched, record.attempts[0]!.resolution!.verdict], [undefined, 'committed'], 'an attestation, not a framework-observed switch'); }); + it('finishes at the next start the bookkeeping a crash cut off after its outcome', async () => { + await turn('one'); await turn('two'); + await undo(1); + const change = asked[0]!; + const restore = failLogOnce('undo-turns'); + try { + await assert.rejects(applyIt(change), /injected log failure/); + } finally { + restore(); + } + const before = framework.getOperatorChangeRecord(change.id)!; + assert.deepEqual([!!before.outcome, before.completed], [true, undefined], 'its outcome stands; its bookkeeping does not'); + await reopen(); + assert.equal(framework.getOperatorChangeRecord(change.id)!.completed, true, 'finished at startup, before any retry'); + const logged = framework.getOperatorLog().filter((e) => e.kind === 'undo-turns' && (e.params as { changeId?: string }).changeId === change.id); + assert.deepEqual(logged.map((e) => (e.params as { recovered?: string }).recovered), ['startup']); + }); + + it('says when an attestation contradicts the active body, and a repeated resolve names the settlement on record', async () => { + await say('m0'); await say('m1'); + await host().handleHostCommand('discord', { command: 'undo', agentName: 'scout', messages: 2, marks: 'all', requesterName: 'nissa' }); + const change = asked[0]!; + const undoCut = failCutAfterInitializerWrite(); + const undoRecord = failJournalOnce('failed'); + try { + await assert.rejects(applyIt(change), /injected strategy initialization failure/); + } finally { + undoCut(); undoRecord(); + } + await reopen(); + assert.equal(branch(), change.sourceBranch, 'the source is active, not the destination'); + const settled = await resolve(change.id, 1, 'committed', 'the operator says it applied'); + assert.match(String(settled.remaining), + new RegExp(`the active branch is ${change.sourceBranch}, not this attempt's destination undo-msgs/scout/op-${change.id}`)); + await assert.rejects(resolve(change.id, 1, 'committed', 'once more'), (e: Error & { code?: string }) => + e.code === 'invalid' && /has no unresolved attempt to settle: its attempt 1 was settled committed by nissa \(live\)/.test(e.message)); + }); + + it("refuses to settle an attempt for an agent it can't restore, leaving it unresolved", async () => { + await say('a0'); await say('a1'); await say('a2'); + await undoWithMarks(2, 'all'); + const change = asked[0]!; + const undoRecord = failJournalOnce('switched'); + try { + await assert.rejects(applyIt(change), /injected switched failure/); + } finally { + undoRecord(); + } + await reopen(); + assert.equal(branch(), `undo/scout/op-${change.id}`, 'its destination is active, so not-committed would restore'); + const agents = (framework as unknown as { agents: Map }).agents; + const scout = agents.get('scout')!; + await assert.rejects( + framework.runAtSafeBoundary({ verb: 'resolve' }, async (lease) => { + agents.delete('scout'); + try { + return await framework.resolveOperatorChange(change.id, 1, 'not-committed', { lease, reason: 'nothing applied', requester: { via: 'test', name: 'nissa' } }); + } finally { + agents.set('scout', scout); + } + }), + (e: Error & { code?: string }) => e.code === 'unknown-agent', + ); + assert.deepEqual(framework.getOperatorChangeRecord(change.id)!.unresolved, { n: 1, target: `undo/scout/op-${change.id}` }, 'no verdict was journaled'); + }); + + it('records the failure of an attempt whose record landed though its write threw, so a restart reads it as not committed', async () => { + await turn('one'); await turn('two'); + await undo(1); + const change = asked[0]!; + const fw = framework as unknown as { journalChange: (entry: { kind: string }) => void }; + const real = fw.journalChange.bind(fw); + let armed = true; + fw.journalChange = (entry) => { + real(entry); + if (armed && entry.kind === 'attempt') { armed = false; throw new Error('injected: the attempt landed, but its write threw'); } + }; + try { + await assert.rejects(applyIt(change), /the attempt landed, but its write threw/); + } finally { + fw.journalChange = real; + } + assert.match(String(framework.getOperatorChangeRecord(change.id)!.attempts[0]!.failed), /its attempt's record failed before the cut ran/); + await reopen(); + assert.equal(framework.getOperatorChangeRecord(change.id)!.unresolved, undefined, 'settled by its own record, not left for an operator'); + const applied = await applyIt(change); + assert.equal(applied.kind === 'undo-turns' && applied.undone, 1, 'a fresh attempt applies'); + }); + it('refuses resume(), forced or not and through host/command, while the active body is an unresolved cut\'s destination, until it is settled', async () => { await say('a0'); await say('a1'); await say('a2'); await undoWithMarks(2, 'all'); @@ -1638,6 +1784,11 @@ describe('operator-change gate: host/command undo by turns', () => { [['scout', 2, 'offline', true, false], ['scout', 1, 'offline', false, true]], 'the refusal and the resolution, each logged'); assert.deepEqual(logged[1].requester, { via: 'cli', name: 'nissa' }); assert.match(settled.settlement, new RegExp(`abandoned; the next start restores ${change.sourceBranch} before any agent initializes`)); + // A repeated resolve is refused, and says how the attempt was settled. + const again = spawnSync(process.execPath, [cli, '--store', storePath, '--operator-change', 'resolve', change.id, + '--attempt', '1', '--verdict', 'not-committed', '--reason', 'once more'], { encoding: 'utf8' }); + assert.equal(again.status, 1); + assert.match(again.stderr, /has no unresolved attempt to settle: its attempt 1 was settled not-committed by nissa \(offline\)/); assert.deepEqual(start(), { started: true, branch: change.sourceBranch, quiesced: false }, 'normal readiness: restored before the agent initialized'); await reopen().catch(() => {}); // the framework in this process was stopped above }); @@ -1670,6 +1821,10 @@ describe('operator-change gate: host/command undo by turns', () => { assert.deepEqual([record.outcome?.markers.status, record.outcome?.markers.queued], ['queued', 1], 'established at startup'); assert.equal(outbox().batches().find((x) => x.id === `op-${change.id}`)?.status, 'active'); assert.equal(quiesced(), false); + const logged = framework.getOperatorLog().find((e) => e.kind === 'undo-turns' && (e.params as { changeId?: string }).changeId === change.id)!; + const params = logged.params as { recovered?: string; attested?: { verdict: string; via: string } }; + assert.deepEqual([params.recovered, params.attested?.verdict, params.attested?.via], ['startup', 'committed', 'offline'], + "the operator's attestation, not a switch the framework saw"); }); it('refuses an undo by messages whose tail changed since staging', async () => { @@ -2387,6 +2542,52 @@ describe('operator journals that cannot be read', () => { await refuses([{ kind: 'switched', changeId: 'never-attempted', n: 1 }], /switched entry/); }); + it("refuses, guard by guard, what recovery can't read as its ledger", async () => { + const cut = (ev: object) => ({ kind: 'attempt', changeId: 'c', changeKind: 'undo-turns', agent: 'scout', n: 1, at: 1, evidence: ev }); + const resolution = { verdict: 'committed', reason: 'it applied', at: 2, via: 'live' }; + const cases: Array<[string, object[], RegExp]> = [ + ['a hide whose target is not its own branch', [hideAttempt({ ...hideEvidence, target: 'elsewhere' })], /a hide stays on its branch/], + ['a cut carrying a hide\'s ids', [cut({ ...evidence, ids: hideEvidence.ids })], /a cut records no hide ids/], + ['a sourceHead that is no count', [cut({ ...evidence, sourceHead: 'one' })], /evidence fields/], + ['marks with an unknown scope', [cut({ ...evidence, marks: { ...evidence.marks, scope: 'some' } })], /marks facts/], + ['a resolution with no verdict it knows', [cut(evidence), { kind: 'resolved', changeId: 'c', n: 1, resolution: { ...resolution, verdict: 'maybe' } }], /resolution/], + ['a resolution with no reason', [cut(evidence), { kind: 'resolved', changeId: 'c', n: 1, resolution: { ...resolution, reason: '' } }], /resolution/], + ['a resolution of a hide, which is never held', [hideAttempt(hideEvidence), { kind: 'resolved', changeId: 'h', n: 1, resolution }], /resolved entry|only a cut is resolved/], + ['a failure of an attempt never made', [cut(evidence), { kind: 'failed', changeId: 'c', n: 2, error: 'x' }], /failed entry/], + ['an attempt recorded twice', [cut(evidence), cut(evidence)], /attempt 1 is already recorded/], + ['a drop after the change began to apply', [cut(evidence), { kind: 'dropped', changeId: 'c', changeKind: 'undo-turns', agent: 'scout', at: 2 }], /its change began to apply/], + ['an attempt after its change was dropped', [{ kind: 'dropped', changeId: 'c', changeKind: 'undo-turns', agent: 'scout', at: 1 }, cut(evidence)], /its change was dropped/], + ['an attempt of a kind that records none', [{ ...cut(evidence), changeKind: 'agent-settings' }], /attempt entry/], + ]; + for (const [what, entries, why] of cases) { + await refuses(entries, why).catch((error: Error) => { throw new Error(`${what}: ${error.message}`); }); + } + }); + + it('reads the store identity from its first usable record, so an unusable one never mints a new identity at each start', async () => { + const storePath = join(dir, 'identity'); + const seeded = JsStore.openOrCreate({ path: storePath }); + seeded.appendJson('framework/store-identity', { id: 42 }); // the oldest record, and unusable + seeded.sync(); + seeded.close(); + const start = () => AgentFramework.create({ + storePath, + membrane: new MockMembrane().asMembrane(), + agents: [{ name: 'scout', model: 'test-model', systemPrompt: 'You are scout.', maxTokens: 1000 }], + modules: [], + }); + const identities: string[] = []; + for (let n = 0; n < 2; n++) { + const fw = await start(); + identities.push(fw.getStoreIdentity()); + const records = fw.getStore().getRecordIdsByType('framework/store-identity').length; + await fw.stop(); + assert.equal(records, 2, 'the unusable record and the one identity minted, never another'); + } + assert.equal(identities[0], identities[1], 'stable across restarts'); + assert.match(identities[0]!, /^[0-9a-f-]{36}$/); + }); + it('refuses a null-snapshot checkpoint in the unstick journal too', async () => { const store = JsStore.openOrCreate({ path: join(dir, 'store') }); const launched = store.appendJson('operator/unstick', { kind: 'attempt-launched', operationId: 'op', agent: 'scout', step: 1 }); @@ -2647,6 +2848,27 @@ describe('operator-change gate: the API server', () => { }); }); + it('refuses to delete a branch an unsettled operator change reads, and deletes any other as before', async () => { + await withApi(true, async ({ framework, send }) => { + const store = framework.getStore(); + store.createBranch('held-destination'); + // A cut attempt with neither its switch nor its failure recorded: its + // recovery reads its destination by name. + const journal = (framework as unknown as { journalChange: (entry: object) => void }); + journal.journalChange({ + kind: 'attempt', changeId: 'held', changeKind: 'undo-turns', agent: 'scout', n: 1, at: Date.now(), + evidence: { target: 'held-destination', source: store.currentBranch().name, sourceHead: store.currentBranch().head, removed: 1, staged: false, + marks: { scope: 'none', refs: [], unmarked: 0, notRemoved: 0 } }, + }); + const refused = await send('branch.delete', { name: 'held-destination' }); + assert.equal(refused.success, false); + assert.match(String(refused.error), /Cannot delete held-destination: operator change held isn't settled yet, and its recovery reads that branch/); + assert.ok(store.listBranches().some((b) => b.name === 'held-destination'), 'still there'); + store.createBranch('free'); + assert.equal((await send('branch.delete', { name: 'free' })).success, true, 'a branch no change reads is deleted as before'); + }); + }); + it('keeps undo, redo and branch switches without a gate', async () => { await withApi(false, async ({ framework, send, texts }) => { const undo = await send('undo', { agentName: 'scout' }); diff --git a/test/recover-cli-operator-log.test.ts b/test/recover-cli-operator-log.test.ts index 91c470490..f582f92fc 100644 --- a/test/recover-cli-operator-log.test.ts +++ b/test/recover-cli-operator-log.test.ts @@ -173,3 +173,39 @@ test('an offline recovery records its marks choice and receipt in the operator l rmSync(dir, { recursive: true, force: true }); } }); + +test('--operator-change opens only an existing store: a mistyped path is refused, and nothing is created', () => { + const dir = mkdtempSync(join(tmpdir(), 'recover-cli-log-')); + try { + const missing = join(dir, 'no-such-store'); + const listed = recover(missing, '--operator-change', 'list'); + assert.equal(listed.status, 1); + assert.match(listed.stderr, new RegExp(`No store at ${missing.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}`)); + assert.equal(existsSync(missing), false, 'no empty store was created to list nothing from'); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); + +test('an operator-change resolve refused on a journal it cannot read is recorded like any other refusal', () => { + const dir = mkdtempSync(join(tmpdir(), 'recover-cli-log-')); + const storePath = join(dir, 'store'); + try { + const store = JsStore.openOrCreate({ path: storePath }); + store.appendJson('operator/changes', { kind: 'attempt', changeId: 'half' }); + store.sync(); + store.close(); + const refused = recover(storePath, '--operator-change', 'resolve', 'half', '--attempt', '1', '--verdict', 'not-committed', + '--reason', 'checking', '--requester', 'nissa'); + assert.equal(refused.status, 1); + assert.match(refused.stderr, /malformed operator\/changes/); + const logged = operatorLog(storePath).filter((e) => e.kind === 'resolve-operator-change'); + assert.equal(logged.length, 1); + assert.match(String(logged[0]!.error), /malformed operator\/changes/); + assert.deepEqual(logged[0]!.requester, { via: 'cli', name: 'nissa' }); + assert.equal(recover(storePath, '--operator-change', 'list').status, 1, 'a list is refused too'); + assert.equal(operatorLog(storePath).filter((e) => e.kind === 'resolve-operator-change').length, 1, 'and records nothing'); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); diff --git a/test/safe-boundary-lease.test.ts b/test/safe-boundary-lease.test.ts index a77af38b2..b128b7823 100644 --- a/test/safe-boundary-lease.test.ts +++ b/test/safe-boundary-lease.test.ts @@ -390,6 +390,66 @@ describe('runAtSafeBoundary', () => { c.cleanup(); }); + const withinSeconds = (p: Promise, seconds = 2) => Promise.race([ + p, + new Promise((_, reject) => setTimeout(() => reject(new Error(`not settled within ${seconds}s`)), seconds * 1000)), + ]); + + it("grants a lease waiting behind a direct surgery when the surgery releases, with no scheduler pass", async () => { + const release = i.reserveStoreForSurgery('rollback', 'scout'); + let granted = false; + const leasing = framework.runAtSafeBoundary({ verb: 'behind surgery' }, async () => { granted = true; }); + await tick(); + assert.equal(granted, false, 'waits while the surgery holds the store'); + release(); + await withinSeconds(leasing); + assert.equal(granted, true); + }); + + it("grants the next lease, and keeps the callback's result, when its release's deferred-write flush throws", async () => { + const internals = framework as unknown as { ackDeferredWrites: () => void }; + const realAck = internals.ackDeferredWrites.bind(framework); + let armed = true; + internals.ackDeferredWrites = () => { + if (armed) { armed = false; throw new Error('injected flush failure'); } + return realAck(); + }; + try { + let next!: Promise; + const result = await framework.runAtSafeBoundary({ verb: 'first' }, async () => { + i.addMessage('user', [{ type: 'text', text: 'deferred during the first' }], undefined, { forAgent: 'scout' }); + next = framework.runAtSafeBoundary({ verb: 'second' }, async () => 'second ran'); + return 'first ran'; + }); + assert.equal(result, 'first ran', "the callback's own outcome stands"); + assert.equal(await withinSeconds(next), 'second ran', 'the waiting lease was granted'); + } finally { + internals.ackDeferredWrites = realAck; + } + }); + + it('defers even a tool_result while a reservation holds the store: no turn is draining then', async () => { + await framework.runAtSafeBoundary({ verb: 'defer' }, async () => { + i.addMessage('user', [{ type: 'tool_result', toolUseId: 'late', content: 'a module answered' }] as unknown as ContentBlock[], undefined, { forAgent: 'scout' }); + assert.equal(i.deferredMessages.length, 1, 'deferred behind the reservation'); + }); + assert.equal(i.deferredMessages.length, 0, 'flushed at release'); + }); + + it("holds a module's edit and removal until the reservation releases, then applies them in order", async () => { + const cm = framework.getAgent('scout')!.getContextManager(); + const first = cm.addMessage('user', [{ type: 'text', text: 'original' }]); + const second = cm.addMessage('user', [{ type: 'text', text: 'to remove' }]); + const ops = framework as unknown as { editMessage(id: string, c: ContentBlock[]): void; removeMessage(id: string): void }; + const texts = () => (cm.getAllMessages() as Array<{ content: Array<{ text?: string }> }>).map((m) => m.content[0]?.text); + await framework.runAtSafeBoundary({ verb: 'change' }, async () => { + ops.editMessage(first, [{ type: 'text', text: 'edited' }]); + ops.removeMessage(second); + assert.deepEqual(texts(), ['original', 'to remove'], 'nothing moved under the holder'); + }); + assert.deepEqual(texts(), ['edited'], 'both applied at release'); + }); + it('keeps no landed-write ids for memory-only deferrals', async () => { i.activeTurnTokens.set('other', 9_999); for (let n = 0; n < 4; n++) i.addMessage('user', [{ type: 'text', text: `ordinary ${n}` }], undefined, { forAgent: 'other' }); From e1cd85b5034952a2d257f685a7ba2944d01b378a Mon Sep 17 00:00:00 2001 From: Agnes-1009 Date: Sun, 11 Oct 2026 16:04:46 -0400 Subject: [PATCH 72/73] test(operator-changes): tests that say what they claim; and an unstick re-run that membrane's error event ended without its error The same review's test findings, plus one code fix that a test exposed. The two commits before this one are its text and its code. **What a faithful fake found.** The setup-failure fake threw synchronously from streamYielding. Membrane never does that: it returns the stream at once and reports a provider's refusal on it, as an `error` event. The fake now fails as membrane does. Two tests then failed, against real defects in the unstick re-run: - **A re-run that the error event ended** was settled "the re-run ended without a response", not with the provider's error. The error branch now settles it with the error, as the setup path's catch already did. - **A re-run that a retry or a provider hold carries on** from that branch was settled failed by the ending stream's teardown. The retry had started its own stream, and the hold kept the request, but the teardown looked only at queued requests. It now asks unstickRunCarried, as the setup path does. That counts a successor's turn, so the ending stream's own token is passed to keep its own turn from counting. - **Both routes stay tested.** The re-run test runs once with the failure on the stream ('event') and once before it ('throw'), since a compile can still throw there. **Tests that now say what they claim** - The undo whose awareness delivery fails is asserted to report that failure on its own, as an ops alert. The receipts the finding named come from activation, not delivery. The undo's own receipt says queued either way. - A resolve with no reason is refused, and leaves the attempt unresolved. - The provider-hold test checks that it waited out the hold, and that the host was asked again at its end and released it. - A request whose signal was already aborted is refused, with nothing granted or queued. - A creation parked behind a waiting lease proceeds once that lease is withdrawn. - A stale lease is refused while another lease is held, not only when none is. - A bounded text whose head cut and tail cut each fall inside a surrogate pair keeps neither half. - The duplicate re-run caller is checked to have joined the first caller's waiter, not inferred from a 50 ms race. **Timing, for a loaded machine** - The subconscious delivery waits up to 10 s, not 250 ms. - The leftover task waits for the next script to let it go, not 0.3 s, so it's always still running when its own script ends. - The re-run helper waits up to 10 s, not 3. **The check, from a clean worktree of this commit as AF's CI runs it (npm install; node 24.19.0, npm 11.17.0):** - **The suite:** `tsc --noEmit` and the build are clean. `npm test` reported 1556 of 1556 passing, across 140 suites, with the four new tests among them. The same five tests at the end of channel-registry-routing.test.ts went unreported, with none failing, as at the commit before. - **Red first:** this commit's five test files, on the previous commit's sources, fail exactly the two that the error event's defects break: the 'event' re-run and the retry-and-hold re-run. The other new and extended tests pin what already held, and their removals show they bite. - **Removals:** each of eight, taken out on its own, fails its test: - the error branch's settle (the 'event' variant fails, the 'throw' one still passes); - the teardown's successor check; - the pre-abort guard; - the withdrawal's wake; - heldLeaseTokens' identity check; - the head trim and the tail trim; - the delivery's ops alert (this one run by hand in the working tree, where the scripted removal didn't build). Co-Authored-By: Agnes (Claude Opus 5.5) --- src/framework.ts | 18 +++++++---- test/discord-awareness-outbox.test.ts | 9 ++++++ test/operator-change-gate.test.ts | 44 +++++++++++++++++++------- test/safe-boundary-lease.test.ts | 36 +++++++++++++++++++++ test/subconscious-module-tools.test.ts | 3 +- test/tool-call-origin.test.ts | 8 +++-- 6 files changed, 98 insertions(+), 20 deletions(-) diff --git a/src/framework.ts b/src/framework.ts index c690ad973..ef6506877 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -12644,10 +12644,13 @@ export class AgentFramework { * queued request (a requeue, or a budget restart), or a request a provider * hold keeps. Another turn carries it only if it is bound to the same * operation and step. */ - private unstickRunCarried(agentName: string, binding: { operationId: string; step: number }): boolean { + private unstickRunCarried(agentName: string, binding: { operationId: string; step: number }, endingToken?: number): boolean { const carries = (r: InferenceRequest | undefined) => r?.unstick?.operationId === binding.operationId && r.unstick.step === binding.step; - return carries(this.activeTurnTriggers.get(agentName)) + // A stream that is ending still holds the agent's turn until its own + // teardown: only a successor's turn (another token) carries it on. + const live = endingToken === undefined || this.activeTurnTokens.get(agentName) !== endingToken; + return (live && carries(this.activeTurnTriggers.get(agentName))) || this.pendingRequests.some((r) => r.agentName === agentName && carries(r)) || (this.providerAccelerationCooldowns.get(agentName)?.heldRequests.some(carries) ?? false); } @@ -14118,6 +14121,10 @@ export class AgentFramework { speech: '', error: err.message, }); + // A provider failure that membrane reports on the stream, as it + // reports a setup failure, ends an unstick re-run with the + // provider's error, as a failure thrown at setup does. + if (trigger?.unstick) this.settleUnstickAttempt(trigger.unstick, 'failed', { error: err.message }); this.emitTrace({ type: 'inference:exhausted', agentName: agent.name, @@ -14378,10 +14385,9 @@ export class AgentFramework { if (this.agents.get(agent.name) === agent && agent.streamId === myStreamId) this.eventGate?.onInferenceEnded(agent.name); } finally { // An unstick re-run that ended without an outcome (aborted, cancelled) - // failed, unless a budget restart carries it on. - if (trigger?.unstick && !this.pendingRequests.some((r) => - r.agentName === agent.name && r.unstick?.operationId === trigger.unstick!.operationId - && r.unstick.step === trigger.unstick!.step)) { + // failed, unless another turn carries it on: a retry that started its + // stream, a requeue or budget restart, or a provider hold. + if (trigger?.unstick && !this.unstickRunCarried(agent.name, trigger.unstick, myTurnToken)) { this.settleUnstickAttempt(trigger.unstick, 'failed', { error: 'the re-run ended without a response' }); } const unsettledKvSubmissions = drainKvSubmissionIds?.() ?? []; diff --git a/test/discord-awareness-outbox.test.ts b/test/discord-awareness-outbox.test.ts index bfe52f79c..d33b5ca29 100644 --- a/test/discord-awareness-outbox.test.ts +++ b/test/discord-awareness-outbox.test.ts @@ -647,6 +647,15 @@ test('outside error text is bounded before it is journaled', withJournal((outbox assert.ok((op.attempts[0].error ?? '').length <= 500); })); +test('a bounded text never splits a surrogate pair, at the head cut or the tail cut', () => { + // The head keeps 360 units and the tail 100: each cut here falls inside a pair. + const text = `${'x'.repeat(359)}😀${'y'.repeat(1000)}😀${'z'.repeat(99)}`; + const bounded = boundDiscordAwarenessText(text); + assert.ok(!/[\uD800-\uDBFF](?![\uDC00-\uDFFF])|(? { h.store.appendJson(DISCORD_AWARENESS_RECORD_TYPE, { records: 'nope' }); assert.throws(() => h.reopen().batches(), /Corrupt Discord awareness journal entry/); diff --git a/test/operator-change-gate.test.ts b/test/operator-change-gate.test.ts index 42368e927..63361bbdd 100644 --- a/test/operator-change-gate.test.ts +++ b/test/operator-change-gate.test.ts @@ -31,7 +31,7 @@ import type { ToolDefinition, ToolResult, } from '../src/index.js'; -import { MockMembrane, createMockResponse } from './helpers/mock-membrane.js'; +import { MockMembrane, MockYieldingStream, createMockResponse } from './helpers/mock-membrane.js'; class Mover implements Module { readonly name = 'mover'; @@ -1358,6 +1358,9 @@ describe('operator-change gate: host/command undo by turns', () => { } await reopen(); assert.equal(branch(), change.sourceBranch, 'the source is active, not the destination'); + await assert.rejects(resolve(change.id, 1, 'committed', ' '), (e: Error & { code?: string }) => + e.code === 'invalid' && /needs a reason/.test(e.message)); + assert.ok(framework.getOperatorChangeRecord(change.id)!.unresolved, 'a verdict with no reason is never recorded'); const settled = await resolve(change.id, 1, 'committed', 'the operator says it applied'); assert.match(String(settled.remaining), new RegExp(`the active branch is ${change.sourceBranch}, not this attempt's destination undo-msgs/scout/op-${change.id}`)); @@ -1865,6 +1868,9 @@ describe('operator-change gate: host/command undo by turns', () => { // The real outbox queues the marks; only delivery is stubbed: it fails // the first time and never finishes the second. fw.syncDiscordAwarenessMarkers = () => { calls++; return calls === 1 ? Promise.reject(new Error('delivery down')) : new Promise(() => {}); }; + const alerts: string[] = []; + const ops = framework as unknown as { opsAlert: (kind: string, agent: string, message: string) => void }; + ops.opsAlert = (kind, _agent, message) => { alerts.push(`${kind}: ${message}`); }; await say('a0'); await say('a1'); await say('a2'); await say('a3'); for (const id of ['rev-a', 'rev-b']) { await undoWithMarks(2, 'all'); // two turns, so the cut removes addressed inputs to mark @@ -1874,6 +1880,9 @@ describe('operator-change gate: host/command undo by turns', () => { assert.equal(applied.kind === 'undo-turns' && applied.markers?.status, 'queued', 'marks queued; the undo stands either way'); } assert.equal(calls, 2, 'delivery started both times, and neither apply waited for it'); + await new Promise((resolve) => setImmediate(resolve)); + assert.deepEqual(alerts, ['discord-awareness-delivery: Awareness mark delivery after undo failed: delivery down'], + 'the failed delivery is reported on its own, as an ops alert'); }); it('refuses a staged undo whose checkpoint is gone though the branch stayed', async () => { @@ -2049,7 +2058,7 @@ describe('operator-change gate: host/command unstick, planned at staging', () => // A re-run's outcome, or 'pending' if it hasn't settled while the framework // was driven for `ms`. - const outcomeWithin = async (outcome: Promise, ms = 3_000) => { + const outcomeWithin = async (outcome: Promise, ms = 10_000) => { let settled: unknown = 'pending'; void outcome.then((o) => { settled = o; }, (e) => { settled = e; }); const deadline = Date.now() + ms; @@ -2059,13 +2068,20 @@ describe('operator-change gate: host/command unstick, planned at staging', () => } return settled; }; - // The provider refuses to set up the stream `times` times, then serves. - const failSetup = (times: number, message: string) => { + // The provider refuses to set up the stream `times` times, then serves. As + // membrane reports it ('event'): the stream comes back at once, and the + // refusal arrives on it as an error event (its runInference runs in the + // background). 'throw' is a failure before any stream, as a compile's is. + const failSetup = (times: number, message: string, how: 'event' | 'throw' = 'event') => { const real = membrane.streamYielding.bind(membrane); let left = times; membrane.streamYielding = (...args: Parameters) => { - if (left-- > 0) throw new Error(message); - return real(...args); + if (left-- <= 0) return real(...args); + if (how === 'throw') throw new Error(message); + membrane.calls.push(args[0]); + const refused = new MockYieldingStream([]); + (refused as unknown as { events: unknown[] }).events.push({ type: 'error', error: new Error(message) }); + return refused; }; return () => { membrane.streamYielding = real; }; }; @@ -2074,12 +2090,12 @@ describe('operator-change gate: host/command unstick, planned at staging', () => providerHoldHook?: (e: Error) => { holdMs: number } | undefined; }; - it('settles and releases a re-run whose turn fails before its stream starts: its caller hears, and the agent is not left busy', async () => { + for (const how of ['event', 'throw'] as const) it(`settles and releases a re-run whose turn fails ${how === 'event' ? 'on its stream, as membrane reports a provider refusal' : 'before its stream starts'}: its caller hears, and the agent is not left busy`, async () => { await unstick(3); const change = asked[0]!; await step(change, 1); (framework as unknown as Policies).errorPolicy = { onInferenceError: () => ({ retry: false }) }; - const undoSetup = failSetup(1, 'provider setup refused the request'); + const undoSetup = failSetup(1, 'provider setup refused the request', how); try { const outcome = await outcomeWithin(framework.rerunUnstick(change as never, { step: 1 })); assert.deepEqual(outcome, { step: 1, status: 'completed', outcome: 'failed', error: 'provider setup refused the request' }); @@ -2119,8 +2135,11 @@ describe('operator-change gate: host/command unstick, planned at staging', () => undoSetup = failSetup(1, 'quota exhausted for now'); try { membrane.pushResponse(createMockResponse([{ type: 'text', text: 'back' }])); - const held = await outcomeWithin(framework.rerunUnstick(change as never, { step: 2 }), 6_000); + const started = Date.now(); + const held = await outcomeWithin(framework.rerunUnstick(change as never, { step: 2 }), 10_000); assert.deepEqual(held, { step: 2, status: 'completed', outcome: 'responded' }, 'held, then run: not failed by the hold'); + assert.ok(Date.now() - started >= 900, 'it waited out the hold before it ran'); + assert.equal(holds, -1, 'the host was asked to hold, then asked again when the hold ended, and released it'); } finally { undoSetup(); } @@ -2235,8 +2254,11 @@ describe('operator-change gate: host/command unstick, planned at staging', () => await framework.runUntilIdle(); assert.equal(framework.getUnstickOperation(change.id)!.attempts[0]!.status, 'completed', 'the outcome is recorded'); const duplicate = framework.rerunUnstick(change as never, { step: 1 }); - const early = await Promise.race([duplicate.then(() => 'resolved'), new Promise((r) => setTimeout(() => r('waiting'), 50))]); - assert.equal(early, 'waiting', 'not before settlement'); + let duplicateSettled = false; + void duplicate.then(() => { duplicateSettled = true; }); + await new Promise((resolve) => setImmediate(resolve)); + assert.deepEqual([...host().unstickAttemptWaiters.keys()], [`${change.id}#1`], 'the duplicate joined the first caller\'s waiter'); + assert.equal(duplicateSettled, false, 'not before settlement'); fw.releaseUnstickAttempt = realRelease; realRelease(held); assert.deepEqual(await duplicate, await first); diff --git a/test/safe-boundary-lease.test.ts b/test/safe-boundary-lease.test.ts index b128b7823..698f88078 100644 --- a/test/safe-boundary-lease.test.ts +++ b/test/safe-boundary-lease.test.ts @@ -184,6 +184,35 @@ describe('runAtSafeBoundary', () => { await framework.runUntilIdle(); }); + it('refuses a request whose signal was already aborted, granting and queueing nothing', async () => { + const controller = new AbortController(); + controller.abort(); + let ran = false; + await assert.rejects( + framework.runAtSafeBoundary({ verb: 'too late', signal: controller.signal }, async () => { ran = true; }), + { name: 'AbortError' }, + ); + assert.deepEqual([ran, i.boundaryWaiters.length, i.heldLease], [false, 0, null]); + }); + + it('lets a creation parked behind a waiting lease proceed once that lease is withdrawn', async () => { + i.activeTurnTokens.set('other', 9_999); // the lease waits for other's turn + const controller = new AbortController(); + const leased = framework.runAtSafeBoundary({ verb: 'withdrawn', signal: controller.signal }, async () => {}); + await tick(); + let created = false; + const creating = framework.createEphemeralAgent({ name: 'parked', model: 'test-model', systemPrompt: 'test' }) + .then((c) => { created = true; return c; }); + await tick(); + assert.equal(created, false, 'parked behind the waiting lease'); + controller.abort(); + await assert.rejects(leased, { name: 'AbortError' }); + const c = await withinSeconds(creating); + assert.equal(created, true, 'woken by the withdrawal'); + c.cleanup(); + i.activeTurnTokens.delete('other'); + }); + it('runs a granted callback to its end whatever the signal does after the grant', async () => { const controller = new AbortController(); let finished = false; @@ -237,6 +266,13 @@ describe('runAtSafeBoundary', () => { framework.puppetToolCall('scout', 'agent_settings', { action: 'get' }, { lease: kept! }), /not the lease currently held/, ); + // Nor while another lease is held: its tokens are that lease's alone. + await framework.runAtSafeBoundary({ verb: 'another' }, async () => { + await assert.rejects( + framework.puppetToolCall('scout', 'agent_settings', { action: 'get' }, { lease: kept! }), + /not the lease currently held/, + ); + }); }); it('lands writes deferred behind it at release, with their ids, and acknowledges them', async () => { diff --git a/test/subconscious-module-tools.test.ts b/test/subconscious-module-tools.test.ts index fbea8f724..9f970a021 100644 --- a/test/subconscious-module-tools.test.ts +++ b/test/subconscious-module-tools.test.ts @@ -90,7 +90,8 @@ describe('Module.getSubconsciousTools', () => { // queued tool-call event (the event loop delivers it to the module). framework.start(); (framework as unknown as Internals).dispatchToolCall(sub, { id: 'toolu_sub_1', name: 'body--answer', input: {} }); - for (let n = 0; n < 50 && answers.calls.length === 0; n++) await new Promise((r) => setTimeout(r, 5)); + // The event loop delivers it: bounded generously, for a loaded machine. + for (const deadline = Date.now() + 10_000; answers.calls.length === 0 && Date.now() < deadline;) await new Promise((r) => setTimeout(r, 5)); assert.deepEqual(answers.calls.map((c) => ({ name: c.name, caller: c.callerAgentName, origin: c.origin })), [ { name: 'answer', caller: sub, origin: undefined }, ]); diff --git a/test/tool-call-origin.test.ts b/test/tool-call-origin.test.ts index 8918ecf2c..66adc9d5a 100644 --- a/test/tool-call-origin.test.ts +++ b/test/tool-call-origin.test.ts @@ -346,12 +346,15 @@ describe("tool-call origin: the agent's own, and nothing else, skips the gate", ` if "${name}" in globals(): break`, ` await asyncio.sleep(0.01)`, ]; - // The puppet's script leaves a task that calls after its exec has ended. + // The puppet's script leaves a task that calls after its exec has ended: + // it waits for the next script to let it go, so it's still running when + // this one ends, however slowly that comes. const first = await framework.puppetToolCall('scout', 'code_execution', { code: [ 'import asyncio', 'async def _after():', - ' await asyncio.sleep(0.3)', + ' while not globals().get("release_late1"):', + ' await asyncio.sleep(0.01)', ' globals()["late1"] = await agent_settings({"action": "update", "context_budget_tokens": 134000})', 'asyncio.ensure_future(_after())', 'print("scheduled")', @@ -368,6 +371,7 @@ describe("tool-call origin: the agent's own, and nothing else, skips the gate", const second = await framework.puppetToolCall('scout', 'code_execution', { code: [ 'import asyncio', + 'release_late1 = True', ...settle('late1'), 'print(late1)', 'async def _during():', From e37235dded315360a8378028abb1d50a32566e96 Mon Sep 17 00:00:00 2001 From: Agnes-1009 Date: Sun, 11 Oct 2026 16:19:41 -0400 Subject: [PATCH 73/73] fix(operator-changes): a fork parked behind a hold wakes when the hold clears, whatever leases wait Ada-1017's review of the previous three commits found that the conversation fork's admission parked and woke on different conditions. It parked while a store hold was set, but it woke through the ephemeral creations' wake (notifyBoundaryCleared), which stays asleep while any lease waits. So a fork could park behind a hold, and then stay parked after the hold cleared, if a lease had arrived meanwhile. The event loop would be blocked in that fork's creation, and the loop's scheduler pass is one of the places a waiting lease is granted. Today nothing becomes busy across a hold, so the release's own grant gets there first, but that's an invariant the fork's rule shouldn't rest on. - **Their own wake:** conversation creations park on their own waiter list. It's woken whenever a hold clears (the reservation's release) and when admission closes (stop()), whatever leases wait. A woken fork takes its creation ticket, so a waiting lease waits for it, or it parks again behind a newer hold. Park and wake now test the same thing. - **The teardown's check:** an ending stream's teardown asks whether another turn carries an unstick re-run on, by comparing the agent's turn token with its own. It now passes its token as given, even when undefined, so a stream that somehow had no token can't count its own trigger and leave the re-run unsettled. Every re-run comes through a turn with a token today; this keeps it so if that changes. - **A doc:** flushDeferredMessageOps says why it runs after the release's deferred-write flush. A coalesced replacement held across the hold then lands its new message before the old one is removed, which ends as it would have with nothing held. **Tests** (conversation-routing): - **New:** a hold, a fork parked behind it, a lease that arrives, and an agent busy when the hold clears, so the lease is still waiting then. The fork proceeds, and the lease is granted after it. - **The fork tests from before** (behind a held lease, and refused at stop()) asserted "no fork yet" after a fixed 50 ms sleep. Under load the channel event might not have been processed by then, and the assertion would pass for that reason. They now wait, with a bound, until a creation is parked behind the hold, then assert. Ada-1017's review found this too. **The check, from a clean worktree of this commit's tree as AF's CI runs it (npm install; node 24.19.0, npm 11.17.0):** - **The suite:** `tsc --noEmit` and the build are clean. `npm test` reported 1557 of 1557 passing, across 140 suites, with the new test among them. The same five tests at the end of channel-registry-routing.test.ts went unreported, with none failing, as at the commits before. - **The removal:** with the fork parking on the ephemeral creations' wake again, the new test fails. Its fork is seen parked, then never proceeds. Co-Authored-By: Agnes (Claude Opus 5.5) --- src/framework.ts | 41 ++++++++++++++++++----- test/conversation-routing.test.ts | 55 +++++++++++++++++++++++++++++-- 2 files changed, 86 insertions(+), 10 deletions(-) diff --git a/src/framework.ts b/src/framework.ts index ef6506877..51713625f 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -1815,6 +1815,10 @@ export class AgentFramework { /** Admissions applyResolvedOperatorChange is carrying out, by admission id: * each lets exactly its change's tool call through the gate. */ private activeAdmissions: Map = new Map(); + /** Conversation forks parked behind a store hold (admitConversationCreation), + * woken whenever a hold clears and when admission closes, whatever leases + * wait. */ + private conversationCreationWaiters: Array<() => void> = []; /** Module edits and removals held while a reservation holds the store, * applied at its release (flushDeferredMessageOps). */ private deferredMessageOps: Array<{ op: 'edit'; id: MessageId; content: ContentBlock[] } | { op: 'remove'; id: MessageId }> = []; @@ -6895,6 +6899,7 @@ export class AgentFramework { queueMicrotask(() => { this.tryGrantSafeBoundary(); this.notifyBoundaryCleared(); + this.wakeConversationCreations(); }); for (const [name, token] of reserved) { if (this.activeTurnTokens.get(name) === token) this.activeTurnTokens.delete(name); @@ -6931,7 +6936,10 @@ export class AgentFramework { /** Apply the module edits and removals held while a reservation held the * store (editMessage, removeMessage). One that no longer applies, such as - * an edit of a message the holder's change removed, is logged. */ + * an edit of a message the holder's change removed, is logged. Runs after + * the release's deferred-write flush, and must stay after it: a coalesced + * replacement held across the hold then lands its new message before the + * old one is removed, which ends as it would have with nothing held. */ private flushDeferredMessageOps(): void { // Optional chaining: prototype-built harnesses leave fields undefined. for (const op of this.deferredMessageOps?.splice(0) ?? []) { @@ -6952,8 +6960,10 @@ export class AgentFramework { this.stopping = true; for (const waiter of this.boundaryWaiters?.splice(0) ?? []) waiter.refuse(stoppingRefusal(waiter.verb)); // Creations parked behind a lease or surgery wake now and are refused - // (tryAdmitEphemeralCreation): none enters the store once stopping. + // (tryAdmitEphemeralCreation, admitConversationCreation): none enters + // the store once stopping. for (const resume of this.boundaryClearedWaiters?.splice(0) ?? []) resume(); + this.wakeConversationCreations(); } /** @@ -7115,10 +7125,19 @@ export class AgentFramework { this.ephemeralPending.add(ticket); return ticket; } - await new Promise((resolve) => this.boundaryClearedWaiters.push(resolve)); + // Woken when the hold clears, whether or not a lease waits by then + // (unlike the ephemeral creations' wake, notifyBoundaryCleared): a + // fork that waited for a waiting lease would wait for the loop it's + // blocking. Woken, it takes its ticket, or parks behind a newer hold. + await new Promise((resolve) => (this.conversationCreationWaiters ??= []).push(resolve)); } } + /** Wake the conversation forks parked behind a store hold. */ + private wakeConversationCreations(): void { + for (const resume of this.conversationCreationWaiters?.splice(0) ?? []) resume(); + } + /** Admit a creation now, returning its pending ticket, or null when it has * to wait. The store hold (surgeryHold) covers a held lease and a direct * rollback or suppress alike: nothing initializes under either. */ @@ -12644,12 +12663,18 @@ export class AgentFramework { * queued request (a requeue, or a budget restart), or a request a provider * hold keeps. Another turn carries it only if it is bound to the same * operation and step. */ - private unstickRunCarried(agentName: string, binding: { operationId: string; step: number }, endingToken?: number): boolean { + private unstickRunCarried( + agentName: string, + binding: { operationId: string; step: number }, + ending?: { token: number | undefined }, + ): boolean { const carries = (r: InferenceRequest | undefined) => r?.unstick?.operationId === binding.operationId && r.unstick.step === binding.step; - // A stream that is ending still holds the agent's turn until its own - // teardown: only a successor's turn (another token) carries it on. - const live = endingToken === undefined || this.activeTurnTokens.get(agentName) !== endingToken; + // From an ending stream's teardown (`ending`), the agent's turn is still + // that stream's own until the teardown clears it: only a successor's + // turn, under another token, carries the re-run on. Compared as given, + // so a stream that never had a token can't count its own trigger. + const live = ending === undefined || this.activeTurnTokens.get(agentName) !== ending.token; return (live && carries(this.activeTurnTriggers.get(agentName))) || this.pendingRequests.some((r) => r.agentName === agentName && carries(r)) || (this.providerAccelerationCooldowns.get(agentName)?.heldRequests.some(carries) ?? false); @@ -14387,7 +14412,7 @@ export class AgentFramework { // An unstick re-run that ended without an outcome (aborted, cancelled) // failed, unless another turn carries it on: a retry that started its // stream, a requeue or budget restart, or a provider hold. - if (trigger?.unstick && !this.unstickRunCarried(agent.name, trigger.unstick, myTurnToken)) { + if (trigger?.unstick && !this.unstickRunCarried(agent.name, trigger.unstick, { token: myTurnToken })) { this.settleUnstickAttempt(trigger.unstick, 'failed', { error: 'the re-run ended without a response' }); } const unsettledKvSubmissions = drainKvSubmissionIds?.() ?? []; diff --git a/test/conversation-routing.test.ts b/test/conversation-routing.test.ts index de56f93d2..44a5c6d4c 100644 --- a/test/conversation-routing.test.ts +++ b/test/conversation-routing.test.ts @@ -178,6 +178,27 @@ describe('Conversation routing', () => { await framework.stop(); }); + /** Wait, with a bound, until something is parked behind a store hold: a + * fork's creation, so an assertion that no fork exists yet means it + * waited, not that its event hadn't been processed. */ + async function parked(framework: AgentFramework) { + const waiting = () => { + const f = framework as unknown as { conversationCreationWaiters?: unknown[]; boundaryClearedWaiters?: unknown[] }; + return (f.conversationCreationWaiters?.length ?? 0) + (f.boundaryClearedWaiters?.length ?? 0); + }; + for (const deadline = Date.now() + 10_000; waiting() === 0;) { + if (Date.now() > deadline) throw new Error('no creation parked behind the hold'); + await new Promise((resolve) => setTimeout(resolve, 5)); + } + } + /** Wait, with a bound, until `ready` holds. */ + async function until(ready: () => boolean, what: string) { + for (const deadline = Date.now() + 10_000; !ready();) { + if (Date.now() > deadline) throw new Error(`not within 10 s: ${what}`); + await new Promise((resolve) => setTimeout(resolve, 5)); + } + } + it('opens a fork only once a held lease releases, so its holder never sees the store change under it', async () => { membrane.pushResponse(createMockResponse([{ type: 'text', text: 'Hi alice!' }])); const framework = await makeFramework(); @@ -189,7 +210,7 @@ describe('Conversation routing', () => { await granted; framework.pushEvent(incomingEvent({ channelId: 'slack:D1', text: 'hello there', channelType: 'im' })); const routing = framework.runUntilIdle(); - await new Promise((resolve) => setTimeout(resolve, 50)); + await parked(framework); assert.equal(framework.getAgent('conversation-slack-D1-g1'), null, 'no fork while the lease is held'); release!(); await holding; @@ -202,6 +223,36 @@ describe('Conversation routing', () => { } }); + it('wakes a fork parked behind a hold when the hold clears, though a lease waits by then; the lease waits for the fork', async () => { + membrane.pushResponse(createMockResponse([{ type: 'text', text: 'Hi alice!' }])); + const framework = await makeFramework(); + const internals = framework as unknown as { + reserveStoreForSurgery(verb: string, agentName: string): () => void; + activeTurnTokens: Map; + }; + try { + const releaseHold = internals.reserveStoreForSurgery('rollback', 'trunk'); + framework.pushEvent(incomingEvent({ channelId: 'slack:D1', text: 'hello there', channelType: 'im' })); + const routing = framework.runUntilIdle(); + await parked(framework); + assert.equal(framework.getAgent('conversation-slack-D1-g1'), null, 'parked behind the hold'); + // A lease arrives, and when the hold clears an agent is busy, so the + // lease is still waiting then. + let leaseRan = false; + const leasing = framework.runAtSafeBoundary({ verb: 'later' }, async () => { leaseRan = true; }); + releaseHold(); + internals.activeTurnTokens.set('trunk', 9_999); + await until(() => framework.getAgent('conversation-slack-D1-g1') !== null, 'the fork proceeds though a lease waits'); + assert.equal(leaseRan, false, 'the lease still waits'); + internals.activeTurnTokens.delete('trunk'); + await leasing; + assert.equal(leaseRan, true, 'granted after the fork'); + await routing; + } finally { + await framework.stop(); + } + }); + it('refuses a fork waiting behind a lease once stop() begins, before it enters the store', async () => { const framework = await makeFramework(); const quiet = console.error; @@ -214,7 +265,7 @@ describe('Conversation routing', () => { await granted; framework.pushEvent(incomingEvent({ channelId: 'slack:D1', text: 'hello there', channelType: 'im' })); const routing = framework.runUntilIdle().catch(() => {}); - await new Promise((resolve) => setTimeout(resolve, 50)); + await parked(framework); const stopping = framework.stop(); release(); await holding;