diff --git a/changelog.d/fix-surgery-marks-receipt.fixed.md b/changelog.d/fix-surgery-marks-receipt.fixed.md new file mode 100644 index 000000000..7aeb18faf --- /dev/null +++ b/changelog.d/fix-surgery-marks-receipt.fixed.md @@ -0,0 +1,49 @@ +- Live `rollbackToMessage` and `suppressMessages` (and message-granular + `host/command` undo, which rides on rollback) return once the body change + has landed, with a marker-scheduling receipt, instead of after every + awareness reaction (๐Ÿ’ค by default) has been delivered. Previously the store + stayed reserved, and the operator got no answer, for the whole serial + drain. At the reported pace of about one reaction per second, 918 queued + marks would take roughly 15 minutes; the household's client timed out + after 60 seconds. Delivery now continues in the background, behind the + same MCPL data-plane gate as before, which holds channel traffic that + arrives after it until the marks are attempted, or deferred when their + server isn't connected. A turn may now run during delivery when it carries + no channel traffic, such as a scheduler timer or a provider admission, and + also when its channel traffic was admitted before the gate went up (queued + when the surgery began, or arriving during its switch or fork). Such a + turn used to wait for the whole drain, since the store stayed reserved; + now it can answer before the marks show. +- The other branch moves no longer wait for Discord either: the web API's + `branch.switch`, `undo` and `redo`, and `host/command` turn undo, return + once the branch has moved, and their marks follow behind the same gate. + Each starts its delivery through + `startDiscordAwarenessDelivery(after, agentName?)`, a new public method + that a host can call after its own branch moves. A delivery that fails to + start raises a `discord-awareness-delivery` ops alert instead of failing + the command, and the next sync, reconnect or start delivers what is still + pending. (The web switch used to await the awareness sync before + answering.) +- A gate installed while a drain is still running (restoring the source + branch mid-delivery, say) waits for that drain and then drains again from + the ledger. So before traffic is released, every mark pending when the + gate was armed is attempted, or deferred until its server is connected, + as before: the drain's remaining reactions, the removals they need, and an + add re-pended during the drain (a rollback, its undo, then a redo). + Inbound MCPL handlers wait out a generation installed while they wait, not + only the one they first saw, and a reconnect's gate opens only once both + its drain and the server's re-registration attempt have settled. So a + reconnect's `mcpl:server-reconnected` and `module:added` traces now come + once that attempt has settled, granted or not, where they came at once + when no marks were pending. +- The result's new `markers` field (`none` | `queued` | `not-scheduled` | + `unresolved`, the exported `SurgeryMarkerReceipt` type) says whether marks + were scheduled, never that Discord accepted them. The `[operator]` stderr + line gains `marks=โ€ฆ` and the operator-actions log entry gains `markers`. + If the ledger write after the switch fails, the rollback that already + applied is no longer reported as failed: the batch is retired + (`not-scheduled`, with a `discord-awareness-not-scheduled` ops alert), or, + if even that write fails, the result says the batch may still be + delivered (`unresolved`, with a `discord-awareness-unresolved` ops alert). A + suppression whose redactions all committed is no longer undone by that + failure. diff --git a/src/api/server.ts b/src/api/server.ts index 58664705b..21e648915 100644 --- a/src/api/server.ts +++ b/src/api/server.ts @@ -543,8 +543,13 @@ export class ApiServer { this.currentBranch = params.name; // Reversible Discord awareness markers are part of the branch projection. - // Do not acknowledge the switch until add/remove operations were attempted. - await this.framework.syncDiscordAwarenessMarkers(); + // The switch is acknowledged once it has landed: delivery of the marks it + // adds and removes starts behind the MCPL data-plane gate, which holds + // channel traffic that arrives after it until they are attempted, or + // deferred while their server is disconnected, and a failure is an ops + // alert rather than a failed switch. (It used to await the awareness + // sync, which after a large rollback took the drain's whole length.) + this.framework.startDiscordAwarenessDelivery('branch switch'); // Materialize config files from the new branch const ws = this.framework.getModule('workspace'); @@ -598,7 +603,8 @@ export class ApiServer { if (result.undone) { this.currentBranch = result.toBranch!; - await this.framework.syncDiscordAwarenessMarkers(); + // Marks follow behind the data-plane gate; the undo doesn't wait for Discord. + this.framework.startDiscordAwarenessDelivery('undo', params.agentName); this.broadcast('branch:switched', { from: result.fromBranch, to: result.toBranch, @@ -618,7 +624,8 @@ export class ApiServer { if (result.redone) { this.currentBranch = result.toBranch!; - await this.framework.syncDiscordAwarenessMarkers(); + // Marks follow behind the data-plane gate; the redo doesn't wait for Discord. + this.framework.startDiscordAwarenessDelivery('redo', params.agentName); this.broadcast('branch:switched', { from: result.fromBranch, to: result.toBranch, diff --git a/src/framework.ts b/src/framework.ts index dbb2e4033..118b3c64e 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -410,12 +410,14 @@ import { DiscordAwarenessOutbox, defaultDiscordAwarenessOutboxPath, extractDiscordAwarenessRefs, + type DiscordAwarenessBatch, } from './recovery/discord-awareness-outbox.js'; import { OperatorActionError, OperatorLog, capIds, defaultOperatorLogPath, + type SurgeryMarkerReceipt, type OperatorLogEntry, type OperatorLogInput, type OperatorRequester, @@ -606,6 +608,21 @@ interface DiscordAwarenessBarrier { promise: Promise; } +/** Several drains' outcomes as one: unavailable only if every one was. */ +function combineDrainOutcomes(outcomes: DiscordAwarenessDrainOutcome[]): DiscordAwarenessDrainOutcome { + if (outcomes.every((outcome) => outcome.status === 'unavailable')) { + return { + status: 'unavailable', + accounted: outcomes.reduce((count, outcome) => count + (outcome.status === 'unavailable' ? outcome.accounted : 0), 0), + }; + } + return { + status: 'delivered', + delivered: outcomes.reduce((count, outcome) => count + (outcome.status === 'delivered' ? outcome.delivered : 0), 0), + failed: outcomes.reduce((count, outcome) => count + (outcome.status === 'delivered' ? outcome.failed : 0), 0), + }; +} + function normalizeDiscordAwarenessDeadline(value: number | undefined): number { if (value === undefined || !Number.isFinite(value) || value <= 0) { return DEFAULT_DISCORD_AWARENESS_DEADLINE_MS; @@ -812,6 +829,13 @@ function bodyGroupRun( }; } +function describeMarkers(m: SurgeryMarkerReceipt): string { + if (m.status === 'queued') return `queued:${m.queued}`; + if (m.status === 'not-scheduled') return 'not-scheduled(batch retired)'; + if (m.status === 'unresolved') return `unresolved(batch ${m.batchId} may still be delivered)`; + return 'none'; +} + function describeRequester(r: OperatorRequester | undefined): string { if (!r) return 'unknown'; return `${r.name ?? r.id ?? 'unknown'} (${r.via})`; @@ -5409,6 +5433,9 @@ export class AgentFramework { /** Discord addresses removed by message-granular undo. The durable outbox * owns eventual delivery; this is also returned for immediate surfaces. */ removedRefs?: Array<{ serverId: string; channelId: string; messageId: string }>; + /** For message-granular undo: whether awareness marks were scheduled + * (not whether Discord has accepted them). */ + markers?: SurgeryMarkerReceipt; hidden?: number; /** For `hide`: the Discord (channelId, messageId) of each removed message * that carried one โ€” so the surface can mark them with a reaction. */ @@ -5652,6 +5679,7 @@ export class AgentFramework { ok: true, messagesRemoved: r.messagesRemoved, removedRefs: r.removedRefs, + markers: r.markers, lastVisible: r.lastVisible, }; } catch (error) { @@ -5688,7 +5716,9 @@ export class AgentFramework { return { ok: true, undone: 0, requested, lastVisible: null }; } - await this.syncDiscordAwarenessMarkers(); + // The branch has moved: delivery of the marks it changes starts behind + // the data-plane gate, and the command doesn't wait for Discord. + this.startDiscordAwarenessDelivery('turn undo', agentName); return { ok: true, undone, requested, lastVisible: await this.lastVisiblePreview(agentName) }; } @@ -6283,6 +6313,12 @@ export class AgentFramework { * Discord messages that left the live context get awareness markers via the * durable outbox, exactly as message-granular `undo` does. * + * Returns once the switch has landed: the body is applied, with a + * marker-scheduling receipt (`markers`). It does not wait for Discord to + * accept the reactions: delivery is started, behind the MCPL data-plane + * gate, and continues without the caller or the store reservation waiting + * on it; its per-message outcomes stay in the outbox ledger. + * * Throws `OperatorActionError` (`agent-busy`, `unknown-message`, โ€ฆ) โ€” the * agent must be idle; nothing is queued. */ @@ -6305,6 +6341,8 @@ export class AgentFramework { * the group's last shard so the sharded message stays whole. */ tailMessageId: string; removedRefs: Array<{ serverId: string; channelId: string; messageId: string }>; + /** Whether awareness marks were scheduled; never a delivery claim. */ + markers: SurgeryMarkerReceipt; lastVisible: { participant?: string; role?: string; preview?: string } | null; }> { const logBase: OperatorLogInput = { @@ -6342,9 +6380,12 @@ export class AgentFramework { } // Gate + reserve the whole store (see reserveStoreForSurgery); held - // until the switch has landed or been rolled back. + // until the switch has landed and its marker scheduling has been + // recorded or reported, or the switch has been rolled back. Remote + // delivery is not waited on. const release = this.reserveStoreForSurgery('roll back', agentName); let branchName: string; + let markers: SurgeryMarkerReceipt; try { // Prepare the external side effect before switching Chronicle. If the // process dies after the switch but before activate(), startup @@ -6375,8 +6416,14 @@ export class AgentFramework { { cause: error }, ); } - if (markerBatch) this.discordAwarenessOutbox!.activate(markerBatch.id); - await this.syncDiscordAwarenessMarkers(); + // The body change has landed. Nothing after this point turns the + // rollback into a failure: marks are an outbox outcome, reported + // apart from the body (`markers`). + markers = this.activateSurgeryMarkers(markerBatch, 'rollback', agentName); + // Installs the MCPL data-plane gate synchronously, before the store is + // released, and starts delivery without awaiting it (the first + // reaction request may already be on the wire when this returns). + this.startDiscordAwarenessDelivery('rollback', agentName); this.materializeConfigMountAfterBranchSwitch(); } finally { release(); @@ -6385,7 +6432,8 @@ export class AgentFramework { console.error( `[operator] rollback agent=${agentName} to=${tailMessageId}` + `${tailMessageId !== opts.messageId ? ` (requested ${opts.messageId}, snapped to end of body group)` : ''}` + - ` removed=${messagesRemoved} branch=${branchName} by=${describeRequester(opts.requester)}`, + ` removed=${messagesRemoved} branch=${branchName} marks=${describeMarkers(markers)}` + + ` by=${describeRequester(opts.requester)}`, ); this.recordOperatorAction({ ...logBase, @@ -6395,6 +6443,7 @@ export class AgentFramework { tailMessageId, messagesRemoved, discordRefs: removedRefs.length, + markers, }, }); return { @@ -6404,6 +6453,7 @@ export class AgentFramework { messagesRemoved, tailMessageId, removedRefs, + markers, lastVisible: await this.lastVisiblePreview(agentName), }; } catch (error) { @@ -6419,6 +6469,9 @@ export class AgentFramework { * removed together. Discord originals get awareness markers via the outbox; * the batch is `explicit`-activated only after every removal succeeded, and * a crash mid-way is finished at next boot (resumePreparedDiscordSuppressions). + * Like rollbackToMessage, it returns once the body change has landed, with + * a marker-scheduling receipt (`markers`); delivery continues without the + * caller waiting on it. * * Not retroactive over derived state: a message already folded into an * autobiographical summary stays in that summary โ€” roll back to before it @@ -6440,6 +6493,8 @@ export class AgentFramework { messagesRemoved: number; removedIds: string[]; removedRefs: Array<{ serverId: string; channelId: string; messageId: string }>; + /** Whether awareness marks were scheduled; never a delivery claim. */ + markers: SurgeryMarkerReceipt; lastVisible: { participant?: string; role?: string; preview?: string } | null; }> { const requestedIds = [...new Set(opts.messageIds.map(String))]; @@ -6492,9 +6547,12 @@ export class AgentFramework { throw new OperatorActionError('invalid', 'Suppression branch name must differ from the active branch'); } // Gate + reserve the whole store (see reserveStoreForSurgery); held - // until the fork is fully redacted or the source is restored. + // until the fork is fully redacted and its marker scheduling has been + // recorded or reported, or the source is restored. Remote delivery is + // not waited on. const release = this.reserveStoreForSurgery('suppress', agentName); let createdBranch: string; + let markers: SurgeryMarkerReceipt; try { const markerBatch = this.discordAwarenessOutbox?.prepare({ agentName, @@ -6535,7 +6593,6 @@ export class AgentFramework { if (iv.group || iv.fromId !== iv.toId) cm.removeMessages(iv.fromId as MessageId, iv.toId as MessageId); else cm.removeMessage(iv.fromId as MessageId); } - if (markerBatch) this.discordAwarenessOutbox!.activate(markerBatch.id); } catch (error) { // A partially suppressed branch is not safe to serve from. Keep it // for diagnosis, put the agent back on the untouched source โ€” and @@ -6548,7 +6605,19 @@ export class AgentFramework { { cause: error }, ); } - await this.syncDiscordAwarenessMarkers(); + // Every redaction committed: the body change has landed and the fork + // is safe to serve from. Activating the marker batch is bookkeeping + // about a world-side effect; if it fails, the suppression still + // stands, and the receipt says what became of the marks: not + // scheduled once the batch is retired, or unresolved if it can't be. + // (The batch's resume journal has nothing left to do, since every + // interval has committed, so retiring the batch gives up only the + // marks.) + markers = this.activateSurgeryMarkers(markerBatch, 'suppress', agentName); + // Installs the MCPL data-plane gate synchronously, before the store is + // released, and starts delivery without awaiting it (the first + // reaction request may already be on the wire when this returns). + this.startDiscordAwarenessDelivery('suppress', agentName); this.materializeConfigMountAfterBranchSwitch(); } finally { release(); @@ -6556,7 +6625,7 @@ export class AgentFramework { console.error( `[operator] suppress agent=${agentName} removed=${messagesRemoved} branch=${createdBranch} ` + - `by=${describeRequester(opts.requester)}`, + `marks=${describeMarkers(markers)} by=${describeRequester(opts.requester)}`, ); const loggedIds = capIds(removedIds); this.recordOperatorAction({ @@ -6568,6 +6637,7 @@ export class AgentFramework { removedIds: loggedIds.ids, ...(loggedIds.truncated ? { removedIdsTruncated: true } : {}), discordRefs: removedRefs.length, + markers, }, }); return { @@ -6577,6 +6647,7 @@ export class AgentFramework { messagesRemoved, removedIds, removedRefs, + markers, lastVisible: await this.lastVisiblePreview(agentName), }; } catch (error) { @@ -6585,6 +6656,108 @@ export class AgentFramework { } } + /** + * Record a surgery's prepared marker batch as active once its body change + * has landed, and say what happened. Never throws: the body is already + * applied, so a ledger failure here is reported in the receipt rather than + * as a failed surgery an operator might retry. A batch that could not be + * activated is retired, so no later reconciliation can deliver it behind a + * receipt saying it was not scheduled (`not-scheduled`). If retiring fails + * too, the receipt says only that the outcome is `unresolved`: the batch is + * retained, and it may still be delivered. A rollback's batch + * (activated by target branch) may be promoted by the very next + * reconciliation, the one the delivery it starts runs; a suppression's + * (activated explicitly) waits for the next start. + * + * A batch with no refs (a suppression journal with nothing addressable) is + * still activated, which closes its resume journal. It schedules no marks, + * so it reports `none` whatever the ledger does. If it can be neither + * activated nor retired, its journal is left prepared and closes at the + * next start: every interval it names has already committed, so the resume + * skips them all and activates it. + */ + private activateSurgeryMarkers( + batch: DiscordAwarenessBatch | null, + verb: 'rollback' | 'suppress', + agentName: string, + ): SurgeryMarkerReceipt { + if (!batch) return { status: 'none', queued: 0 }; + try { + this.discordAwarenessOutbox!.activate(batch.id); + } catch (error) { + const detail = error instanceof Error ? error.message : String(error); + let retired = false; + let retireDetail = ''; + try { + retired = this.discordAwarenessOutbox!.discard(batch.id); + } catch (discardError) { + retireDetail = discardError instanceof Error ? discardError.message : String(discardError); + } + console.error( + `[discord-awareness] ${verb} agent=${agentName}: body applied, but marker batch ${batch.id} ` + + `could not be activated (${detail}); ` + + (retired ? 'batch retired' : `batch retained${retireDetail ? ` (retire failed: ${retireDetail})` : ''}`), + ); + if (batch.refs.length === 0) return { status: 'none', queued: 0 }; + this.opsAlert( + retired ? 'discord-awareness-not-scheduled' : 'discord-awareness-unresolved', + agentName, + retired + ? `${verb} applied, but its ${batch.refs.length} awareness mark(s) were not scheduled ` + + `(batch ${batch.id} retired): ${detail}` + : `${verb} applied, but awareness batch ${batch.id} (${batch.refs.length} mark(s)) could be ` + + `neither activated nor retired; it may still be delivered: ${detail}`, + { data: { batchId: batch.id, retired } }, + ); + return retired + ? { status: 'not-scheduled', queued: 0, error: detail } + : { status: 'unresolved', queued: 0, batchId: batch.id, error: detail }; + } + if (batch.refs.length === 0) return { status: 'none', queued: 0 }; + return { status: 'queued', queued: batch.refs.length, batchId: batch.id }; + } + + /** + * Start awareness delivery after a branch move (a surgery, a switch, an + * undo or redo) without the caller awaiting it. No operator response waits + * for Discord to accept marks: the MCPL data-plane gate alone holds channel + * traffic that arrives after it until they are attempted. + * `syncDiscordAwarenessMarkers` installs that gate synchronously, before + * its first await, so calling this while a surgery still holds the store + * puts the gate up before the store is released. With no drain running, the + * drain starts in the same synchronous run (the first reaction request can + * be written before this returns); with one running, this generation's pass + * follows it (drainForGeneration). Either way delivery continues + * independently: the caller's response and any store reservation stop + * waiting on remote reactions. A turn may run during delivery when its wake + * carries no channel traffic (a scheduler timer, a provider admission), and + * also when its channel traffic was admitted before the gate went up + * (queued when a surgery began, or arriving during its switch or fork): the + * surgery's store reservation, which used to hold such a turn through the + * drain, now ends once the gate is up. A failure is accounted by the gate + * itself: the current generation's failure recycles the connections, and a + * superseded one leaves that to the newer generation, whose own pass meets + * the same ledger. Here it is logged and raised as an ops alert, and the + * next sync, reconnect or start delivers what is still pending. + * + * @param after What moved the branch, for the log and the alert + * ("rollback", "branch switch", "undo" โ€ฆ). + * @param agentName The agent the move was for, or "framework" for a + * branch switch that names none. + */ + startDiscordAwarenessDelivery(after: string, agentName = 'framework'): void { + if (!this.discordAwarenessOutbox) return; + this.syncDiscordAwarenessMarkers().catch((error) => { + const detail = error instanceof Error ? error.message : String(error); + console.error(`[discord-awareness] delivery after ${after} agent=${agentName} failed: ${detail}`); + this.opsAlert( + 'discord-awareness-delivery', + agentName, + `Awareness mark delivery after ${after} failed: ${detail}`, + ); + }); + } + /** Re-materialize config files from the (new) active branch โ€” fire and * forget; the gate picks changes up via mtime. Mirrors undoLastTurn. */ private materializeConfigMountAfterBranchSwitch(): void { @@ -13887,7 +14060,11 @@ export class AgentFramework { await this.refreshMcplTools(); } - /** Reconcile the durable ledger with Chronicle, then deliver every server's work. */ + /** + * Reconcile the durable ledger with Chronicle, then deliver every server's + * work. A drain still running is waited for and then followed by a fresh + * pass (drainForGeneration). + */ async syncDiscordAwarenessMarkers(_onlyServerId?: string): Promise { if (!this.discordAwarenessOutbox) return; // A targeted retry may discover pending work for other Discord servers @@ -14102,46 +14279,70 @@ export class AgentFramework { throw new DiscordAwarenessAccountingError('branch reconciliation', error); } const pending = this.readDiscordAwarenessPending(); - if (pending.length === 0) { + // A drain that is still running holds operations it read before this + // reconciliation, and it sends them whatever the ledger now says. A branch + // move can hide them from `pending`: restoring a rollback's source makes + // its marks undesired, and an add whose reply has not arrived is not yet + // present, so the ledger shows nothing to do while the drain goes on to + // send its saved adds and then the removals they need. So a generation + // accounts for every running drain as well as for every pending + // operation, and waits for those drains instead of releasing the data + // planes beside them (drainForGeneration). + const serverIds = [...new Set([ + ...pending.map((operation) => operation.ref.serverId), + ...this.discordAwarenessDrains.keys(), + ])]; + if (serverIds.length === 0) { return { requiresBarrier: false, promise: Promise.resolve({ status: 'delivered', delivered: 0, failed: 0 }), }; } - const serverIds = [...new Set(pending.map((operation) => operation.ref.serverId))]; const promise = Promise.all( - serverIds.map((serverId) => this.drainDiscordAwarenessOutbox(serverId)), - ).then((outcomes): DiscordAwarenessDrainOutcome => { - if (outcomes.every((outcome) => outcome.status === 'unavailable')) { - return { - status: 'unavailable', - accounted: outcomes.reduce( - (count, outcome) => count + (outcome.status === 'unavailable' ? outcome.accounted : 0), - 0, - ), - }; - } - return { - status: 'delivered', - delivered: outcomes.reduce( - (count, outcome) => count + (outcome.status === 'delivered' ? outcome.delivered : 0), - 0, - ), - failed: outcomes.reduce( - (count, outcome) => count + (outcome.status === 'delivered' ? outcome.failed : 0), - 0, - ), - }; - }); + serverIds.map((serverId) => this.drainForGeneration(serverId)), + ).then(combineDrainOutcomes); return { requiresBarrier: true, promise }; } + /** + * One server's drain for a new generation: a gate completes only after + * every operation pending when it was armed has been attempted. A drain + * already running can't promise that. It never re-sends a key it has + * attempted, so an operation re-pended while it runs (a rollback, its undo, + * then a redo, all mid-drain) stays pending, and it may already have made + * its last read of the ledger, missing a batch activated since. So a + * generation that finds a drain running waits for it, then drains again + * with a fresh pass that reads the ledger anew. Two generations waiting on + * the same drain share the fresh pass that follows it. + */ + private drainForGeneration(serverId: string): Promise { + const running = this.discordAwarenessDrains.get(serverId); + if (!running) return this.drainDiscordAwarenessOutbox(serverId); + return running.then((first) => this.drainDiscordAwarenessOutbox(serverId) + .then((fresh) => combineDrainOutcomes([first, fresh]))); + } + + /** + * Wait out the awareness gate for an inbound event: the generation current + * now, and any generation installed while waiting, until none is current + * or the current one is one already waited for. A handler that waited only + * for the generation it first saw would go on behind a newer one armed + * meanwhile. Rejects as the generation it waited for rejects. + */ + private async awaitDiscordAwarenessBarriers(): Promise { + let waited: DiscordAwarenessBarrier | null = null; + for (let barrier = this.discordAwarenessBarrier; barrier && barrier !== waited; barrier = this.discordAwarenessBarrier) { + waited = barrier; + await barrier.promise; + } + } + /** * Install one framework-global Discord-awareness generation before releasing - * any inference-bearing event. With pending work, every MCPL data plane is - * paused while all control planes remain live. With no pending work, ready() - * runs synchronously so request responders preserve their historical - * same-stack behavior. + * any inference-bearing event. With pending work or a drain still running, + * every MCPL data plane is paused while all control planes remain live. + * With neither, ready() runs synchronously so request responders preserve + * their historical same-stack behavior. */ private installMcplDataPlaneGate(): DiscordAwarenessBarrier { for (const connection of this.mcplServerRegistry?.getAllServers() ?? []) { @@ -14836,8 +15037,10 @@ export class AgentFramework { ) => { const answer = answerOnce(responder, 'push/event', connection.id); try { - const barrier = this.discordAwarenessBarrier; - if (barrier) await barrier.promise; + // Only when a generation is current: with none, the event is handled in + // the turn it arrives, as it always was (the push coalescer's ordering of + // a push against its retraction depends on that). + if (this.discordAwarenessBarrier) await this.awaitDiscordAwarenessBarriers(); // Checked here, after the barrier and before anything records the // push as accepted: a stopped host refuses it. if (this.queue?.isClosed) { @@ -14867,8 +15070,7 @@ export class AgentFramework { if (this.inferenceRouter && responder) { const answer = answerOnce(responder, 'inference/request', connection.id); try { - const barrier = this.discordAwarenessBarrier; - if (barrier) await barrier.promise; + if (this.discordAwarenessBarrier) await this.awaitDiscordAwarenessBarriers(); // A stopping host starts no inference for a server: refused, like a // push, before the router runs. if (this.queue?.isClosed) { @@ -14983,8 +15185,7 @@ export class AgentFramework { ) => { const answer = answerOnce(responder, 'channels/incoming', connection.id); try { - const barrier = this.discordAwarenessBarrier; - if (barrier) await barrier.promise; + if (this.discordAwarenessBarrier) await this.awaitDiscordAwarenessBarriers(); if (this.queue?.isClosed) { this.refuseServerItem('channels/incoming', connection.id, answer, { kind: 'refused' }); return; @@ -15003,8 +15204,7 @@ export class AgentFramework { if (!responder) return; const answer = answerOnce(responder, 'host/command', connection.id); try { - const barrier = this.discordAwarenessBarrier; - if (barrier) await barrier.promise; + if (this.discordAwarenessBarrier) await this.awaitDiscordAwarenessBarriers(); // A stopping host runs no host command: refused like the requests // above, before its handler runs. if (this.queue?.isClosed) { @@ -15051,7 +15251,7 @@ export class AgentFramework { // grant) before the data-plane barrier releases โ€” otherwise the first // post-reconnect events land while the grant is still empty and are // rejected fail-closed instead of delivered. - void (async () => { + const reregistered = (async () => { try { const config = this.mcplServerConfigs.get(connection.id); if (config) { @@ -15066,7 +15266,11 @@ export class AgentFramework { this.releaseMcplDataPlaneGate(awarenessBarrier); this.handleToolsListChanged(connection.id); })(); - void awarenessBarrier.promise.then(() => { + // The gate completes, and the data planes go ready, only once both the + // awareness drain and the re-registration have finished. A drain can + // settle first (one already running, or little to send), and the grant + // must exist before the first data-plane event flows. + void Promise.all([awarenessBarrier.promise, reregistered]).then(() => { if ( awarenessBarrier.requiresBarrier && !this.completeMcplDataPlaneGate(awarenessBarrier) diff --git a/src/index.ts b/src/index.ts index 643662281..9b6052365 100644 --- a/src/index.ts +++ b/src/index.ts @@ -143,7 +143,12 @@ export type { } from './recovery/discord-awareness-outbox.js'; // Live operator surgery (rollback / suppress) and its durable action log export { OperatorLog, OperatorActionError, defaultOperatorLogPath } from './operator-log.js'; -export type { OperatorLogEntry, OperatorLogInput, OperatorRequester } from './operator-log.js'; +export type { + OperatorLogEntry, + OperatorLogInput, + OperatorRequester, + SurgeryMarkerReceipt, +} from './operator-log.js'; export { createOfflineRecoveryBranch } from './recovery/offline-branch.js'; export type { OfflineRecoveryBranchOptions, diff --git a/src/operator-log.ts b/src/operator-log.ts index 710e6b505..b0d322e73 100644 --- a/src/operator-log.ts +++ b/src/operator-log.ts @@ -118,6 +118,36 @@ export function capIds(ids: string[]): { ids: string[]; count: number; truncated : { ids, count: ids.length }; } +/** + * What a live surgery did about Discord awareness marks, reported apart from + * the body change itself. A surgery that returns has applied its body change; + * this says only whether marks were scheduled: + * + * - `none` โ€” no marks were scheduled: the change removed no addressable + * Discord message, or the awareness outbox is disabled (no + * `discordAwarenessOutboxPath` and no `storePath`). It is not proof that + * no addressable Discord message was removed: the surgery's `removedRefs` + * (`discordRefs` in the operator log) say whether any was. + * - `queued` โ€” the outbox batch is durably active, and its delivery was + * started in the background. If starting it fails (reconciliation reading + * or writing the ledger), a `discord-awareness-delivery` ops alert reports + * the failure, the batch stays durably active, and the next sync, + * reconnect or start delivers it. It is not a delivery claim: per-message + * outcomes stay in the outbox ledger. + * - `not-scheduled` โ€” the body change landed, but recording the batch as + * active failed, so it was retired from the ledger: none of its marks will + * be delivered. + * - `unresolved` โ€” the ledger could record neither the activation nor the + * retirement. `batchId` names a retained batch whose scheduling outcome is + * unresolved: any later reconciliation or restart that can read it may + * promote and deliver it, and nothing here promises otherwise. + */ +export type SurgeryMarkerReceipt = + | { status: 'none'; queued: 0 } + | { status: 'queued'; queued: number; batchId: string } + | { status: 'not-scheduled'; queued: 0; error: string } + | { status: 'unresolved'; queued: 0; batchId: string; error: string }; + /** Thrown by live surgery methods when a request cannot be honored. `code` * lets surfaces distinguish "agent busy โ€” quiesce first" from bad input. */ export class OperatorActionError extends Error { diff --git a/test/discord-awareness-outbox.test.ts b/test/discord-awareness-outbox.test.ts index 4a35a21f5..fb0f3bed1 100644 --- a/test/discord-awareness-outbox.test.ts +++ b/test/discord-awareness-outbox.test.ts @@ -270,6 +270,9 @@ test('message-granular undo prepares markers before switching branches', async ( }); assert.equal(result.ok, true); assert.deepEqual(result.removedRefs.map((ref: { messageId: string }) => ref.messageId), ['m1', 'm2']); + // The command's receipt says the two marks were scheduled, under the batch the ledger holds. + const [batch] = framework.discordAwarenessOutbox.batches(); + assert.deepEqual(result.markers, { status: 'queued', queued: 2, batchId: batch.id }); assert.deepEqual(calls.map((call) => call.split('/')[0]), ['branch:undo-msgs', 'switch:undo-msgs']); assert.deepEqual( framework.discordAwarenessOutbox.pending('discord').map((operation: { ref: { messageId: string } }) => operation.ref.messageId), diff --git a/test/fixtures/awareness-barrier-mcpl-server.mjs b/test/fixtures/awareness-barrier-mcpl-server.mjs index a16b342b3..f8789d22d 100644 --- a/test/fixtures/awareness-barrier-mcpl-server.mjs +++ b/test/fixtures/awareness-barrier-mcpl-server.mjs @@ -14,6 +14,10 @@ const dataMethod = process.env.DATA_METHOD ?? 'push/event'; const serverId = process.env.SERVER_ID ?? 'discord'; const arrivalPath = process.env.ARRIVAL_PATH; const waitForArrivalPath = process.env.WAIT_FOR_ARRIVAL_PATH; +// After a reconnect (generation 2 on), the host's initial policy Request +// (featureSets/update) is answered only once this file exists, so a test can +// hold the server's re-registration open. +const policyHoldPath = process.env.POLICY_HOLD_PATH; let generation = 1; if (generationPath) { @@ -52,6 +56,7 @@ let reactionQueued = false; let pendingControl = false; let listChangeTriggered = false; let reactionOrdinal = 0; +let heldPolicy = null; let buf = ''; function ledgerDeliveryStatus() { @@ -195,6 +200,11 @@ function handle(message) { // host times out (15s), the grant stays empty, and every push/channel // in these tests is rejected fail-closed. if (message.id !== undefined && message.id !== null) { + if (policyHoldPath && generation >= 2 && !existsSync(policyHoldPath)) { + heldPolicy = message; + log('policy-held'); + return; + } reply(message.id, { accepted: true }); } return; @@ -257,6 +267,12 @@ process.stdin.on('data', (chunk) => { }); const timer = setInterval(() => { + if (heldPolicy && policyHoldPath && existsSync(policyHoldPath)) { + const held = heldPolicy; + heldPolicy = null; + log('policy-response'); + reply(held.id, { accepted: true }); + } if (pendingToolRequest && !registered && waitForArrivalPath && existsSync(waitForArrivalPath)) { serviceReaction(pendingToolRequest); maybeFinishRegistration(); diff --git a/test/fixtures/surgery-marks-mcpl-server.mjs b/test/fixtures/surgery-marks-mcpl-server.mjs new file mode 100644 index 000000000..4d2a0a117 --- /dev/null +++ b/test/fixtures/surgery-marks-mcpl-server.mjs @@ -0,0 +1,126 @@ +// Fake Discord MCPL server for test/surgery-marks-receipt.test.ts. +// +// Every tools/call is appended to CALLS_PATH. While HOLD_PATH is set and that +// file does not exist, replies are withheld, standing in for Discord pacing a +// long reaction drain. ANSWER_LIMIT_PATH, when that file exists, caps how many +// calls are answered in all: later calls wait until the number in it grows. +// When PROBE_PATH appears, the server sends one inference-bearing +// channels/incoming request and appends to EVENTS_PATH when the host answers +// it, so a test can see whether the MCPL data plane was held. +import { appendFileSync, existsSync, readFileSync } from 'node:fs'; + +const callsPath = process.env.CALLS_PATH; +const eventsPath = process.env.EVENTS_PATH; +const holdPath = process.env.HOLD_PATH; +const probePath = process.env.PROBE_PATH; +const answerLimitPath = process.env.ANSWER_LIMIT_PATH; +const channelId = process.env.CHANNEL_ID ?? 'discord:g1:c1'; + +const send = (message) => process.stdout.write(JSON.stringify(message) + '\n'); +const reply = (id, result) => send({ jsonrpc: '2.0', id, result }); +const request = (id, method, params) => send({ jsonrpc: '2.0', id, method, params }); +const event = (name, extra = {}) => { + if (eventsPath) appendFileSync(eventsPath, JSON.stringify({ event: name, at: Date.now(), ...extra }) + '\n'); +}; + +const held = []; +let answered = 0; +let probed = false; +let buf = ''; + +function answerLimit() { + if (!answerLimitPath || !existsSync(answerLimitPath)) return Infinity; + const limit = Number(readFileSync(answerLimitPath, 'utf8').trim()); + return Number.isFinite(limit) ? limit : Infinity; +} + +function mayAnswer() { + return (!holdPath || existsSync(holdPath)) && answered < answerLimit(); +} + +function answer(message) { + answered++; + event('reaction-answered', { messageId: message.params?.arguments?.messageId, name: message.params?.name }); + reply(message.id, { content: [{ type: 'text', text: 'Reaction applied' }] }); +} + +function handle(message) { + if (message.method === 'initialize') { + reply(message.id, { + capabilities: { + experimental: { + mcpl: { + version: '0.5', + pushEvents: true, + channels: { register: true, incoming: true }, + featureSets: { chat: { description: 'chat', uses: ['pushEvents'] } }, + }, + }, + }, + }); + return; + } + if (message.method === 'featureSets/update') { + if (message.id !== undefined && message.id !== null) reply(message.id, { accepted: true }); + return; + } + if (message.method === 'notifications/initialized') { + request(100, 'channels/register', { + channels: [{ id: channelId, type: 'discord', label: 'busy', direction: 'bidirectional' }], + }); + return; + } + if (message.method === 'tools/list') { + reply(message.id, { + tools: [ + { name: 'add_reaction', description: 'add', inputSchema: { type: 'object' } }, + { name: 'remove_reaction', description: 'remove', inputSchema: { type: 'object' } }, + ], + }); + return; + } + if (message.method === 'tools/call') { + if (callsPath) { + appendFileSync(callsPath, JSON.stringify({ at: Date.now(), name: message.params?.name, args: message.params?.arguments }) + '\n'); + } + if (held.length > 0 || !mayAnswer()) held.push(message); + else answer(message); + return; + } + if (message.id === 900) { + event('probe-answered', { error: message.error ?? null }); + } +} + +process.stdin.on('data', (chunk) => { + buf += chunk.toString('utf8'); + let newline; + while ((newline = buf.indexOf('\n')) >= 0) { + const line = buf.slice(0, newline); + buf = buf.slice(newline + 1); + if (!line.trim()) continue; + try { + handle(JSON.parse(line)); + } catch (error) { + event('server-error', { message: error instanceof Error ? error.message : String(error) }); + } + } +}); + +const timer = setInterval(() => { + while (held.length > 0 && mayAnswer()) answer(held.shift()); + if (!probed && probePath && existsSync(probePath)) { + probed = true; + event('probe-sent'); + request(900, 'channels/incoming', { + messages: [{ + channelId, + messageId: 'probe-during-delivery', + author: { id: 'bystander', name: 'Bystander' }, + timestamp: new Date().toISOString(), + content: [{ type: 'text', text: 'ordinary traffic while marks are delivered' }], + }], + }); + } +}, 5); +timer.unref(); diff --git a/test/mcpl-awareness-barrier.test.ts b/test/mcpl-awareness-barrier.test.ts index 9cb27c5d3..6530bee6a 100644 --- a/test/mcpl-awareness-barrier.test.ts +++ b/test/mcpl-awareness-barrier.test.ts @@ -9,6 +9,7 @@ import { writeFileSync, } from 'node:fs'; import { join } from 'node:path'; +import { EventEmitter } from 'node:events'; import { AgentFramework } from '../src/framework.js'; import { McplServerConnection } from '../src/mcpl/server-connection.js'; @@ -533,6 +534,86 @@ test('reconnect pauses fresh inbound data but permits registration and awareness } }); +test('a reconnect gate opens only after re-registration, even when the awareness drain settles first', async () => { + // The gate is global: marks pending for one server keep every data plane + // paused. Here the reconnecting server's re-registration (its ยง5.3 policy + // reply) is held while another server's drain finishes. The reconnecting + // server's data must wait for its grant, not only for the drain. + mkdirSync(TEST_ROOT, { recursive: true }); + const dir = mkdtempSync(join(TEST_ROOT, 'awareness-reconnect-grant-')); + const outboxPath = join(dir, 'awareness.json'); + const statusPath = join(dir, 'status.jsonl'); + const crashPath = join(dir, 'crash'); + const policyHoldPath = join(dir, 'release-policy'); + const server = (id: 'discord' | 'heartbeat', env: Record, reconnect = false): McplServerConfig => ({ + id, + command: process.execPath, + args: [FIXTURE], + requestTimeoutMs: 0, + env: { STATUS_PATH: statusPath, LEDGER_PATH: outboxPath, SERVER_ID: id, ...env }, + enabledFeatureSets: ['chat'], + ...(reconnect ? { reconnect: true, reconnectIntervalMs: 20, reconnectMaxIntervalMs: 20 } : {}), + }); + const outbox = new DiscordAwarenessOutbox(outboxPath); + let framework: AgentFramework | undefined; + try { + framework = await AgentFramework.create({ + storePath: join(dir, 'store'), + discordAwarenessOutboxPath: outboxPath, + discordAwarenessDeadlineMs: 1_000, + membrane: new MockMembrane().asMembrane(), + agents: [{ name: 'assistant', model: 'test', systemPrompt: 'test' }], + modules: [], + mcplServers: [ + server('discord', {}), + server('heartbeat', { + CRASH_PATH: crashPath, + GENERATION_PATH: join(dir, 'heartbeat-generation'), + POLICY_HOLD_PATH: policyHoldPath, + }, true), + ], + }); + await waitFor('both servers\' startup data answered', () => ['discord', 'heartbeat'].every((id) => + records(statusPath).some((r) => r.serverId === id && r.generation === 1 && r.event === 'push-response'))); + + // A mark pending for discord, then the heartbeat server drops and comes back. + const batch = outbox.prepare({ + agentName: 'assistant', + sourceBranch: 'source', + targetBranch: 'main', + activationPolicy: 'explicit', + refs: [{ serverId: 'discord', channelId: 'discord:guild:startup', messageId: 'message-3' }], + }); + assert.ok(batch); + outbox.activate(batch.id); + writeFileSync(crashPath, 'crash'); + + await waitFor('the reconnect generation\'s policy held', () => + records(statusPath).some((r) => r.serverId === 'heartbeat' && r.generation === 2 && r.event === 'policy-held')); + assert.ok( + records(statusPath).some((r) => r.serverId === 'heartbeat' && r.generation === 2 && r.event === 'data-request-sent'), + 'the reconnected server has already sent the data its grant must hold', + ); + await waitFor('discord\'s mark delivered while re-registration is held', () => + outbox.batches()[0]?.refs[0]?.deliveryStatus === 'applied'); + await new Promise((resolve) => setTimeout(resolve, 200)); + assert.equal( + records(statusPath).some((r) => r.serverId === 'heartbeat' && r.generation === 2 && r.event === 'push-response'), + false, + 'the reconnected server\'s data waits for its grant, though the drain has settled', + ); + + writeFileSync(policyHoldPath, 'release'); + await waitFor('the reconnected server\'s data after its grant', () => + records(statusPath).some((r) => r.serverId === 'heartbeat' && r.generation === 2 && r.event === 'push-response')); + const push = records(statusPath).find((r) => r.serverId === 'heartbeat' && r.generation === 2 && r.event === 'push-response'); + assert.equal(push?.accepted, true, 'delivered under the re-established grant, not rejected fail-closed'); + } finally { + await framework?.stop(); + rmSync(dir, { recursive: true, force: true }); + } +}); + test('startup fails explicitly when awareness reconciliation throws', async () => { mkdirSync(TEST_ROOT, { recursive: true }); const dir = mkdtempSync(join(TEST_ROOT, 'awareness-reconcile-failure-')); @@ -772,6 +853,90 @@ test('overlapping reconnect/list-change generations release only the newest barr assert.equal(readyCalls, 1); }); +test('inbound handlers wait out a generation installed while they wait', async () => { + const framework = Object.create(AgentFramework.prototype) as any; + const first = deferred(); + const second = deferred(); + framework.discordAwarenessBarrier = { generation: 1, requiresBarrier: true, promise: first.promise }; + const handled: string[] = []; + framework.pushHandler = { handlePushEvent: async () => { handled.push('push-event'); } }; + framework.inferenceRouter = { handleInferenceRequest: async () => { handled.push('inference-request'); } }; + framework.channelRegistry = { handleIncoming: async () => { handled.push('channels-incoming'); } }; + framework.handleHostCommand = async () => { handled.push('host-command'); return { ok: true }; }; + const connection = new EventEmitter() as any; + connection.id = 'discord'; + framework.wireMcplEvents(connection); + const responder = { id: 1, respond: () => {}, respondError: () => {} }; + for (const event of ['push-event', 'inference-request', 'channels-incoming', 'host-command']) { + connection.emit(event, {}, responder); + } + + // A newer generation is armed while the handlers wait on the first. + framework.discordAwarenessBarrier = { generation: 2, requiresBarrier: true, promise: second.promise }; + first.resolve({ status: 'delivered', delivered: 0, failed: 0 }); + await new Promise((resolve) => setTimeout(resolve, 20)); + assert.deepEqual(handled, [], 'no handler runs behind the newer generation'); + + framework.discordAwarenessBarrier = null; + second.resolve({ status: 'delivered', delivered: 0, failed: 0 }); + await new Promise((resolve) => setTimeout(resolve, 20)); + assert.deepEqual(handled.sort(), ['channels-incoming', 'host-command', 'inference-request', 'push-event']); +}); + +test('an inbound handler waiting on a generation that fails refuses its request and never runs it', async () => { + // A generation fails when the awareness ledger can't be read or written, so + // its marks were not all attempted: nothing that waited behind it may run, + // and each request is answered with the failure rather than left waiting. + const framework = Object.create(AgentFramework.prototype) as any; + const failing = deferred(); + framework.discordAwarenessBarrier = { generation: 1, requiresBarrier: true, promise: failing.promise }; + const handled: string[] = []; + framework.pushHandler = { handlePushEvent: async () => { handled.push('push-event'); } }; + framework.inferenceRouter = { handleInferenceRequest: async () => { handled.push('inference-request'); } }; + framework.channelRegistry = { handleIncoming: async () => { handled.push('channels-incoming'); } }; + framework.handleHostCommand = async () => { handled.push('host-command'); return { ok: true }; }; + const connection = new EventEmitter() as any; + connection.id = 'discord'; + framework.wireMcplEvents(connection); + const answers: Array<{ event: string; code?: number; result?: unknown }> = []; + for (const event of ['push-event', 'inference-request', 'channels-incoming', 'host-command']) { + connection.emit(event, {}, { + id: 1, + respond: (result: unknown) => { answers.push({ event, result }); }, + respondError: (code: number) => { answers.push({ event, code }); }, + }); + } + + failing.reject(new Error('injected accounting failure')); + await waitFor('every request to be answered', () => answers.length === 4); + assert.deepEqual(handled, [], 'nothing runs behind a failed generation'); + assert.deepEqual( + answers.sort((a, b) => a.event.localeCompare(b.event)), + ['channels-incoming', 'host-command', 'inference-request', 'push-event'].map((event) => ({ event, code: -32603 })), + 'each request is refused with the failure', + ); +}); + +test('with no generation current, an inbound event is handled in the turn it arrives', () => { + // The push coalescer orders a push against its retraction by arrival, so a + // handler with no gate to wait out must not yield first. + const framework = Object.create(AgentFramework.prototype) as any; + framework.discordAwarenessBarrier = null; + const handled: string[] = []; + framework.pushHandler = { handlePushEvent: async () => { handled.push('push-event'); } }; + framework.inferenceRouter = { handleInferenceRequest: async () => { handled.push('inference-request'); } }; + framework.channelRegistry = { handleIncoming: async () => { handled.push('channels-incoming'); } }; + framework.handleHostCommand = async () => { handled.push('host-command'); return { ok: true }; }; + const connection = new EventEmitter() as any; + connection.id = 'discord'; + framework.wireMcplEvents(connection); + const responder = { id: 1, respond: () => {}, respondError: () => {} }; + for (const event of ['push-event', 'inference-request', 'channels-incoming', 'host-command']) { + connection.emit(event, {}, responder); + } + assert.deepEqual(handled.sort(), ['channels-incoming', 'host-command', 'inference-request', 'push-event']); +}); + test('synchronous zero-pending flush rechecks the gate after a nested list change', () => { const connection = new (McplServerConnection as any)( 'discord', null, null, diff --git a/test/surgery-marks-receipt.test.ts b/test/surgery-marks-receipt.test.ts new file mode 100644 index 000000000..8be05a8e7 --- /dev/null +++ b/test/surgery-marks-receipt.test.ts @@ -0,0 +1,669 @@ +/** + * Live surgery reports its body change apart from the Discord awareness marks + * it schedules, and does not wait on Discord to accept them. + * + * Field incident (2026-10-07): a web-UI rollback removed 918 Discord messages + * from a resident's context. rollbackToMessage awaited the serial reaction + * drain inside its store reservation, so the operator's surgery-result never + * arrived within the client's 60 s, the store stayed reserved, and every MCPL + * data plane stayed paused for the whole drain. These tests hold the fake + * server's reaction replies to model that drain. + * + * The receipt is a scheduling statement (`markers`), never a delivery claim, + * and a ledger failure after the body change landed is reported as + * `not-scheduled` instead of a failed surgery an operator might retry. + */ +import { describe, it, beforeEach, afterEach } from 'node:test'; +import assert from 'node:assert/strict'; +import { fileURLToPath } from 'node:url'; +import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { AgentFramework, ApiServer, DiscordAwarenessOutbox } from '../src/index.js'; +import type { OperatorLogEntry } from '../src/index.js'; +import { MockMembrane } from './helpers/mock-membrane.js'; + +const FIXTURE = fileURLToPath(new URL('./fixtures/surgery-marks-mcpl-server.mjs', import.meta.url)); + +function jsonl>(path: string): T[] { + if (!existsSync(path)) return []; + return readFileSync(path, 'utf8').split('\n').filter(Boolean).map((line) => JSON.parse(line) as T); +} + +async function waitFor(description: string, predicate: () => boolean, timeoutMs = 5_000): Promise { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + if (predicate()) return; + await new Promise((resolve) => setTimeout(resolve, 10)); + } + assert.fail(`timed out waiting for ${description}`); +} + +/** Resolve with the promise's value, or fail if it is still pending after ms. */ +async function settlesWithin(promise: Promise, ms: number, what: string): Promise { + let timer: ReturnType | undefined; + try { + return await Promise.race([ + promise, + new Promise((_, reject) => { + timer = setTimeout(() => reject(new Error(`${what} still pending after ${ms} ms`)), ms); + }), + ]); + } finally { + if (timer) clearTimeout(timer); + } +} + +describe('surgery marker receipt', () => { + let dir: string; + let storePath: string; + let callsPath: string; + let eventsPath: string; + let holdPath: string; + let answerLimitPath: string; + let probePath: string; + let framework: AgentFramework; + + const cm = () => framework.getAgent('resident')!.getContextManager(); + const outboxPath = () => join(storePath, 'recovery', 'discord-awareness-outbox.json'); + const ledger = () => new DiscordAwarenessOutbox(outboxPath()).batches(); + type Internals = { + discordAwarenessBarrier: { generation: number; requiresBarrier: boolean } | null; + mcplServerRegistry: { getServer(id: string): { dataPlaneReady: boolean } | undefined } | null; + reserveStoreForSurgery(verb: string, agentName: string): () => void; + recordTurnCheckpoint(agentName: string): void; + handleHostCommand(serverId: string, params: Record): Promise>; + }; + const internals = () => framework as unknown as Internals; + /** The delivery gate is up: a generation that requires it, and discord's data plane paused. */ + const assertGateUp = (when: string) => { + assert.equal(internals().discordAwarenessBarrier?.requiresBarrier, true, `${when}: an awareness generation requires the gate`); + assert.equal(internals().mcplServerRegistry?.getServer('discord')?.dataPlaneReady, false, `${when}: discord's data plane is paused`); + }; + const probeAnswer = () => jsonl<{ event: string; error?: unknown }>(eventsPath).find((e) => e.event === 'probe-answered'); + /** The kinds of the Discord-awareness ops alerts raised from now on, in order. */ + const awarenessAlerts = (): string[] => { + const kinds: string[] = []; + framework.onTrace((event) => { + const alert = event as unknown as { type: string; kind?: string }; + if (alert.type === 'ops:alert' && alert.kind?.startsWith('discord-awareness')) kinds.push(alert.kind); + }); + return kinds; + }; + + /** With `appStore`, the framework gets an app-owned store and no storePath, so its outbox is disabled. */ + async function start(withDiscord: boolean, appStore?: unknown, membrane = new MockMembrane()): Promise { + framework = await AgentFramework.create({ + ...(appStore ? { store: appStore as never } : { storePath }), + membrane: membrane.asMembrane(), + agents: [{ name: 'resident', model: 'test-model', systemPrompt: 'You are a resident.' }], + modules: [], + maintenanceIntervalMs: 0, + ...(withDiscord ? { + mcplServers: [{ + id: 'discord', + command: process.execPath, + args: [FIXTURE], + env: { + CALLS_PATH: callsPath, + EVENTS_PATH: eventsPath, + HOLD_PATH: holdPath, + ANSWER_LIMIT_PATH: answerLimitPath, + PROBE_PATH: probePath, + }, + enabledFeatureSets: ['chat'], + }], + } : {}), + }); + } + + /** A kept message, the rollback target, then `n` addressable messages. */ + function seed(n: number): { tail: string; removed: string[] } { + cm().addMessage('Operator', [{ type: 'text', text: 'kept' }], { + serverId: 'discord', channelId: 'discord:g1:dm', messageId: 'kept', + }); + const tail = String(cm().addMessage('Operator', [{ type: 'text', text: 'last good' }], { + serverId: 'discord', channelId: 'discord:g1:dm', messageId: 'tail', + })); + const removed: string[] = []; + for (let i = 0; i < n; i++) { + removed.push(String(cm().addMessage(`Member${i}`, [{ type: 'text', text: `ambient ${i}` }], { + serverId: 'discord', channelId: 'discord:g1:c1', messageId: `amb-${i}`, + }))); + } + return { tail, removed }; + } + + beforeEach(() => { + dir = mkdtempSync(join(tmpdir(), 'surgery-marks-')); + storePath = join(dir, 'store'); + callsPath = join(dir, 'calls.jsonl'); + eventsPath = join(dir, 'events.jsonl'); + holdPath = join(dir, 'release-reactions'); + answerLimitPath = join(dir, 'answer-limit'); + probePath = join(dir, 'send-probe'); + }); + + afterEach(async () => { + writeFileSync(holdPath, '1'); + await framework?.stop(); + rmSync(dir, { recursive: true, force: true }); + }); + + it('rollback returns once the batch is active, before Discord has accepted any reaction', async () => { + await start(true); + const { tail } = seed(3); + + const result = await settlesWithin( + framework.rollbackToMessage('resident', { messageId: tail }), + 3_000, + 'rollbackToMessage with reactions held', + ); + + assert.equal(result.messagesRemoved, 3); + assert.equal(result.markers.status, 'queued'); + assert.equal(result.markers.queued, 3); + const [batch] = ledger(); + assert.equal(batch.status, 'active', 'the batch is durably active before the call returns'); + assert.equal(result.markers.status === 'queued' && result.markers.batchId, batch.id); + assert.equal( + jsonl(eventsPath).filter((e) => e.event === 'reaction-answered').length, + 0, + 'no reaction had been accepted when the rollback returned', + ); + + // Delivery continues in the background and completes once Discord answers. + writeFileSync(holdPath, '1'); + await waitFor('all three reactions answered', () => + jsonl(eventsPath).filter((e) => e.event === 'reaction-answered').length === 3); + await framework.syncDiscordAwarenessMarkers(); + const entries = ledger()[0].refs; + assert.deepEqual(entries.map((ref) => ref.deliveryStatus), ['applied', 'applied', 'applied']); + assert.deepEqual( + jsonl<{ name: string; args: { emoji: string } }>(callsPath).map((call) => `${call.name}:${call.args.emoji}`), + ['add_reaction:๐Ÿ’ค', 'add_reaction:๐Ÿ’ค', 'add_reaction:๐Ÿ’ค'], + ); + + const logged = new Map( + framework.getOperatorLog({ limit: 10 }).map((entry) => [entry.kind, entry]), + ); + assert.deepEqual(logged.get('rollback')?.result?.markers, result.markers); + }); + + it('releases the store while delivery runs, but keeps MCPL traffic behind the delivery gate', async () => { + await start(true); + const { removed } = seed(4); + + await settlesWithin( + framework.rollbackToMessage('resident', { messageId: removed[1] }), + 3_000, + 'rollbackToMessage with reactions held', + ); + await waitFor('the first reaction call to reach the server', () => jsonl(callsPath).length >= 1); + + // The store reservation is released: a second surgery is admitted while + // the first one's reactions are still unanswered. + const second = await settlesWithin( + framework.suppressMessages('resident', { messageIds: [removed[0]] }), + 3_000, + 'a second surgery during delivery', + ); + assert.equal(second.messagesRemoved, 1); + assert.equal(second.markers.status, 'queued'); + + // The gate was installed before release: ordinary channel traffic sent + // during delivery is not answered until the reactions are. + assertGateUp('during the first surgery\'s delivery'); + writeFileSync(probePath, '1'); + await waitFor('the probe to be sent', () => jsonl(eventsPath).some((e) => e.event === 'probe-sent')); + await new Promise((resolve) => setTimeout(resolve, 200)); + assert.equal( + jsonl(eventsPath).some((e) => e.event === 'probe-answered'), + false, + 'channel traffic waits behind the delivery gate', + ); + + writeFileSync(holdPath, '1'); + await waitFor('the probe to be answered after delivery', () => + jsonl(eventsPath).some((e) => e.event === 'probe-answered')); + assert.equal(probeAnswer()?.error, null, 'the probe was answered, not refused'); + // The server's own event order, not timestamps (two events can share a + // millisecond), and every expected reaction by id: one compared only with + // the reactions that happened to be answered would pass if the second + // surgery's batch were never delivered. + const events = jsonl<{ event: string; messageId?: string }>(eventsPath); + const probeIndex = events.findIndex((e) => e.event === 'probe-answered'); + const answeredBeforeProbe = events.slice(0, probeIndex) + .filter((e) => e.event === 'reaction-answered') + .map((e) => e.messageId) + .sort(); + assert.deepEqual( + answeredBeforeProbe, + ['amb-0', 'amb-2', 'amb-3'], + 'both surgeries\' reactions are answered before channel traffic is released', + ); + }); + + it('restoring the source while delivery runs keeps traffic gated until the running drain is done', async () => { + await start(true); + const { tail } = seed(3); + + const rollback = await settlesWithin( + framework.rollbackToMessage('resident', { messageId: tail }), + 3_000, + 'rollbackToMessage with reactions held', + ); + await waitFor('the first reaction call to reach the server', () => jsonl(callsPath).length >= 1); + + // The operator restores the source the way the branch-switch API does + // (store switch, then synchronization) while the first add_reaction still + // waits for its reply. Reconciliation makes every mark undesired, and the + // held one is not present yet, so the ledger shows nothing pending: only + // the running drain knows it will still send its saved adds. + framework.getStore().switchBranch(rollback.sourceBranch); + const restored = framework.syncDiscordAwarenessMarkers(); + // Nothing pending for discord: only the running drain keeps the gate up. + assert.equal(new DiscordAwarenessOutbox(outboxPath()).pending('discord').length, 0, 'the ledger shows nothing pending'); + assertGateUp('after the restore'); + + writeFileSync(probePath, '1'); + await waitFor('the probe to be sent', () => jsonl(eventsPath).some((e) => e.event === 'probe-sent')); + await new Promise((resolve) => setTimeout(resolve, 200)); + assert.equal( + jsonl(eventsPath).some((e) => e.event === 'probe-answered'), + false, + 'channel traffic waits for the running drain even though nothing appears pending', + ); + + writeFileSync(holdPath, '1'); + await settlesWithin(restored, 5_000, "the restore's synchronization"); + await waitFor('the probe to be answered after delivery', () => + jsonl(eventsPath).some((e) => e.event === 'probe-answered')); + assert.equal(probeAnswer()?.error, null, 'the probe was answered, not refused'); + + // Every reaction request the drain made, including the removals the + // restore needs, was answered before channel traffic was released, and + // the restore leaves no mark behind. + const events = jsonl<{ event: string }>(eventsPath); + const probeIndex = events.findIndex((e) => e.event === 'probe-answered'); + const answered = (from: number, to?: number) => + events.slice(from, to).filter((e) => e.event === 'reaction-answered').length; + assert.equal(answered(0, probeIndex), jsonl(callsPath).length, 'every reaction call answered before the probe'); + assert.equal(answered(probeIndex), 0, 'no reaction answered after the probe'); + assert.deepEqual( + ledger()[0].refs.map((ref) => [ref.messageId, ref.desired, ref.markerPresent, ref.deliveryStatus]), + [['amb-0', false, false, 'applied'], ['amb-1', false, false, 'applied'], ['amb-2', false, false, 'applied']], + ); + }); + + it('a rollback whose marker bookkeeping fails after the switch reports the body as applied', async () => { + await start(true); + const { tail } = seed(2); + const alerts = awarenessAlerts(); + const original = DiscordAwarenessOutbox.prototype.activate; + DiscordAwarenessOutbox.prototype.activate = function () { + throw new Error('injected ledger failure'); + }; + let result: Awaited>; + try { + result = await framework.rollbackToMessage('resident', { messageId: tail }); + } finally { + DiscordAwarenessOutbox.prototype.activate = original; + } + + assert.equal(cm().currentBranch().name, result.targetBranch, 'the rollback stands'); + assert.equal(result.messagesRemoved, 2); + assert.equal(result.markers.status, 'not-scheduled'); + assert.match(result.markers.status === 'not-scheduled' ? result.markers.error : '', /injected ledger failure/); + const entry = framework.getOperatorLog({ limit: 10 }).find((e) => e.kind === 'rollback'); + assert.equal(entry?.error, undefined, 'logged as an applied rollback, not a failure'); + assert.equal((entry?.result?.markers as { status?: string } | undefined)?.status, 'not-scheduled'); + assert.deepEqual(alerts, ['discord-awareness-not-scheduled'], 'the operator is told its marks were not scheduled'); + + // "Not scheduled" holds across the reconciliation delivery runs and + // across a restart: the batch was retired, so nothing can promote it. + writeFileSync(holdPath, '1'); + await framework.syncDiscordAwarenessMarkers(); + assert.deepEqual(ledger(), []); + await framework.stop(); + await start(true); + await framework.syncDiscordAwarenessMarkers(); + assert.deepEqual(ledger(), []); + assert.equal(jsonl(callsPath).length, 0, 'no reaction was ever sent'); + }); + + it('reports unresolved bookkeeping when the batch can be neither activated nor retired', async () => { + await start(false); + const { tail } = seed(2); + const alerts = awarenessAlerts(); + const activate = DiscordAwarenessOutbox.prototype.activate; + const discard = DiscordAwarenessOutbox.prototype.discard; + DiscordAwarenessOutbox.prototype.activate = function () { + throw new Error('injected ledger failure'); + }; + DiscordAwarenessOutbox.prototype.discard = function () { + throw new Error('injected retire failure'); + }; + let result: Awaited>; + try { + result = await framework.rollbackToMessage('resident', { messageId: tail }); + } finally { + DiscordAwarenessOutbox.prototype.activate = activate; + DiscordAwarenessOutbox.prototype.discard = discard; + } + assert.equal(cm().currentBranch().name, result.targetBranch, 'the rollback stands'); + // The receipt promises nothing about this batch: it is still in the + // ledger, and a reconciliation may promote and deliver it. + assert.equal(result.markers.status, 'unresolved'); + const batches = ledger(); + assert.equal(batches.length, 1); + assert.equal(result.markers.status === 'unresolved' && result.markers.batchId, batches[0].id); + assert.match(result.markers.status === 'unresolved' ? result.markers.error : '', /injected ledger failure/); + assert.deepEqual(alerts, ['discord-awareness-unresolved'], 'the operator is told the batch may still be delivered'); + }); + + it('a suppression whose marker bookkeeping fails after the last redaction stands', async () => { + await start(false); + const { removed } = seed(3); + const alerts = awarenessAlerts(); + const original = DiscordAwarenessOutbox.prototype.activate; + DiscordAwarenessOutbox.prototype.activate = function () { + throw new Error('injected ledger failure'); + }; + let result: Awaited>; + try { + result = await framework.suppressMessages('resident', { messageIds: [removed[1]] }); + } finally { + DiscordAwarenessOutbox.prototype.activate = original; + } + + assert.equal(cm().currentBranch().name, result.targetBranch, 'the suppression is not undone'); + assert.deepEqual(result.removedIds, [removed[1]]); + assert.ok(!cm().getAllMessages().some((m) => String(m.id) === removed[1])); + assert.equal(result.markers.status, 'not-scheduled'); + assert.deepEqual(ledger(), [], 'every interval committed, so the retired journal loses nothing'); + const entry = framework.getOperatorLog({ limit: 10 }).find((e) => e.kind === 'suppress'); + assert.equal(entry?.error, undefined, 'logged as an applied suppression'); + assert.deepEqual(entry?.result?.markers, result.markers); + assert.deepEqual(alerts, ['discord-awareness-not-scheduled']); + }); + + it('installs the delivery gate before the surgery releases the store', async () => { + await start(true); + const { tail } = seed(2); + const atRelease: Array<{ installed: boolean; requires?: boolean }> = []; + const reserve = internals().reserveStoreForSurgery.bind(framework); + internals().reserveStoreForSurgery = (verb, agentName) => { + const release = reserve(verb, agentName); + return () => { + const barrier = internals().discordAwarenessBarrier; + atRelease.push({ installed: barrier !== null, requires: barrier?.requiresBarrier }); + release(); + }; + }; + await settlesWithin(framework.rollbackToMessage('resident', { messageId: tail }), 3_000, 'rollback with reactions held'); + assert.deepEqual(atRelease, [{ installed: true, requires: true }]); + }); + + it('a switch, an undo, a redo and a turn undo during a running drain each start delivery and return while replies are held', async () => { + await start(true); + internals().recordTurnCheckpoint('resident'); + const { tail } = seed(3); + const rollback = await settlesWithin( + framework.rollbackToMessage('resident', { messageId: tail }), + 3_000, + 'rollbackToMessage with reactions held', + ); + await waitFor('the first reaction call to reach the server', () => jsonl(callsPath).length >= 1); + + // The web API's commands, called as its WebSocket dispatch calls them. + const server = new ApiServer(framework, { port: 0 }) as unknown as { + cmdBranchSwitch(params: { name: string }): Promise<{ switched: boolean }>; + cmdUndo(params: { agentName: string }): Promise<{ undone?: boolean }>; + cmdRedo(params: { agentName: string }): Promise<{ redone?: boolean }>; + }; + // Each command starts delivery of the marks it moved before it returns: it + // arms a newer generation, which holds traffic, and doesn't wait for it. + let generation = internals().discordAwarenessBarrier!.generation; + const startedDelivery = (when: string) => { + assertGateUp(when); + const current = internals().discordAwarenessBarrier!.generation; + assert.ok(current > generation, `${when}: a newer generation, the delivery the command started`); + generation = current; + }; + const switched = await settlesWithin(server.cmdBranchSwitch({ name: rollback.sourceBranch }), 3_000, 'web branch switch'); + assert.equal(switched.switched, true); + startedDelivery('after the switch'); + const undone = await settlesWithin(server.cmdUndo({ agentName: 'resident' }), 3_000, 'web undo'); + assert.equal(undone.undone, true); + startedDelivery('after the undo'); + const redone = await settlesWithin(server.cmdRedo({ agentName: 'resident' }), 3_000, 'web redo'); + assert.equal(redone.redone, true); + startedDelivery('after the redo'); + const turnUndo = await settlesWithin( + internals().handleHostCommand('discord', { command: 'undo', agentName: 'resident', turns: 1 }), + 3_000, + 'host/command turn undo', + ); + assert.equal(turnUndo.undone, 1); + startedDelivery('after the turn undo'); + + // None of them waited for Discord: no reaction has been answered. + assert.equal(jsonl(eventsPath).filter((e) => e.event === 'reaction-answered').length, 0); + // Once Discord answers, the deliveries the commands started finish on + // their own, with no further sync, and leave nothing pending. + writeFileSync(holdPath, '1'); + await waitFor('the deliveries the commands started to finish', () => internals().discordAwarenessBarrier === null); + assert.equal(new DiscordAwarenessOutbox(outboxPath()).pending().length, 0, 'every mark the moves needed was delivered'); + }); + + it('a rollback, its restore and a redo during one drain: every mark is sent before traffic is released', async () => { + await start(true); + const { tail } = seed(2); + const rollback = await settlesWithin( + framework.rollbackToMessage('resident', { messageId: tail }), + 3_000, + 'rollbackToMessage with reactions held', + ); + await waitFor('the first add to reach the server', () => jsonl(callsPath).length >= 1); + + // The source is restored while the first add waits for its reply. + framework.getStore().switchBranch(rollback.sourceBranch); + const restored = framework.syncDiscordAwarenessMarkers(); + // The drain sends both adds, then the removals the restore needs. The + // first removal is held. + writeFileSync(answerLimitPath, '2'); + writeFileSync(holdPath, '1'); + await waitFor('the first removal to reach the server', () => + jsonl<{ name: string }>(callsPath).some((call) => call.name === 'remove_reaction')); + // The rollback is redone while that removal is in flight. The drain has + // already attempted both adds, so it won't send them again itself. + framework.getStore().switchBranch(rollback.targetBranch); + const redone = framework.syncDiscordAwarenessMarkers(); + writeFileSync(probePath, '1'); + await waitFor('the probe to be sent', () => jsonl(eventsPath).some((e) => e.event === 'probe-sent')); + assertGateUp('after the redo'); + + writeFileSync(answerLimitPath, '99'); + await settlesWithin(Promise.all([restored, redone]), 5_000, 'the restore and redo synchronizations'); + await waitFor('the probe to be answered after delivery', () => probeAnswer() !== undefined); + assert.equal(probeAnswer()?.error, null, 'the probe was answered, not refused'); + + assert.deepEqual( + ledger()[0].refs.map((ref) => [ref.messageId, ref.desired, ref.markerPresent]), + [['amb-0', true, true], ['amb-1', true, true]], + 'the redo\'s marks are on Discord', + ); + const events = jsonl<{ event: string; name?: string; messageId?: string }>(eventsPath); + const probeIndex = events.findIndex((e) => e.event === 'probe-answered'); + assert.deepEqual( + events.slice(0, probeIndex).filter((e) => e.event === 'reaction-answered').map((e) => `${e.name}:${e.messageId}`), + [ + 'add_reaction:amb-0', 'add_reaction:amb-1', + 'remove_reaction:amb-0', 'remove_reaction:amb-1', + 'add_reaction:amb-0', 'add_reaction:amb-1', + ], + 'the redo\'s adds were sent again, before channel traffic was released', + ); + }); + + it('a turn for channel traffic admitted before the gate may start before the drain completes', async () => { + // Accepted, and stated in the changelog: the surgery's store reservation + // ends once the gate is up, so no operator command waits on Discord, and + // the gate holds only traffic that arrives after it. Before this change + // the reservation held such a turn until the whole drain had finished. + // This pins that accepted boundary: holding pre-gate work by some other + // means would be a deliberate change to it, made together with the text. + const membrane = new MockMembrane(); + await start(true, undefined, membrane); + framework.start(); + const { tail } = seed(3); + // The switch the rollback awaits goes on only once one ordinary channel + // message has been admitted and answered, before any gate is up. + const switchBranch = cm().switchBranch.bind(cm()); + (cm() as unknown as { switchBranch: (name: string) => Promise }).switchBranch = async (name: string) => { + writeFileSync(probePath, '1'); + await waitFor('the channel message answered during the switch', () => probeAnswer() !== undefined); + return switchBranch(name); + }; + const turnsBefore = membrane.calls.length; + + await settlesWithin(framework.rollbackToMessage('resident', { messageId: tail }), 3_000, 'rollbackToMessage with reactions held'); + assert.equal(probeAnswer()?.error, null, 'the channel message was admitted'); + await waitFor('a turn for the channel message', () => membrane.calls.length > turnsBefore); + + // The turn started while the drain still runs: the first reaction request + // is on the wire, but every reply is still held and the gate is still up. + await waitFor('the first reaction request to reach the server', () => jsonl(callsPath).length >= 1); + assert.equal(jsonl(eventsPath).filter((e) => e.event === 'reaction-answered').length, 0, 'no reaction reply has been answered'); + assertGateUp('while that turn runs'); + assert.ok( + JSON.stringify(membrane.calls[turnsBefore]).includes('ordinary traffic while marks are delivered'), + 'the turn carries the channel message', + ); + }); + + it('with the awareness outbox disabled, a rollback schedules no marks and puts up no gate', async () => { + const { JsStore } = await import('@animalabs/chronicle'); + await start(true, JsStore.openOrCreate({ path: join(dir, 'app-store') })); + const { tail } = seed(2); + const result = await settlesWithin( + framework.rollbackToMessage('resident', { messageId: tail }), + 3_000, + 'rollbackToMessage without an outbox', + ); + assert.equal(result.messagesRemoved, 2); + assert.equal(result.removedRefs.length, 2, 'it removed addressable Discord messages'); + assert.deepEqual(result.markers, { status: 'none', queued: 0 }); + assert.equal(internals().discordAwarenessBarrier, null, 'no gate'); + + // Reactions are still held, and none is needed: traffic flows at once. + writeFileSync(probePath, '1'); + await waitFor('the probe to be answered with no gate', () => probeAnswer() !== undefined); + assert.equal(probeAnswer()?.error, null); + assert.equal(jsonl(callsPath).length, 0, 'no reaction was sent'); + }); + + it('a delivery that fails to start raises an ops alert and leaves no unhandled rejection', async () => { + await start(true); + const { tail } = seed(2); + const alerts: Array<{ kind: string; message: string }> = []; + framework.onTrace((event) => { + if (event.type === 'ops:alert') alerts.push(event as unknown as { kind: string; message: string }); + }); + const unhandled: unknown[] = []; + const onUnhandled = (reason: unknown) => { unhandled.push(reason); }; + process.on('unhandledRejection', onUnhandled); + const reconcile = DiscordAwarenessOutbox.prototype.reconcileForBranch; + DiscordAwarenessOutbox.prototype.reconcileForBranch = function () { + throw new Error('injected ledger read failure'); + }; + let result: Awaited>; + try { + result = await settlesWithin( + framework.rollbackToMessage('resident', { messageId: tail }), + 3_000, + 'rollbackToMessage whose delivery fails to start', + ); + await waitFor('the delivery alert', () => alerts.some((alert) => alert.kind === 'discord-awareness-delivery')); + await new Promise((resolve) => setTimeout(resolve, 50)); + } finally { + DiscordAwarenessOutbox.prototype.reconcileForBranch = reconcile; + process.off('unhandledRejection', onUnhandled); + } + assert.deepEqual(unhandled, [], 'the background failure is handled'); + assert.match( + alerts.find((alert) => alert.kind === 'discord-awareness-delivery')!.message, + /after rollback failed: .*injected ledger read failure/, + ); + // Scheduling succeeded: the batch stays durably active for the next sync. + assert.equal(result.markers.status, 'queued'); + assert.equal(ledger()[0].status, 'active'); + }); + + it('a suppression with nothing addressable reports none, whether or not its journal can be closed', async () => { + await start(false); + const alerts = awarenessAlerts(); + const local = String(cm().addMessage('resident', [{ type: 'text', text: 'local only' }], {})); + const other = String(cm().addMessage('resident', [{ type: 'text', text: 'also local' }], {})); + cm().addMessage('resident', [{ type: 'text', text: 'after' }], {}); + + const closed = await framework.suppressMessages('resident', { messageIds: [local] }); + assert.deepEqual(closed.markers, { status: 'none', queued: 0 }); + const [journal] = ledger(); + assert.equal(journal?.status, 'active', 'activated, which closes the resume journal'); + assert.deepEqual(journal.refs, []); + + // When the journal can be neither activated nor retired, it still + // schedules nothing, and the receipt says none, not unresolved. + const activate = DiscordAwarenessOutbox.prototype.activate; + const discard = DiscordAwarenessOutbox.prototype.discard; + DiscordAwarenessOutbox.prototype.activate = function () { throw new Error('injected ledger failure'); }; + DiscordAwarenessOutbox.prototype.discard = function () { throw new Error('injected retire failure'); }; + let stuck: Awaited>; + try { + stuck = await framework.suppressMessages('resident', { messageIds: [other] }); + } finally { + DiscordAwarenessOutbox.prototype.activate = activate; + DiscordAwarenessOutbox.prototype.discard = discard; + } + assert.deepEqual(stuck.markers, { status: 'none', queued: 0 }); + assert.equal(ledger().filter((batch) => batch.status === 'prepared').length, 1, 'its journal is left prepared'); + assert.deepEqual(alerts, [], 'no marks were at stake, so no alert'); + }); + + it('a suppression whose batch can be neither activated nor retired reports unresolved', async () => { + await start(false); + const { removed } = seed(3); + const alerts = awarenessAlerts(); + const activate = DiscordAwarenessOutbox.prototype.activate; + const discard = DiscordAwarenessOutbox.prototype.discard; + DiscordAwarenessOutbox.prototype.activate = function () { throw new Error('injected ledger failure'); }; + DiscordAwarenessOutbox.prototype.discard = function () { throw new Error('injected retire failure'); }; + let result: Awaited>; + try { + result = await framework.suppressMessages('resident', { messageIds: [removed[1]] }); + } finally { + DiscordAwarenessOutbox.prototype.activate = activate; + DiscordAwarenessOutbox.prototype.discard = discard; + } + assert.equal(cm().currentBranch().name, result.targetBranch, 'the suppression stands'); + assert.equal(result.markers.status, 'unresolved'); + const [batch] = ledger(); + assert.equal(batch.status, 'prepared'); + assert.equal(result.markers.status === 'unresolved' && result.markers.batchId, batch.id); + const entry = framework.getOperatorLog({ limit: 10 }).find((e) => e.kind === 'suppress'); + assert.deepEqual(entry?.result?.markers, result.markers); + assert.deepEqual(alerts, ['discord-awareness-unresolved']); + }); + + it('a surgery that removes nothing addressable schedules no marks', async () => { + await start(false); + const keep = String(cm().addMessage('Operator', [{ type: 'text', text: 'kept' }], {})); + cm().addMessage('resident', [{ type: 'text', text: 'local only' }], {}); + const result = await framework.rollbackToMessage('resident', { messageId: keep }); + assert.equal(result.messagesRemoved, 1, 'the rollback did change the body'); + assert.deepEqual(result.markers, { status: 'none', queued: 0 }); + assert.equal(ledger().length, 0); + }); +});