diff --git a/README.md b/README.md index 92fe15c82..180c94114 100644 --- a/README.md +++ b/README.md @@ -239,6 +239,30 @@ Optional host-side implementation of the MCP Live protocol. External servers (ga } ``` +#### Event coalescing (RFC-006) + +The host advertises `eventCoalescing` (both lanes, deferred rendering, +channel-scoped pushes, a one-hour retry window). A server may add +`coalesce: { key }` to a `push/event` or a `channels/incoming` message so that a +later occurrence of the same subject **replaces** the earlier one while it is +still unread, and **appends** once a model has seen it; `retract: true` +withdraws unread content and appends the supplied deletion notice only when +some version was read (or history is unknown). `initial: true` on a create lets +the host know the subject's history is complete, so a create → edit → delete +that nobody read leaves no trace. `deferred: true` (push only) holds a batch of +notices outside context and asks the server for the content with `push/render` +when a turn is assembled. + +Plain content stays in context and is delivered on arrival through the ordinary +channel/push path; the host only remembers where it landed. "Unread" means: +above the agent's consumed watermark (advanced at every compile of a model +request), on the current branch, not a sharded message, and not folded into a +summary by compression. On any doubt — restart, branch switch, eviction — the +host appends, as it does today. There is no per-subject cap: a busy channel's +unread backlog is simply its unread backlog. Receipts and subject history are +kept in the `mcpl/coalescing` state (retry window 1 h); audit lines are +`mcpl:coalescing` trace events. + ### Streaming Lifecycle 1. **Start**: Framework calls `agent.startStreamWithInjections()` → `YieldingStream` diff --git a/changelog.d/mcpl-event-coalescing.added.md b/changelog.d/mcpl-event-coalescing.added.md new file mode 100644 index 000000000..5ff997d46 --- /dev/null +++ b/changelog.d/mcpl-event-coalescing.added.md @@ -0,0 +1 @@ +- MCPL event coalescing (RFC-006, mcpl PR #5 revision 7): `coalesce` on `push/event` and `channels/incoming` — replace-if-unread, atomic retraction with a conditional deletion notice, `initial` birth marker, deferred batches rendered through `push/render` at assembly, channel-scoped pushes delivered as messages of their channel. Content stays in context; "unread" is above the agent's consumed watermark and not folded by compression. Advertised as `eventCoalescing` with a one-hour retry window. diff --git a/src/agent.ts b/src/agent.ts index 0855aafdb..2085f0ff5 100644 --- a/src/agent.ts +++ b/src/agent.ts @@ -522,8 +522,50 @@ export class Agent { this.updateHotContextSettings({ preparedWindowTokens: null }); } + /** + * RFC-006 consumed watermark: the newest stored message at the last compile + * of a model request, per branch. Everything above it has never been in a + * request and may still be replaced or withdrawn in place by its sender. + */ + private consumedWatermark: { branch: string; sequence: number } | null = null; + + getConsumedWatermark(): { branch: string; sequence: number } | null { + return this.consumedWatermark; + } + + /** Mark everything currently stored as consumed (boot, branch switch). */ + markContextConsumed(): void { + // Tolerates partial context-manager doubles (tests build Agents over + // stubs): without a watermark nothing is ever treated as unread. + this.consumedWatermark = this.pendingWatermark(); + } + + /** The watermark a successful compile will establish (taken before, applied after). */ + private pendingWatermark(): { branch: string; sequence: number } | null { + const cm = this.contextManager as Partial; + try { + if (typeof cm.currentBranch !== 'function' || typeof cm.getAllMessages !== 'function') return null; + // The chronicle head, not this slot's last message: a compile reads + // everything stored on the branch so far, including messages merged in + // from another slot (the subconscious reads the residents' shared slot). + const store = typeof cm.getStore === 'function' ? cm.getStore() as { currentSequence?: () => number } : undefined; + let sequence = typeof store?.currentSequence === 'function' ? store.currentSequence() : undefined; + if (sequence === undefined) { + const count = typeof cm.getMessageCount === 'function' ? cm.getMessageCount() : cm.getAllMessages().length; + sequence = count > 0 ? cm.getAllMessages().at(-1)?.sequence ?? 0 : 0; + } + return { branch: cm.currentBranch().name, sequence }; + } catch { + return null; + } + } + async compileContext(budget?: TokenBudget): Promise { + // RFC-006 §3.3: consumption is determined at assembly — a compile that + // fails assembled nothing, so the watermark moves only on success. + const watermark = this.pendingWatermark(); const result = await this.contextManager.compile(this.resolveBudget(budget)); + if (watermark) this.consumedWatermark = watermark; if (!budget) this.settleRuntimeSettingsTransition(); return result; } @@ -538,9 +580,11 @@ export class Agent { injections?: ContextInjection[], opts?: { kvUnifiedImmutablePrefixHash?: string }, ): Promise { + const watermark = this.pendingWatermark(); const result = await this.contextManager.compile( this.resolveBudget(budget), injections, opts as never, ); + if (watermark) this.consumedWatermark = watermark; if (!budget) this.settleRuntimeSettingsTransition(); return result; } diff --git a/src/api/types.ts b/src/api/types.ts index db0d4ad22..21079d291 100644 --- a/src/api/types.ts +++ b/src/api/types.ts @@ -369,6 +369,8 @@ export type ApiEventType = | 'module:added' | 'module:removed' | 'message:added' + | 'message:removed' + | 'mcpl:coalescing' // Server-specific events | 'connected' | 'speech' diff --git a/src/framework.ts b/src/framework.ts index 6724e0219..510b9183a 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -69,7 +69,13 @@ import { maskNegotiatedCapabilities } from './mcpl/capability-mask.js'; import { HookOrchestrator } from './mcpl/hook-orchestrator.js'; import { ToolLifecycleEmitter, parseToolObserveParams } from './mcpl/tool-lifecycle.js'; import { BUILTIN_TOOL_CLASSES, normalizeClassTable, parseDeclaredClasses, resolveToolClass, type ToolClass } from './mcpl/tool-classes.js'; -import { PushHandler, type McplPushEvent } from './mcpl/push-handler.js'; +import { PushHandler, convertBlock as convertPushBlock, type McplPushEvent } from './mcpl/push-handler.js'; +import { + PushCoalescer, CoalesceError, PUSH_COALESCING_SUPPORT, validateCoalesceMember, validateCoalescedContent, + coalescingSubjectKey, type CoalescedOccurrence, type CoalescingPlacement, type CoalescingSnapshot, + type CoalescingReceiptRecord, +} from './mcpl/push-coalescer.js'; +import type { ChannelIncomingMessage, ChannelIncomingMessageResult, McplContentBlock, PushEventResult } from './mcpl/types.js'; import { parseProsePrefix, parseHybridProsePrefix } from './mcpl/prose-grammar.js'; import { ProseStreamRouter } from './mcpl/prose-stream-router.js'; import { detectKnownToolWrapperProse } from './tool-wrapper-prose-guard.js'; @@ -92,7 +98,7 @@ import { splitPreservingImageSlots, type ParsedImagePlaceholder, } from './tool-image-ledger.js'; -import { randomUUID } from 'node:crypto'; +import { randomUUID, createHash } from 'node:crypto'; import { PyRunner, buildInjectedTools } from './code-execution/py-runner.js'; import { buildCodeExecutionToolDefinition, @@ -411,6 +417,35 @@ import { const FRAMEWORK_STATE_ID = 'framework/state'; /** Snapshot slot mirroring context writes deferred while quiesced (#122). */ const DEFERRED_WRITES_ID = 'framework/deferred-writes'; +/** The event a coalesced occurrence delivers through — one per lane. */ +interface CoalescedChannelEvent { + type: 'mcpl:channel-incoming'; + serverId: string; + channelId: string; + messageId: string; + threadId?: string; + author: { id: string; name: string }; + content: ContentBlock[]; + timestamp: string; + metadata?: Record; + tags?: string[]; + triggerInference?: boolean; + targetAgents?: string[]; + coalescingSubject?: string; + /** Occurrence id (RFC-006 §3.1); channel messages carry it only when coalesced. */ + eventId?: string; + /** RFC-006 assembly: materialized for this agent's turn (store directly). */ + assemblingFor?: string; + /** RFC-006 §3.2: deliver into this agent only (replacement / notice audience). */ + deliverTo?: string; +} +type CoalescedDelivery = + | { lane: 'channel'; event: CoalescedChannelEvent } + | { lane: 'push'; event: McplPushEvent }; +/** RFC-006 coalescing bookkeeping: receipts (retry window) + subject history. */ +const COALESCING_STATE_ID = 'mcpl/coalescing'; +/** Receipts written since the last snapshot flush (bridges the throttle window). */ +const COALESCING_RECENT_ID = 'mcpl/coalescing-recent'; /** Above this serialized size the deferred-write mirror is skipped (kept in * memory only) rather than rewriting a multi-MB snapshot per deferral. */ const DEFERRED_WRITES_PERSIST_CAP_BYTES = 4 * 1024 * 1024; @@ -1226,6 +1261,24 @@ export class AgentFramework { 'BUILTIN_TOOL_CLASSES', ); private pushHandler: PushHandler | null = null; + /** RFC-006 coalescing (see mcpl/push-coalescer.ts for the design). */ + private pushCoalescer: PushCoalescer | null = null; + /** Both lanes are admitted in arrival order: a create, edit and delete of + * one subject must not overtake one another while routing awaits. */ + private coalescingAdmission: Promise = Promise.resolve(); + private coalescingSaveTimer: ReturnType | null = null; + private coalescingSnapshotDirty: (() => CoalescingSnapshot) | null = null; + private coalescingRecentReceipts: CoalescingReceiptRecord[] = []; + /** + * Agents that read the residents' shared un-namespaced message slot: every + * ordinary resident (their context managers are not isolated) and the + * subconscious (isolated, but merging that slot read-only). A message in + * that slot is unread only if NONE of them has compiled past it. Forks and + * ephemeral agents have isolated slots and are their own only readers. + */ + private readonly sharedSlotAgents = new Set(); + /** Group commit: one fsync per event-loop turn, shared by every waiter. */ + private coalescingCommit: Promise | null = null; private inferenceRouter: InferenceRouter | null = null; private channelRegistry: ChannelRegistry | null = null; private checkpointManager: CheckpointManager | null = null; @@ -1404,6 +1457,13 @@ export class AgentFramework { } catch { // Already registered } + for (const id of [COALESCING_STATE_ID, COALESCING_RECENT_ID]) { + try { + store.registerState({ id, strategy: 'snapshot' }); + } catch { + // Already registered + } + } try { store.registerState({ @@ -1802,6 +1862,8 @@ export class AgentFramework { * Stop the event loop. */ async stop(): Promise { + this.pushCoalescer?.suspend(); + this.flushCoalescingSnapshot(); this.running = false; // Flushed-but-unsynced deferred writes: sync and ack now, while the // store is still open, rather than leaving them to a reboot replay. @@ -3870,6 +3932,8 @@ export class AgentFramework { }); const agent = new Agent(config, contextManager, this.membrane); + agent.markContextConsumed(); // RFC-006: nothing stored before now is replaceable + agent.markContextConsumed(); // RFC-006: nothing stored before now is replaceable this.ephemeralCandidates.set(agent, contextManager); const cleanup = () => { @@ -6174,6 +6238,8 @@ export class AgentFramework { }); const agent = new Agent(config, contextManager, this.membrane); + agent.markContextConsumed(); // RFC-006: nothing stored before now is replaceable + this.sharedSlotAgents.add(agent); const restoredSettings = this.readAgentRuntimeSettings(config.name); if (restoredSettings) { agent.restoreRuntimeSettings( @@ -6248,6 +6314,8 @@ export class AgentFramework { allowedTools: [...SUBCONSCIOUS_TOOL_NAMES, 'think', 'skip_reply', 'end_turn'], }; const agent = new Agent(agentConfig, contextManager, this.membrane); + agent.markContextConsumed(); // RFC-006: nothing stored before now is replaceable + this.sharedSlotAgents.add(agent); // reads the shared slot through its merged view this.agents.set(name, agent); this.agentConfigs.set(name, agentConfig); this.subconsciousAgentName = name; @@ -6657,6 +6725,17 @@ export class AgentFramework { this.handleMcplPushEvent(event as unknown as McplPushEvent); } + await this.dispatchProcessEventToModules(event, startTime); + } + + /** + * The module half of handleProcessEvent: every module's onProcess, their + * responses, the tool-call dispatch, the completion trace and the process + * log. Shared with RFC-006 coalesced deliveries, which bypass the queue + * (the coalescer needs the placement synchronously) but must still reach + * modules exactly like the ordinary deliveries do. + */ + private async dispatchProcessEventToModules(event: ProcessEvent, startTime = Date.now()): Promise { // Dispatch to all modules, tracking responses with module names const responses: ModuleProcessResponse[] = []; for (const module of this.moduleRegistry.getAllModules()) { @@ -6780,9 +6859,16 @@ export class AgentFramework { tags?: string[]; triggerInference?: boolean; targetAgents?: string[]; - }): Promise { + /** RFC-006: set when the coalescer delivers; rides the wake it queues. */ + coalescingSubject?: string; + eventId?: string; + assemblingFor?: string; + deliverTo?: string; + }): Promise { const metadata: Record = { ...event.metadata, + ...(event.eventId ? { eventId: event.eventId } : {}), + ...(event.coalescingSubject ? { coalescingSubject: event.coalescingSubject } : {}), channelId: event.channelId, messageId: event.messageId, author: event.author, @@ -6795,9 +6881,31 @@ export class AgentFramework { // Per-channel conversation routing: messages go to the channel's fork // agent (spawned from the template on first qualifying message), never // to the primary conversation. - if (this.conversationRouter) { - await this.routeConversationIncoming(event, metadata); - return; + // RFC-006 §3.2: a replacement or deletion notice follows the delivery it + // corrects — into the agent that holds (or read) the prior occurrence, + // never a context fresh routing would pick. Gone agent → delivered nowhere. + if (event.deliverTo) { + const target = this.agents.get(event.deliverTo); + if (!target) return undefined; + if (this.conversationRouter && this.conversationAgentHomes.has(target.name)) { + metadata.triggered = event.triggerInference ?? false; + const id = target.getContextManager().addMessage('user', event.content, metadata); + this.emitTrace({ type: 'message:added', messageId: id, source: 'mcpl:channel-incoming' }); + if (event.triggerInference) { + this.pendingRequests.push({ + agentName: target.name, reason: 'mcpl:channel-incoming', source: event.serverId, timestamp: Date.now(), + channelId: event.channelId, + counterparty: event.author?.id ? `${event.serverId}:user:${event.author.id}` : undefined, + addressed: isAddressedMessage(event.tags, event.metadata), + ...(event.coalescingSubject ? { coalescingSubject: event.coalescingSubject } : {}), + }); + } + return { agent: target.name, messageId: id }; + } + if (this.conversationRouter) return undefined; // the trunk never holds channel traffic + event = { ...event, targetAgents: [target.name] }; + } else if (this.conversationRouter) { + return this.routeConversationIncoming(event, metadata); } // Addressed-while-closed invitation — parity with the push-event path @@ -6844,7 +6952,8 @@ export class AgentFramework { metadata.tuneOut = { epochId: divert.epochId }; } - const id = this.addMessage('user', incomingContent, metadata); + const placement: CoalescingPlacement = { agent: '' }; + const id = this.addMessage('user', incomingContent, metadata, { placement, bypassDeferralFor: event.assemblingFor }); this.emitTrace({ type: 'message:added', messageId: id, source: 'mcpl:channel-incoming' }); if (event.triggerInference && !divert) { @@ -6870,9 +6979,11 @@ export class AgentFramework { // Addressed messages outrank ambient chatter when a batched wake // picks the turn's frozen speech locus. addressed, + ...(event.coalescingSubject ? { coalescingSubject: event.coalescingSubject } : {}), }); } } + return placement.agent ? placement : undefined; } /** @@ -6891,9 +7002,10 @@ export class AgentFramework { metadata?: Record; tags?: string[]; triggerInference?: boolean; + coalescingSubject?: string; }, messageMetadata: Record, - ): Promise { + ): Promise { const router = this.conversationRouter!; const descriptor = this.channelRegistry?.getDescriptor(event.channelId); @@ -6970,8 +7082,10 @@ export class AgentFramework { channelId: event.channelId, counterparty: event.author?.id ? `${event.serverId}:user:${event.author.id}` : undefined, addressed: isAddressedMessage(event.tags, event.metadata), + ...(event.coalescingSubject ? { coalescingSubject: event.coalescingSubject } : {}), }); } + return { agent: agent.name, messageId: id }; } /** @@ -7024,6 +7138,8 @@ export class AgentFramework { const config: AgentConfig = { ...templateConfig, name, strategy: undefined }; const agent = new Agent(config, contextManager, this.membrane); + agent.markContextConsumed(); // RFC-006: nothing stored before now is replaceable + agent.markContextConsumed(); // RFC-006: nothing stored before now is replaceable this.agents.set(name, agent); this.agentConfigs.set(name, config); this.conversationAgentHomes.set(name, channelId); @@ -7238,10 +7354,454 @@ export class AgentFramework { }; } + // ========================================================================== + // RFC-006 event coalescing (see mcpl/push-coalescer.ts for the design) + // ========================================================================== + + /** + * RFC-006 §3.2 binding identity: the configured server id bound to its + * endpoint/command. Survives reconnects to the same peer; changes when the + * id is reassigned to another endpoint (restartMcplServer with a new config). + */ + private coalescingBinding(serverId: string): string { + const config = this.mcplServerConfigs.get(serverId); + return createHash('sha256').update(JSON.stringify([serverId, config?.url ?? null, config?.command ?? null, config?.args ?? null])).digest('hex').slice(0, 16); + } + + private initializePushCoalescer(): void { + const coalescer = new PushCoalescer({ + isUnread: (p) => this.isUnreadPlacement(p), + remove: (p) => this.removePlacement(p), + deliver: (occ, materialized, assemblingFor) => this.deliverCoalesced(occ, materialized, assemblingFor), + wakeForBatch: (occ) => this.wakeForCoalescedBatch(occ), + cancelWake: (subject) => this.cancelCoalescedWakes(subject), + authorized: (occ) => this.coalescedAuthorized(occ), + audience: (occ) => this.coalescedAudience(occ), + render: (occ, params) => { + const server = this.mcplServerRegistry?.getServer(occ.serverId); + if (!server?.isConnected) throw new Error('server disconnected'); + return server.sendPushRender(params); + }, + audit: (record) => this.emitTrace({ type: 'mcpl:coalescing', ...record }), + save: (snapshot) => this.scheduleCoalescingSnapshot(snapshot), + recordReceipt: (record) => { + // Written now, not at the throttled flush: an acknowledged acceptance + // must not lose its receipt, the subject's birth, or (deferred) the + // accepted work itself to a crash. A failure here fails the acceptance. + this.coalescingRecentReceipts.push(record); + this.store.setStateJson(COALESCING_RECENT_ID, this.coalescingRecentReceipts); + }, + wasPublished: (subject, eventId) => { + // Boot-time only: the occurrence's durable delivery identity + // (subject + eventId in message metadata) in any agent's context. + for (const agent of this.agents.values()) { + try { + if (agent.getContextManager().getAllMessages().some((m) => + m.metadata?.coalescingSubject === subject && m.metadata?.eventId === eventId)) return true; + } catch { /* a context that cannot be read cannot prove publication */ } + } + return false; + }, + commit: () => this.commitCoalescingDurable(), + saveNow: (snapshot) => { + if (this.coalescingSaveTimer) { clearTimeout(this.coalescingSaveTimer); this.coalescingSaveTimer = null; } + this.coalescingSnapshotDirty = null; + this.store.setStateJson(COALESCING_STATE_ID, snapshot); + this.coalescingRecentReceipts = []; + this.store.setStateJson(COALESCING_RECENT_ID, []); + }, + }); + try { + coalescer.restore(this.store.getStateJson(COALESCING_STATE_ID) as CoalescingSnapshot | null); + const recent = this.store.getStateJson(COALESCING_RECENT_ID) as CoalescingReceiptRecord[] | null; + if (Array.isArray(recent)) { + coalescer.restoreReceipts(recent); + // Until the next snapshot flush checkpoints the recovered state, these + // records are its only durable form: keep them in the writable buffer + // so the next acceptance appends to them instead of replacing them. + this.coalescingRecentReceipts = recent; + } + } catch (err) { + console.error('[coalescing] could not restore coalescing state; starting with unknown history:', err); + } + this.pushCoalescer = coalescer; + // Batches restored from a snapshot still owe the agent a wake (§14 v41). + for (const occ of coalescer.pendingBatchOccurrences()) void this.wakeForCoalescedBatch(occ); + } + + /** + * Durability barrier for coalescing (RFC-006 §3.1/§3.2): everything + * written so far is fsynced before the caller proceeds. Group-committed — + * all acceptances that arrive in one event-loop turn share one + * `store.sync()` — so throughput is bounded by fsync rate per batch, not + * per message. + */ + private commitCoalescingDurable(): Promise { + if (!this.coalescingCommit) { + this.coalescingCommit = new Promise((resolve, reject) => { + setImmediate(() => { + this.coalescingCommit = null; + try { this.store.sync(); resolve(); } catch (err) { reject(err); } + }); + }); + } + return this.coalescingCommit; + } + + private scheduleCoalescingSnapshot(snapshot: () => CoalescingSnapshot): void { + this.coalescingSnapshotDirty = snapshot; + if (this.coalescingSaveTimer) return; + const timer = setTimeout(() => this.flushCoalescingSnapshot(), 2_000); + timer.unref?.(); + this.coalescingSaveTimer = timer; + } + + private flushCoalescingSnapshot(): void { + if (this.coalescingSaveTimer) { clearTimeout(this.coalescingSaveTimer); this.coalescingSaveTimer = null; } + const snapshot = this.coalescingSnapshotDirty; + this.coalescingSnapshotDirty = null; + if (!snapshot) return; + try { + this.store.setStateJson(COALESCING_STATE_ID, snapshot()); + this.coalescingRecentReceipts = []; + this.store.setStateJson(COALESCING_RECENT_ID, []); + } catch (err) { + console.error('[coalescing] could not persist coalescing state:', err); + } + } + + /** Serialize both lanes' admissions: create, edit and delete of one subject stay ordered. */ + private admitCoalesced(work: () => Promise): Promise { + const run = this.coalescingAdmission.then(work, work); + this.coalescingAdmission = run.then(() => undefined, () => undefined); + return run; + } + + private handleCoalescedIncoming( + serverId: string, + message: ChannelIncomingMessage, + event: CoalescedChannelEvent, + ): Promise { + return this.admitCoalesced(async () => { + if (!this.pushCoalescer) throw new CoalesceError('coalesce', 'coalescing unavailable', -32000); + validateCoalesceMember(message.coalesce, 'channel'); + if (typeof message.eventId !== 'string' || !message.eventId) { + throw new CoalesceError('eventId', 'eventId is required with coalesce'); + } + if (typeof message.timestamp !== 'string') throw new CoalesceError('timestamp', 'timestamp is required'); + validateCoalescedContent(message.content); + const c = message.coalesce; + const result = await this.pushCoalescer.accept({ + serverId, + binding: this.coalescingBinding(serverId), + scope: { kind: 'channel', id: message.channelId }, + key: c.key, + eventId: message.eventId, + timestamp: message.timestamp, + retract: !!c.retract, + deferred: false, + initial: !!c.initial, + tags: event.tags, + content: message.content, + identity: { messageId: message.messageId, author: message.author, threadId: message.threadId }, + event: { lane: 'channel', event: { ...event, eventId: message.eventId } }, + }); + return { messageId: message.messageId, accepted: true, coalesce: result.coalesce }; + }); + } + + private handleCoalescedPush(serverId: string, params: PushEventParams, event: McplPushEvent): Promise { + return this.admitCoalesced(async () => { + if (!this.pushCoalescer) throw new CoalesceError('coalesce', 'coalescing unavailable', -32000); + validateCoalesceMember(params.coalesce, 'push'); + if (typeof params.eventId !== 'string' || !params.eventId) throw new CoalesceError('eventId', 'eventId is required'); + if (typeof params.timestamp !== 'string') throw new CoalesceError('timestamp', 'timestamp is required'); + validateCoalescedContent(params.payload?.content); + const c = params.coalesce; + if (c.channelId) this.assertChannelScopedPush(serverId, c.channelId); + const origin = params.origin ?? {}; + const str = (v: unknown): string | undefined => (typeof v === 'string' && v ? v : undefined); + return this.pushCoalescer.accept({ + serverId, + binding: this.coalescingBinding(serverId), + featureSet: params.featureSet, + scope: c.channelId ? { kind: 'channel', id: c.channelId } : { kind: 'featureSet', id: params.featureSet }, + key: c.key, + eventId: params.eventId, + timestamp: params.timestamp, + retract: !!c.retract, + deferred: !!c.deferred, + initial: !!c.initial, + data: c.data, + tags: event.tags, + content: params.payload.content, + ...(c.channelId ? { + identity: { + ...(str(origin.messageId) ? { messageId: str(origin.messageId) } : {}), + ...(str(origin.threadId) ? { threadId: str(origin.threadId) } : {}), + ...(str(origin.authorId) ? { author: { id: str(origin.authorId)!, name: str(origin.authorName) ?? str(origin.authorId)! } } : {}), + }, + } : {}), + event: { lane: 'push', event }, + }); + }); + } + + /** §3.2: a channel-scoped push needs what a channels/incoming message needs. */ + private assertChannelScopedPush(serverId: string, channelId: string): void { + const server = this.mcplServerRegistry?.getServer(serverId); + if (!server?.policyEstablished || !server.grant.has('channels.incoming')) { + throw new CoalesceError('coalesce.channelId', 'Channel not permitted', -32017); + } + if (!this.channelRegistry?.isDeclaredChannel(serverId, channelId)) { + throw new CoalesceError('coalesce.channelId', 'Unknown channel', -32023); + } + } + + private coalescedAuthorized(occ: CoalescedOccurrence): boolean { + const server = this.mcplServerRegistry?.getServer(occ.serverId); + if (!server?.isConnected || !server.policyEstablished) return false; + // §3.2: work admitted under another binding (the id since reassigned to a + // different endpoint) never acquires the replacement peer's authority — + // and its private notice data is never sent to that peer. + if (occ.binding !== this.coalescingBinding(occ.serverId)) return false; + if (occ.event.lane === 'push') { + if (!server.grant.has('pushEvents')) return false; + try { this.featureSetManager?.validateInbound(occ.serverId, occ.event.event.featureSet); } catch { return false; } + } + if (occ.scope.kind === 'channel' || occ.event.lane === 'channel') { + // §3.2 / §5.4 rule 4: the channel's CURRENT authority, whichever lane. + if (!server.grant.has('channels.incoming')) return false; + if (occ.event.lane === 'push') return this.channelRegistry?.isDeclaredChannel(occ.serverId, occ.scope.id) ?? false; + return !!this.channelRegistry?.getDescriptor(occ.scope.id); + } + return true; + } + + private async coalescedAudience(occ: CoalescedOccurrence): Promise { + if (occ.deliverTo) return this.agents.has(occ.deliverTo) ? [occ.deliverTo] : []; + const explicit = occ.event.event.targetAgents; + if (explicit) return explicit.filter((n) => this.agents.has(n)); + if (occ.scope.kind === 'channel' && this.conversationRouter) { + const router = this.conversationRouter; + const bound = router.getBinding(occ.scope.id)?.agentName; + if (bound) return this.agents.has(bound) ? [bound] : []; + // No fork yet: the same bind decision an incoming message would get. A + // batch that qualifies spawns its fork now so it has someone to wake. + const metadata = (occ.event.lane === 'channel' ? occ.event.event.metadata : occ.event.event.origin) ?? {}; + const decision = router.route({ + channelId: occ.scope.id, + mentioned: metadata.mentioned === true, + kind: ConversationRouter.classifyChannel(this.channelRegistry?.getDescriptor(occ.scope.id), metadata), + }); + if (decision.kind !== 'spawn') return decision.kind === 'unbound' ? [] : (this.agents.has(decision.agentName) ? [decision.agentName] : []); + try { + await this.createConversationAgent(decision.agentName, occ.scope.id); + router.bind(occ.scope.id, decision.agentName, decision.generation); + this.persistConversationRouterState(); + return [decision.agentName]; + } catch (err) { + console.error(`[coalescing] could not spawn a conversation agent for ${occ.scope.id}:`, err); + return []; + } + } + return [...this.agents.keys()].filter((n) => !this.conversationAgentHomes.has(n) && n !== this.subconsciousAgentName); + } + + /** Deliver through the lane's ordinary path; report where it landed. */ + private async deliverCoalesced( + occ: CoalescedOccurrence, + materialized?: McplContentBlock[], + assemblingFor?: string, + ): Promise { + const coalescingSubject = coalescingSubjectKey(occ.serverId, occ.binding, occ.scope, occ.key); + const narrowing = occ.deliverTo ? { deliverTo: occ.deliverTo } : {}; + if (occ.event.lane === 'channel') { + const event = { ...occ.event.event, coalescingSubject, ...narrowing, ...(assemblingFor ? { assemblingFor } : {}) }; + const placement = await this.handleMcplChannelIncoming(event); + await this.dispatchProcessEventToModules(event as unknown as ProcessEvent); + return placement; + } + if (occ.scope.kind === 'channel') { + // §3.2: a channel-scoped push addresses the channel's subject namespace, + // so it is delivered as a message OF that channel — same routing + // (conversation forks, tune-out, closed-channel invitation), same reply + // locus, and the subject's stable platform identity (§3.1, vector 43). + const push = occ.event.event; + const identity = { ...this.pushCoalescer?.identity(coalescingSubject), ...occ.identity }; + const origin = push.origin ?? {}; + const str = (v: unknown): string | undefined => (typeof v === 'string' && v ? v : undefined); + const event: CoalescedChannelEvent = { + type: 'mcpl:channel-incoming', + serverId: occ.serverId, + channelId: occ.scope.id, + messageId: identity.messageId ?? str(origin.messageId) ?? occ.eventId, + threadId: identity.threadId ?? str(origin.threadId), + author: identity.author ?? { id: str(origin.authorId) ?? '', name: str(origin.authorName) ?? str(origin.authorId) ?? 'Someone' }, + content: materialized ? materialized.map(convertPushBlock) : push.content, + timestamp: occ.timestamp, + metadata: origin, + tags: push.tags, + triggerInference: assemblingFor ? false : push.triggerInference, + targetAgents: push.targetAgents, + coalescingSubject, + eventId: occ.eventId, + ...narrowing, + ...(assemblingFor ? { assemblingFor } : {}), + }; + const placement = await this.handleMcplChannelIncoming(event); + await this.dispatchProcessEventToModules(event as unknown as ProcessEvent); + return placement; + } + const event: McplPushEvent = { + ...occ.event.event, + eventId: occ.eventId, + timestamp: occ.timestamp, + ...(materialized ? { content: materialized.map(convertPushBlock) } : {}), + coalescingSubject, + // §5.2: the materialized occurrence is consumed by the request being + // assembled; the wake that started it is not repeated. + ...(assemblingFor ? { assemblingFor, triggerInference: false } : {}), + ...(occ.deliverTo ? { targetAgents: [occ.deliverTo] } : {}), + }; + const placement = this.handleMcplPushEvent(event); + await this.dispatchProcessEventToModules(event as unknown as ProcessEvent); + return placement; + } + + private async wakeForCoalescedBatch(occ: CoalescedOccurrence): Promise { + const subject = coalescingSubjectKey(occ.serverId, occ.binding, occ.scope, occ.key); + this.cancelCoalescedWakes(subject); + if (occ.event.lane !== 'push' || !occ.event.event.triggerInference) return; + const origin = occ.event.event.origin; + // The wake carries the batch's channel so the turn freezes its reply + // locus there (the materialized delivery at assembly wakes nobody). + const channelId = occ.scope.kind === 'channel' ? occ.scope.id : this.derivePushEventChannel(origin)?.channelId; + const authorId = occ.identity?.author?.id ?? (typeof origin?.authorId === 'string' ? origin.authorId : undefined); + for (const agentName of await this.coalescedAudience(occ)) { + this.pendingRequests.push({ + agentName, + reason: 'mcpl:push-event', + source: occ.serverId, + timestamp: Date.now(), + channelId, + counterparty: authorId ? `${occ.serverId}:user:${authorId}` : undefined, + addressed: isAddressedMessage(occ.tags, origin), + coalescingSubject: subject, + }); + } + } + + /** Withdraw unstarted wakes whose sole cause is this subject (§4.2). */ + private cancelCoalescedWakes(subject: string): void { + this.pendingRequests = this.pendingRequests.filter((r) => r.coalescingSubject !== subject); + for (const cooldown of this.providerAccelerationCooldowns.values()) { + cooldown.heldRequests = cooldown.heldRequests.filter((r) => r.coalescingSubject !== subject); + } + } + + /** + * A placement in the deferred queue is upgraded in place once the flush has + * stored it: the stored copy carries `metadata.deferredWriteId`, found by + * scanning the unread tail (above the watermark) — never below it. + */ + private resolvePlacement(p: CoalescingPlacement): void { + if (!p.deferredId || this.deferredMessages.some((m) => m.id === p.deferredId)) return; + const agent = this.agents.get(p.agent); + if (!agent) return; + const cm = agent.getContextManager(); + const watermark = agent.getConsumedWatermark(); + const floor = watermark && watermark.branch === cm.currentBranch().name ? watermark.sequence : Number.POSITIVE_INFINITY; + const all = cm.getAllMessages(); + for (let i = all.length - 1; i >= 0 && all[i].sequence > floor; i--) { + if (all[i].metadata?.deferredWriteId === p.deferredId) { + p.messageId = all[i].id; + delete p.deferredId; + return; + } + } + } + + private isUnreadPlacement(p: CoalescingPlacement): boolean { + this.resolvePlacement(p); + if (p.deferredId) return this.deferredMessages.some((m) => m.id === p.deferredId); + if (!p.messageId) return false; + const agent = this.agents.get(p.agent); + return !!agent && this.isUnreadStoredMessage(agent, p.messageId); + } + + /** + * Unread = stored, above the agent's consumed watermark on the current + * branch, not a body-group shard (immutable), and not folded into any + * summary (compression is consumption, §3.3). Conservative on every doubt. + */ + private isUnreadStoredMessage(agent: Agent, messageId: MessageId): boolean { + const cm = agent.getContextManager(); + let message: ReturnType; + try { message = cm.getMessage(messageId); } catch { return false; } + if (!message || message.bodyGroupId) return false; + // Every agent that can read this message must not have compiled past it. + // Residents share one message slot, so a message one of them read is + // history for all of them (the never-rewrite rule is about the message, + // not the agent that happened to be asked). A tune-out-diverted message + // never enters a resident's compiled view; only the subconscious reads + // it through its merged view — no subconscious, nobody reads it. + let readers: Agent[]; + if ((message.metadata as { tuneOut?: unknown } | undefined)?.tuneOut) { + const subconscious = this.subconsciousAgentName ? this.agents.get(this.subconsciousAgentName) : undefined; + if (!subconscious) return true; + readers = [subconscious]; + } else { + readers = this.sharedSlotAgents.has(agent) + ? [...this.sharedSlotAgents].filter((a) => this.agents.get(a.name) === a) + : [agent]; + } + const branch = cm.currentBranch().name; + const ts = message.timestamp instanceof Date ? message.timestamp.getTime() : Number(message.timestamp); + for (const reader of readers) { + const watermark = reader.getConsumedWatermark(); + if (!watermark || watermark.branch !== branch || message.sequence <= watermark.sequence) return false; + // Compression is consumption (§3.3): folded into any reader's summary. + try { + const rcm = reader.getContextManager(); + const strategy = rcm.getStrategy() as { getSummary?: (id: string) => { sourceIds?: string[]; sourceLevel?: number } | null }; + for (const summary of rcm.getSummariesInRange({ fromMs: ts, toMs: ts })) { + const entry = strategy.getSummary?.(summary.id); + if (!entry || (entry.sourceLevel ?? 0) > 0 || entry.sourceIds?.includes(messageId)) return false; + } + } catch { + return false; + } + } + return true; + } + + private removePlacement(p: CoalescingPlacement): boolean { + this.resolvePlacement(p); + if (p.deferredId) { + const before = this.deferredMessages.length; + this.deferredMessages = this.deferredMessages.filter((m) => m.id !== p.deferredId); + if (this.deferredMessages.length === before) return false; + if (this.quiesced || this.deferredWritesPersisted) this.persistDeferredWrites(); + return true; + } + if (!p.messageId) return false; + const agent = this.agents.get(p.agent); + if (!agent) return false; + try { + agent.getContextManager().removeMessage(p.messageId); + this.emitTrace({ type: 'message:removed', messageId: p.messageId, source: 'mcpl:coalescing' }); + return true; + } catch (err) { + console.error(`[coalescing] could not remove unread message ${p.messageId} for ${agent.name}:`, err); + return false; + } + } + /** * Convert an MCPL push event to a context message. */ - private handleMcplPushEvent(event: McplPushEvent): void { + private handleMcplPushEvent(event: McplPushEvent): CoalescingPlacement | undefined { const triggerChannel = this.derivePushEventChannel(event.origin); if (triggerChannel && this.channelRegistry) { this.channelRegistry.ensureChannelRegistered( @@ -7259,6 +7819,7 @@ export class AgentFramework { eventId: event.eventId, triggered: event.triggerInference ?? false, ...(event.tags ? { tags: event.tags } : {}), + ...(event.coalescingSubject ? { coalescingSubject: event.coalescingSubject } : {}), }; // `origin.channelId` is the server-internal raw id (a bare Discord // snowflake) — unroutable as a locus and unresolvable by the agent. Store @@ -7291,7 +7852,8 @@ export class AgentFramework { } } - const id = this.addMessage('user', content, metadata); + const placement: CoalescingPlacement = { agent: '' }; + const id = this.addMessage('user', content, metadata, { placement, bypassDeferralFor: event.assemblingFor }); this.emitTrace({ type: 'message:added', messageId: id, source: 'mcpl:push-event' }); if (event.triggerInference) { @@ -7310,9 +7872,11 @@ export class AgentFramework { // they must outrank ambient chatter in a batched wake's locus // selection just like their channels/incoming counterparts. addressed: isAddressedMessage(event.tags, event.origin), + ...(event.coalescingSubject ? { coalescingSubject: event.coalescingSubject } : {}), }); } } + return placement.agent ? placement : undefined; } /** @@ -7470,7 +8034,9 @@ export class AgentFramework { // route the resumed turn into the wrong channel. const newestByKey = new Map(); for (const request of parked) { - const key = `${request.reason}|${request.addressed ? 'a' : 'n'}`; + // A coalesced wake is withdrawable by its subject (RFC-006 §4.2); it + // must not absorb an unrelated wake that would vanish with it. + const key = `${request.reason}|${request.addressed ? 'a' : 'n'}|${request.coalescingSubject ?? ''}`; const prev = newestByKey.get(key); if (!prev || request.timestamp >= prev.timestamp) { newestByKey.set(key, request); @@ -8207,6 +8773,18 @@ export class AgentFramework { } } + // RFC-006 §5: deferred batches for this agent render now, at the assembly + // boundary — after the deferred flush (same window: turn alive, compile + // not yet run) and before the checkpoint (they are the turn's inputs). + // Not on a context-budget restart: that continues the same logical turn. + if (attempt === 0 && trigger?.reason !== 'context_budget_restart' && this.pushCoalescer?.pendingBatches()) { + try { + await this.pushCoalescer.assemble(agent.name); + } catch (err) { + console.error(`[coalescing] assembly for ${agent.name} failed:`, err); + } + } + // Record turn checkpoint before inference (only on first attempt, not retries) if (attempt === 0) { this.recordTurnCheckpoint(agent.name); @@ -11255,6 +11833,14 @@ export class AgentFramework { * minting a second, independently replayable one. */ deferredWriteId?: string; + /** Out-param: where the message landed (RFC-006 needs to find it again). */ + placement?: { agent?: string; messageId?: MessageId; deferredId?: string }; + /** + * RFC-006 assembly: the named agent's turn is alive but its compile has + * not run yet (the same window the turn-start deferred flush writes + * in). A message for THAT agent is stored directly instead of deferred. + */ + bypassDeferralFor?: string; } ): MessageId { // Route to the named agent, else the primary (not ephemeral subagents). @@ -11304,6 +11890,7 @@ export class AgentFramework { : this.pendingAssistantBlocks.size > 0; if ( !hasToolResult && + opts?.bypassDeferralFor !== agent.name && (this.quiesced || midToolCycle || this.activeTurnTokens.has(agent.name) || @@ -11320,14 +11907,17 @@ export class AgentFramework { } this.deferredMessages.push({ id, seq: seq ?? ++this.deferredSeq, participant, content, metadata, forAgent: opts?.forAgent }); if (this.quiesced || this.deferredWritesPersisted) this.persistDeferredWrites(); + if (opts?.placement) { opts.placement.agent = agent.name; opts.placement.deferredId = id; } return '' as MessageId; // Deferred — flushed at the target's next boundary } - return agent.getContextManager().addMessage( + const stored = agent.getContextManager().addMessage( participant, content, opts?.deferredWriteId ? withDeferredWriteId(metadata, opts.deferredWriteId) : metadata, ); + if (opts?.placement) { opts.placement.agent = agent.name; opts.placement.messageId = stored; } + return stored; } /** @@ -11884,11 +12474,13 @@ export class AgentFramework { ?? (this.eventGate ? this.eventGate.asShouldTriggerCallback() : undefined); // Push events handler (Step 6) + this.initializePushCoalescer(); this.pushHandler = new PushHandler( this.featureSetManager, (event) => this.pushEvent(event as unknown as ProcessEvent), (event) => this.emitTrace(event as { type: TraceEvent['type']; [key: string]: unknown }), triggerFilter, + (serverId, params, event) => this.handleCoalescedPush(serverId, params, event), ); // Server-initiated inference router (Step 6) @@ -11918,6 +12510,8 @@ export class AgentFramework { (event) => this.emitTrace(event as { type: TraceEvent['type']; [key: string]: unknown }), { store: this.store, + handleCoalescedIncoming: (serverId, message, event) => + this.handleCoalescedIncoming(serverId, message, event as CoalescedChannelEvent), sendTypingFn: (serverId, channelId, metadata, op) => { const server = this.mcplServerRegistry!.getServer(serverId); if (server) { @@ -11992,6 +12586,9 @@ export class AgentFramework { // implemented for years — are declared (AUDIT-001/issue #76 item 12). this.mcplHostCapabilities = { version: '0.5', + // RFC-006 (mcpl PR #5, revision 7): both lanes, deferred rendering, + // channel-scoped pushes, one-hour retry window. + eventCoalescing: { ...PUSH_COALESCING_SUPPORT }, pushEvents: true, contextHooks: { beforeInference: true, @@ -12968,7 +13565,7 @@ export class AgentFramework { ) => { const barrier = this.discordAwarenessBarrier; if (barrier) await barrier.promise; - this.pushHandler?.handlePushEvent(connection.id, params, responder as never); + await this.pushHandler?.handlePushEvent(connection.id, params, responder as never); }); // Handle server-initiated inference requests (Step 6) @@ -12976,6 +13573,11 @@ export class AgentFramework { params: McplInferenceRequestParams, responder?: { id: string | number; respond: (result: unknown) => void; respondError: (code: number, message: string) => void }, ) => { + // RFC-006 §5.3 / SPEC §10.7: no inference from inside a render. + if (this.pushCoalescer?.isRendering(connection.id)) { + responder?.respondError(-32600, 'Cannot request inference while push/render is in progress'); + return; + } if (this.inferenceRouter && responder) { const barrier = this.discordAwarenessBarrier; if (barrier) await barrier.promise; @@ -13066,7 +13668,7 @@ export class AgentFramework { ) => { const barrier = this.discordAwarenessBarrier; if (barrier) await barrier.promise; - this.channelRegistry?.handleIncoming(connection.id, params, responder as never); + await this.channelRegistry?.handleIncoming(connection.id, params, responder as never); }); // Handle host-level admin commands from a surface (e.g. Discord /undo) diff --git a/src/mcpl/channel-registry.ts b/src/mcpl/channel-registry.ts index ee1f9b2d3..01c81ec1a 100644 --- a/src/mcpl/channel-registry.ts +++ b/src/mcpl/channel-registry.ts @@ -26,6 +26,7 @@ import type { ChannelsIncomingParams, ChannelsIncomingResult, ChannelIncomingMessageResult, + ChannelIncomingMessage, ChannelsPublishParams, ChannelsOpenResult, ChannelHistoryRequest, @@ -36,6 +37,7 @@ import type { McplServerRegistry } from './server-registry.js'; import type { FeatureSetManager } from './feature-set-manager.js'; import type { ToolDefinition, ToolResult, ProcessEvent } from '../types/index.js'; import { expandCoreTags } from './tags.js'; +import { validateCoalescedContent } from './push-coalescer.js'; import { CapabilityGrant } from './capability-grant.js'; // ============================================================================ @@ -452,6 +454,17 @@ const CHANNEL_TOOL_DEFINITIONS: ToolDefinition[] = [ // ============================================================================ interface ChannelRegistryOptions { + /** + * RFC-006: an admitted `channels/incoming` message carrying `coalesce`, with + * the event the ordinary path would have queued. The handler decides + * replace / append / withdraw and returns the per-message result. Throws a + * `CoalesceError` for a malformed or unauthorized occurrence. + */ + handleCoalescedIncoming?: ( + serverId: string, + message: ChannelIncomingMessage, + event: McplChannelIncomingEvent, + ) => Promise; /** Chronicle store used for durable desired channel lifecycle state. */ store?: JsStore; /** Callback to determine whether an incoming message should trigger inference. */ @@ -643,6 +656,7 @@ export class ChannelRegistry { /** One-time migration inputs from the retired recipe auto-open policy. */ private legacyPolicies = new Map(); private migratedLegacyPolicies = new Set(); + private handleCoalescedIncoming?: ChannelRegistryOptions['handleCoalescedIncoming']; constructor( serverRegistry: McplServerRegistry, @@ -658,6 +672,7 @@ export class ChannelRegistry { ) => void; }, ) { + this.handleCoalescedIncoming = options?.handleCoalescedIncoming; this.serverRegistry = serverRegistry; this.featureSetManager = featureSetManager; this.pushEventFn = pushEventFn; @@ -838,14 +853,23 @@ export class ChannelRegistry { * Converts each message's content, pushes McplChannelIncomingEvent to the * queue, and responds with per-message results. */ - handleIncoming( + async handleIncoming( serverId: string, params: ChannelsIncomingParams, responder?: Responder, - ): void { + ): Promise { const results: ChannelIncomingMessageResult[] = []; for (const message of params.messages) { + if (!message || typeof message.channelId !== 'string' || !message.channelId + || typeof message.messageId !== 'string' || !message.messageId) { + results.push({ + messageId: typeof message?.messageId === 'string' ? message.messageId : '', + accepted: false, + reason: message?.coalesce !== undefined ? 'coalesce_invalid' : 'invalid channel/message identity', + }); + continue; + } // §14.5 FIRST, before ANY semantic processing: admission against the // actually-registered channel precedes tag expansion and content // conversion — decoding an unregistered sender's payload (including @@ -890,6 +914,17 @@ export class ChannelRegistry { // ACCEPTED from here down: semantic processing only for admitted // messages. §16.3 core-tag closure, then content conversion. + const coalesced = message.coalesce !== undefined && !!this.handleCoalescedIncoming; + if (coalesced) { + // RFC-006 §13: malformed content on a coalesced item is that item's + // failure, not the batch's — check the shape before converting. + try { + validateCoalescedContent(message.content); + } catch (error) { + results.push({ messageId: message.messageId, accepted: false, reason: 'coalesce_invalid' }); + continue; + } + } if (message.tags) message.tags = expandCoreTags(message.tags); const convertedContent: ContentBlock[] = message.content.map(convertBlock); @@ -897,16 +932,19 @@ export class ChannelRegistry { // deliberately after §14.5 validation: a rejected message from an // unregistered channel must not retarget outbound speech (the locus is // exactly the authority a self-attested channel would be stealing). - this.defaultPublishChannel = message.channelId; - this.defaultPublishMessageId = message.messageId; - this.defaultPublishThreadId = message.threadId; - { + // A coalesced item is "accepted" only once the coalescer admits it, so + // for those this runs after the hook (below). + const markAccepted = () => { + this.defaultPublishChannel = message.channelId; + this.defaultPublishMessageId = message.messageId; + this.defaultPublishThreadId = message.threadId; // A server sending channels/incoming is authoritative evidence that // the transport is actually open. This repairs transient status only; // durable desired state still changes exclusively through lifecycle // operations. this.channels.get(incomingKey)!.open = true; - } + }; + if (!coalesced) markAccepted(); // Determine whether to trigger inference let triggerInference = true; @@ -945,6 +983,26 @@ export class ChannelRegistry { triggerInference, }; + if (coalesced) { + // RFC-006 §14.3: a coalesced message is admitted like any other and + // then handed, with the event the ordinary path would have queued, to + // the coalescer, which replaces, appends or withdraws. Malformed + // `coalesce` is a per-message failure; siblings are unaffected. + try { + const result = await this.handleCoalescedIncoming!(serverId, message, event); + if (result.accepted) markAccepted(); + results.push(result); + } catch (error) { + const err = error as Error & { code?: number }; + results.push({ + messageId: message.messageId, + accepted: false, + reason: err.code === -32602 ? 'coalesce_invalid' : err.message, + }); + } + continue; + } + // Push to the processing queue // Cast through unknown because McplChannelIncomingEvent matches the // CustomEvent `${string}:${string}` type pattern but lacks an index signature. @@ -1155,6 +1213,18 @@ export class ChannelRegistry { * Get the descriptor for a channel by its channelId (first match across * servers). Used by the conversation router for DM classification. */ + /** + * RFC-006 §3.2: is `channelId` registered BY `serverId` through a server + * declaration (channels/register, channels/changed) or an authorized open? + * A placeholder minted from a push's `origin` (ensureChannelRegistered) + * does not count: a channel id appearing in an untrusted field is not a + * registration. + */ + isDeclaredChannel(serverId: string, channelId: string): boolean { + const entry = this.channels.get(`${serverId}:${channelId}`); + return !!entry && !(entry.descriptor.metadata as { lazyRegistered?: boolean } | undefined)?.lazyRegistered; + } + getDescriptor(channelId: string): ChannelDescriptor | undefined { return this.findChannelEntry(channelId)?.descriptor; } diff --git a/src/mcpl/push-coalescer.ts b/src/mcpl/push-coalescer.ts new file mode 100644 index 000000000..9536962f7 --- /dev/null +++ b/src/mcpl/push-coalescer.ts @@ -0,0 +1,821 @@ +/** + * RFC-006 event coalescing (mcpl PR #5, revision 7): replace-if-unread, deferred + * rendering, and atomic retraction, for both delivery lanes. + * + * The design keeps content IN context. A plain occurrence is delivered through + * the ordinary channel/push path the moment it is admitted, exactly as today; + * the coalescer only remembers WHERE it landed. A later occurrence of the same + * subject edits or removes that message in place while it is still unread + * (above the agent's consumed watermark, not folded by compression), and + * appends once it has been read. Nothing is staged, journaled or capped: on + * any uncertainty (restart, branch switch, eviction) the host behaves as an + * append-only host would, which is what the RFC's principle 4 asks for. + * + * Only deferred batches (§5) wait outside context, because a notice has no + * model-visible content until `push/render` produces it. + * + * All host effects go through {@link CoalescerHost}; this class is pure + * bookkeeping and is unit-tested without a framework. + */ +import type { McplContentBlock, PushEventResult } from './types.js'; + +export const PUSH_COALESCING_SUPPORT = { + pushEvents: true, channelsIncoming: true, deferred: true, channelScopedPush: true, + retryWindowMs: 3_600_000, +} as const; + +export const COALESCE_OUTCOMES = ['first', 'replaced', 'appended', 'retracted', 'noted', 'consumed'] as const; +export type CoalesceOutcome = typeof COALESCE_OUTCOMES[number]; + +class CancelledRender extends Error {} + +export class CoalesceError extends Error { + constructor(readonly field: string, message: string, readonly code = -32602) { + super(message); + this.name = 'CoalesceError'; + } +} + +/** Where an unread occurrence currently lives. */ +export interface CoalescingPlacement { + agent: string; + /** Stored message id in that agent's context manager. */ + messageId?: string; + /** Entry id in the framework's deferred-write queue (turn-alive deferral). */ + deferredId?: string; +} + +export interface CoalescingScope { kind: 'featureSet' | 'channel'; id: string } + +/** One admitted occurrence, normalized from either lane. */ +export interface CoalescedOccurrence { + serverId: string; + /** Host-owned binding identity (§3.2): stable across reconnects to the same + * configured peer, different when the id is reassigned to another one. */ + binding: string; + /** Declaring feature set (push lane), echoed in push/render. */ + featureSet?: string; + scope: CoalescingScope; + key: string; + eventId: string; + timestamp: string; + retract: boolean; + deferred: boolean; + initial: boolean; + data?: unknown; + tags?: string[]; + /** Wire content (fallback for deferred; the notice for retract). */ + content: McplContentBlock[]; + /** Stable platform identity for channel subjects (§3.1). */ + identity?: { messageId?: string; author?: { id: string; name: string }; threadId?: string }; + /** The event the ordinary path would have queued — the host delivers it. */ + event: E; + /** Set by the coalescer: deliver into this agent only (§3.2 audience may + * narrow, never widen — a replacement or notice follows the prior delivery). */ + deliverTo?: string; +} + +export interface DeferredNotice { eventId: string; timestamp: string; data?: unknown } + +export interface PushRenderParams { + featureSet: string; + channelId?: string; + key: string; + eventId: string; + notices: DeferredNotice[]; + dropped: number; +} +export interface PushRenderResult { content: McplContentBlock[]; timestamp?: string } + +interface DeferredBatch { + /** Latest notice; its content is the fallback, its tags/event drive delivery. */ + latest: CoalescedOccurrence; + notices: DeferredNotice[]; + dropped: number; + /** Set on a batch restored from a snapshot whose render may have started. */ + noRender?: boolean; +} + +interface Rendering { + batch: DeferredBatch; + cancelled: boolean; + done: Promise; +} + +interface SubjectState { + history: 'none' | 'some' | 'unknown'; + occupant?: { eventId: string; placement: CoalescingPlacement }; + batch?: DeferredBatch; + rendering?: Rendering; + consumedEventId?: string; + identity?: CoalescedOccurrence['identity']; + /** Agent that received the last delivery; replacements and notices follow it. */ + audienceAgent?: string; + touchedAt: number; +} + +export interface CoalescingSnapshot { + version: 1; + receipts: Array<[string, { result: PushEventResult; at: number }]>; + subjects: Array<{ + subject: string; + history: SubjectState['history']; + consumedEventId?: string; + identity?: CoalescedOccurrence['identity']; + occupant?: { eventId: string; placement: CoalescingPlacement }; + audienceAgent?: string; + batch?: { latest: CoalescedOccurrence; notices: DeferredNotice[]; dropped: number; rendering?: boolean }; + }>; +} +/** One receipt, written synchronously at acceptance (bridges the snapshot throttle). */ +export interface CoalescingReceiptRecord { + key: string; + result: PushEventResult; + at: number; + subject: string; + /** History the subject was created with, for a subject first seen by this receipt. */ + born?: SubjectState['history']; + /** The subject's pending batch AFTER this acceptance: the accepted work + * itself (so a crash before the snapshot flush cannot lose it while keeping + * the receipt that suppresses its retry), or `null` when the acceptance + * left no batch — a retraction or plain replacement must be recoverable + * too. Records replay in order, so the last one per subject wins. */ + batch: { latest: CoalescedOccurrence; notices: DeferredNotice[]; dropped: number } | null; +} + +export interface CoalescerHost { + /** True iff the placement still exists and no model request has included it. */ + isUnread(placement: CoalescingPlacement): boolean; + /** False when the unread occurrence could not be removed (it may still be read). */ + remove(placement: CoalescingPlacement): boolean; + /** Ordinary delivery of the event; undefined when routing delivered nowhere. + * A replacement is delivered as a fresh message after the unread prior was + * removed: it lands where a fresh event lands (§4.1 allows either place). */ + deliver(occurrence: CoalescedOccurrence, materialized?: McplContentBlock[], assemblingFor?: string): Promise; + /** Queue a wake for a batch that has no model-visible content yet. */ + wakeForBatch(occurrence: CoalescedOccurrence): Promise; + /** Withdraw unstarted wakes whose sole cause is this subject. */ + cancelWake(subject: string): void; + /** The batch's audience still holds the authority it was admitted under. */ + authorized(occurrence: CoalescedOccurrence): boolean; + /** Agents that read this occurrence's delivery target (may spawn a fork). */ + audience(occurrence: CoalescedOccurrence): Promise; + render(occurrence: CoalescedOccurrence, params: PushRenderParams): Promise; + audit(record: Record): void; + /** Persist (throttled by the host); the thunk builds the snapshot lazily. */ + save(snapshot: () => CoalescingSnapshot): void; + /** Persist the snapshot NOW (a render is about to start: that boundary must + * be recoverable before the RPC is issued, §3.2). Throws on failure. */ + saveNow?(snapshot: CoalescingSnapshot): void; + /** Persist one receipt now, before the acceptance is acknowledged. Throws + * on failure, which fails the acceptance (the producer retries). */ + recordReceipt?(record: CoalescingReceiptRecord): void; + /** Make every write so far DURABLE (fsync, group-committed by the host). + * Awaited before an acceptance is acknowledged and before push/render is + * issued: a kill after either boundary must find the receipt, the + * accepted work, and the render-start on disk. Rejects on failure. */ + commit?(): Promise; + /** Recovery: is this occurrence's content in a context (its durable + * delivery identity — subject + eventId in message metadata — is found + * there)? A published occurrence is read from recovery on (the watermark + * restarts at the head), whatever a stale record says about its history. */ + wasPublished?(subject: string, eventId: string): boolean; + now?(): number; +} + +export interface CoalescerOptions { + retryWindowMs?: number; + /** Base delay before re-waking a batch whose render-start commit failed + * (doubles per attempt, capped at 60× base). Default 1 s. */ + recoveryBackoffMs?: number; + maxSubjects?: number; + maxNotices?: number; + maxContentBytes?: number; + maxDataBytes?: number; +} + +export function coalescingSubjectKey(serverId: string, binding: string, scope: CoalescingScope, key: string): string { + return JSON.stringify([serverId, binding, scope.kind, scope.id, key]); +} +export function coalescingReceiptKey(serverId: string, binding: string, eventId: string): string { + return JSON.stringify([serverId, binding, eventId]); +} +/** Merge identity fields, never letting an absent field erase a known one (§3.1). */ +function mergeIdentity(prior: CoalescedOccurrence['identity'], next: CoalescedOccurrence['identity']): CoalescedOccurrence['identity'] { + const out = { ...prior }; + for (const [k, v] of Object.entries(next ?? {})) if (v !== undefined) (out as Record)[k] = v; + return out; +} + +/** Validate wire content before it is stored, rendered or converted. */ +export function validateCoalescedContent(content: unknown, maxBytes = 1024 * 1024): asserts content is McplContentBlock[] { + if (!Array.isArray(content)) throw new CoalesceError('payload.content', 'content must be an array'); + for (const b of content as Array>) { + if (!b || typeof b !== 'object' || Array.isArray(b)) throw new CoalesceError('payload.content', 'invalid content block'); + if (b.type === 'text' && typeof b.text === 'string') continue; + if (b.type === 'resource' && typeof b.uri === 'string') continue; + if ((b.type === 'image' || b.type === 'audio') + && (typeof b.uri === 'string' || (typeof b.data === 'string' && typeof b.mimeType === 'string'))) continue; + throw new CoalesceError('payload.content', 'invalid content block'); + } + if (Buffer.byteLength(JSON.stringify(content)) > maxBytes) throw new CoalesceError('payload.content', 'content exceeds host byte limit'); +} + +/** Validate the `coalesce` member of either lane (§13). */ +export function validateCoalesceMember( + c: unknown, + lane: 'push' | 'channel', + maxDataBytes = 4096, +): asserts c is NonNullable { + if (!c || typeof c !== 'object' || Array.isArray(c)) throw new CoalesceError('coalesce', 'coalesce must be an object'); + const m = c as Record; + if (typeof m.key !== 'string' || !Buffer.byteLength(m.key) || Buffer.byteLength(m.key) > 256) { + throw new CoalesceError('coalesce.key', 'key must be 1..256 UTF-8 bytes'); + } + for (const field of ['deferred', 'retract', 'initial'] as const) { + if (m[field] !== undefined && typeof m[field] !== 'boolean') throw new CoalesceError(`coalesce.${field}`, 'expected boolean'); + } + if (m.channelId !== undefined && (typeof m.channelId !== 'string' || !m.channelId)) { + throw new CoalesceError('coalesce.channelId', 'channelId must be a non-empty string'); + } + if (lane === 'channel' && ('channelId' in m || 'deferred' in m)) { + throw new CoalesceError('coalesce', 'channel messages cannot select scope or deferred mode'); + } + if (m.deferred && m.retract) throw new CoalesceError('coalesce', 'deferred and retract are exclusive'); + if (m.initial && m.retract) throw new CoalesceError('coalesce', 'initial and retract are exclusive'); + if (m.data !== undefined && !m.deferred) throw new CoalesceError('coalesce.data', 'data requires deferred'); + if (m.data !== undefined && Buffer.byteLength(JSON.stringify(m.data)) > maxDataBytes) { + throw new CoalesceError('coalesce.data', `data exceeds ${maxDataBytes} bytes`); + } +} + +export class PushCoalescer { + private readonly subjects = new Map>(); + /** `committed`: the durability barrier succeeded after this receipt was + * written. An uncommitted receipt deduplicates effects but is not an + * acknowledgement until a commit succeeds. */ + private readonly receipts = new Map(); + private readonly retryWindowMs: number; + private readonly maxSubjects: number; + private readonly maxNotices: number; + /** Outstanding push/render RPCs per server, including cancelled ones (§10.7). */ + private readonly rendersInFlight = new Map(); + /** + * The ONE critical section. Every state transition — an admission, the + * freezing of a batch for rendering, the settlement of a render's result — + * runs under it, awaits included (fork spawn, durability commit, delivery). + * Only the push/render RPC itself runs outside, so it stays cancellable. + * An admission therefore never observes a half-done assembly transition, + * and an assembly never acts on a batch an admission moved on from. + */ + private serial: Promise = Promise.resolve(); + /** Liveness after a failed freeze: a bounded, backed-off re-wake per + * subject. `attempt` outlives the armed timer: it counts consecutive + * failures and resets only on success or disposal of the pending work. */ + private readonly recovery = new Map; attempt: number }>(); + private suspended = false; + + constructor(private readonly host: CoalescerHost, private readonly options: CoalescerOptions = {}) { + this.retryWindowMs = options.retryWindowMs ?? PUSH_COALESCING_SUPPORT.retryWindowMs; + this.maxSubjects = options.maxSubjects ?? 4_096; + this.maxNotices = options.maxNotices ?? 64; + } + + private now(): number { return this.host.now?.() ?? Date.now(); } + + private locked(fn: () => Promise): Promise { + const run = this.serial.then(fn, fn); + this.serial = run.then(() => undefined, () => undefined); + return run; + } + + // --------------------------------------------------------------------------- + // Observation + // --------------------------------------------------------------------------- + + /** The unread occupant's or batch's placement/identity, for routing continuity. */ + identity(subject: string): CoalescedOccurrence['identity'] | undefined { + return this.subjects.get(subject)?.identity; + } + + pendingBatches(): number { + let n = 0; + for (const s of this.subjects.values()) if (s.batch) n++; + return n; + } + + /** §10.7: a server handling push/render must not issue inference/request — + * including a render whose batch was cancelled while the RPC is outstanding. */ + isRendering(serverId: string): boolean { + return (this.rendersInFlight.get(serverId) ?? 0) > 0; + } + + receipt(serverId: string, binding: string, eventId: string): PushEventResult | undefined { + const entry = this.receiptEntry(coalescingReceiptKey(serverId, binding, eventId)); + return entry && structuredClone(entry.result); + } + + private receiptEntry(key: string): { result: PushEventResult; at: number; committed: boolean } | undefined { + const entry = this.receipts.get(key); + if (!entry) return undefined; + if (this.now() - entry.at > this.retryWindowMs) { this.receipts.delete(key); return undefined; } + return entry; + } + + /** Acknowledge a receipt: durable first. A receipt whose commit failed is + * committed now (no effects are repeated); if that fails again, so does + * the acknowledgement, and the producer retries once more. */ + private async acknowledge(key: string, entry: { result: PushEventResult; at: number; committed: boolean }, subject: string, eventId: string): Promise { + if (!entry.committed) { + try { + await this.host.commit?.(); + entry.committed = true; + } catch (error) { + this.host.audit({ kind: 'persist-failed', subject, eventId, error: String(error) }); + throw new CoalesceError('eventId', 'host could not make the acceptance durable; retry', -32000); + } + } + return structuredClone(entry.result); + } + + // --------------------------------------------------------------------------- + // Admission + // --------------------------------------------------------------------------- + + /** + * Admit an occurrence that has already passed the lane's ordinary checks and + * the coalesce-member validation. Returns the wire result, performing the + * host effects (replace / remove / deliver) synchronously so a create, edit + * and delete cannot overtake one another. + */ + accept(occurrence: CoalescedOccurrence): Promise { + return this.locked(() => this.admit(occurrence)); + } + + private async admit(occurrence: CoalescedOccurrence): Promise { + if (this.suspended) throw new CoalesceError('serverId', 'host is stopping', -32000); + const subject = coalescingSubjectKey(occurrence.serverId, occurrence.binding, occurrence.scope, occurrence.key); + const duplicate = this.receiptEntry(coalescingReceiptKey(occurrence.serverId, occurrence.binding, occurrence.eventId)); + if (duplicate) { + // A producer retrying acknowledged work is also a chance to notice a + // batch left asleep by a failed freeze: wake it now, without effects. + const state = this.subjects.get(subject); + if (state?.batch && !state.rendering) { this.disarmRecovery(subject); await this.host.wakeForBatch(state.batch.latest); } + return this.acknowledge(coalescingReceiptKey(occurrence.serverId, occurrence.binding, occurrence.eventId), duplicate, subject, occurrence.eventId); + } + const born = this.subjects.has(subject) ? undefined : (occurrence.initial && !occurrence.retract ? 'none' as const : 'unknown' as const); + const state = this.subjectFor(subject, occurrence); + const occupantUnread = this.refreshOccupant(state); + const priorEventId = state.occupant?.eventId ?? state.batch?.latest.eventId ?? state.rendering?.batch.latest.eventId ?? state.consumedEventId; + this.host.audit({ kind: 'received', subject, eventId: occurrence.eventId, retract: occurrence.retract, deferred: occurrence.deferred }); + + let outcome: CoalesceOutcome; + if (occurrence.retract) { + outcome = await this.retract(subject, state, occurrence, occupantUnread); + } else if (occurrence.deferred) { + outcome = await this.acceptDeferred(subject, state, occurrence, occupantUnread); + } else { + outcome = await this.acceptPlain(subject, state, occurrence, occupantUnread); + } + if (occurrence.scope.kind === 'channel' && occurrence.identity && !occurrence.retract) { + state.identity = mergeIdentity(state.identity, occurrence.identity); + } + state.touchedAt = this.now(); + const result: PushEventResult = { accepted: true, coalesce: { outcome, ...(priorEventId ? { priorEventId } : {}) } }; + const receiptKey = coalescingReceiptKey(occurrence.serverId, occurrence.binding, occurrence.eventId); + const entry = { result: structuredClone(result), at: this.now(), committed: false }; + try { + this.host.recordReceipt?.({ + key: receiptKey, result: entry.result, at: entry.at, subject, ...(born ? { born } : {}), + batch: state.batch ? structuredClone({ latest: state.batch.latest as CoalescedOccurrence, notices: state.batch.notices, dropped: state.batch.dropped }) : null, + }); + } catch (error) { + // Not acknowledged: the work would be lost while its receipt suppressed + // a retry. The producer retries within the window. + this.host.audit({ kind: 'persist-failed', subject, eventId: occurrence.eventId, error: String(error) }); + throw new CoalesceError('eventId', 'host could not persist the acceptance; retry', -32000); + } + this.receipts.set(receiptKey, entry); + this.prune(); + this.persist(); + // Written but not yet durable: the in-memory receipt deduplicates a retry + // while this process lives, and the retry commits it (acknowledge); after + // a kill nothing of it exists and the retry is a fresh acceptance. + return this.acknowledge(receiptKey, entry, subject, occurrence.eventId); + } + + private subjectFor(subject: string, occurrence: CoalescedOccurrence): SubjectState { + let state = this.subjects.get(subject); + if (!state) { + // §3.3: an untracked subject is `unknown` unless the producer marks the + // birth of the subject. A retraction is never a birth. + state = { history: occurrence.initial && !occurrence.retract ? 'none' : 'unknown', touchedAt: this.now() }; + this.subjects.set(subject, state); + } + return state; + } + + /** Re-check the occupant against the host: consumed occupants become history. */ + private refreshOccupant(state: SubjectState): boolean { + if (!state.occupant) return false; + if (this.host.isUnread(state.occupant.placement)) return true; + state.history = 'some'; + state.consumedEventId = state.occupant.eventId; + state.occupant = undefined; + return false; + } + + private scheduleRecovery(subject: string, state: SubjectState): void { + const prior = this.recovery.get(subject); + if (prior?.timer) clearTimeout(prior.timer); + const attempt = (prior?.attempt ?? 0) + 1; + const base = this.options.recoveryBackoffMs ?? 1_000; + const delay = Math.min(base * 2 ** (attempt - 1), base * 60); + const entry = { attempt } as { timer?: ReturnType; attempt: number }; + entry.timer = setTimeout(() => { + // Disarm, but keep the attempt count: a freeze that fails again backs + // off further. The count resets only on success or disposal. + if (this.recovery.get(subject) === entry) entry.timer = undefined; + if (this.suspended || this.subjects.get(subject) !== state || !state.batch || state.rendering) return; + // Authority and wake policy are the host's, re-evaluated at fire time. + if (!this.host.authorized(state.batch.latest)) return; + this.host.audit({ kind: 'recovery-wake', subject, eventId: state.batch.latest.eventId, attempt, delayMs: delay }); + void this.host.wakeForBatch(state.batch.latest).catch(() => { /* audited by the host */ }); + }, delay); + (entry.timer as { unref?: () => void }).unref?.(); + this.recovery.set(subject, entry); + } + + /** Disarm the timer only (a producer retry wakes now; the count stands). */ + private disarmRecovery(subject: string): void { + const prior = this.recovery.get(subject); + if (prior?.timer) { clearTimeout(prior.timer); prior.timer = undefined; } + } + + /** Success or disposal of the pending work: forget the failure series. */ + private clearRecovery(subject: string): void { + const prior = this.recovery.get(subject); + if (prior?.timer) clearTimeout(prior.timer); + this.recovery.delete(subject); + } + + private dropBatches(subject: string, state: SubjectState): boolean { + this.clearRecovery(subject); + let dropped = false; + if (state.batch) { this.host.audit({ kind: 'displaced', subject, eventId: state.batch.latest.eventId, batch: true }); state.batch = undefined; dropped = true; } + if (state.rendering) { + this.host.audit({ kind: 'render-cancelled', subject, eventId: state.rendering.batch.latest.eventId }); + state.rendering.cancelled = true; + state.rendering = undefined; + dropped = true; + } + return dropped; + } + + private async retract(subject: string, state: SubjectState, occurrence: CoalescedOccurrence, occupantUnread: boolean): Promise { + if (occupantUnread && state.occupant) { + this.host.audit({ kind: 'removed', subject, eventId: state.occupant.eventId }); + if (!this.host.remove(state.occupant.placement)) { + // Still in context and possibly about to be read: say so, append the notice. + this.host.audit({ kind: 'remove-failed', subject, eventId: state.occupant.eventId }); + state.history = 'unknown'; + } + } + state.occupant = undefined; + this.dropBatches(subject, state); + this.host.cancelWake(subject); + state.consumedEventId = undefined; + if (state.history === 'none') return 'retracted'; + if (!occurrence.content.length) return 'consumed'; + // The notice is an ordinary occurrence: it rides the normal delivery path + // (tags → gate policy → wake) and is consumed like any message — in the + // context that read a version, never a new one (§3.2). + const placement = await this.host.deliver({ ...occurrence, deliverTo: state.audienceAgent }); + if (placement) { + state.occupant = { eventId: occurrence.eventId, placement }; + state.audienceAgent = placement.agent; + } + return 'noted'; + } + + private async acceptPlain(subject: string, state: SubjectState, occurrence: CoalescedOccurrence, occupantUnread: boolean): Promise { + const hadBatch = this.dropBatches(subject, state); + if (occupantUnread && state.occupant) { + const prior = state.occupant; + this.host.audit({ kind: 'displaced', subject, eventId: prior.eventId }); + if (this.host.remove(prior.placement)) { + this.host.cancelWake(subject); + // The replacement follows the prior delivery's audience (§3.2). + const placement = await this.host.deliver({ ...occurrence, deliverTo: prior.placement.agent }); + state.occupant = placement ? { eventId: occurrence.eventId, placement } : undefined; + if (placement) state.audienceAgent = placement.agent; + return 'replaced'; + } + // Could not remove: the prior may still be read. Append, as for consumed. + this.host.audit({ kind: 'remove-failed', subject, eventId: prior.eventId }); + state.history = 'unknown'; + state.consumedEventId = prior.eventId; + state.occupant = undefined; + } + if (hadBatch) this.host.cancelWake(subject); + const placement = await this.host.deliver(occurrence); + state.occupant = placement ? { eventId: occurrence.eventId, placement } : undefined; + if (placement) state.audienceAgent = placement.agent; + if (hadBatch) return 'replaced'; + return state.consumedEventId || state.history !== 'none' ? 'appended' : 'first'; + } + + private async acceptDeferred(subject: string, state: SubjectState, occurrence: CoalescedOccurrence, occupantUnread: boolean): Promise { + let outcome: CoalesceOutcome = 'first'; + if (occupantUnread && state.occupant) { + // §5.1: a notice displaces an unread plain occurrence and opens a batch + // — in the audience that held the plain occurrence (§3.2). + this.host.audit({ kind: 'displaced', subject, eventId: state.occupant.eventId }); + if (this.host.remove(state.occupant.placement)) { + outcome = 'replaced'; + occurrence = { ...occurrence, deliverTo: state.occupant.placement.agent }; + } else { this.host.audit({ kind: 'remove-failed', subject, eventId: state.occupant.eventId }); state.history = 'unknown'; } + this.host.cancelWake(subject); + state.occupant = undefined; + } else if (state.batch?.latest.deliverTo) { + // Later notices stay in the batch's audience. + occurrence = { ...occurrence, deliverTo: state.batch.latest.deliverTo }; + } + const notice: DeferredNotice = { eventId: occurrence.eventId, timestamp: occurrence.timestamp, ...(occurrence.data !== undefined ? { data: structuredClone(occurrence.data) } : {}) }; + if (state.batch) { + state.batch.notices.push(notice); + if (state.batch.notices.length > this.maxNotices) { state.batch.notices.shift(); state.batch.dropped++; } + state.batch.latest = occurrence; + outcome = 'replaced'; + } else { + // Rule 1: a notice during a render opens a NEW batch ("first"). + state.batch = { latest: occurrence, notices: [notice], dropped: 0 }; + } + await this.host.wakeForBatch(occurrence); + return outcome; + } + + // --------------------------------------------------------------------------- + // Assembly (deferred batches only — plain content is already in context) + // --------------------------------------------------------------------------- + + /** + * Render and materialize every pending batch whose audience includes + * `agentName`. Called by the host at a turn's assembly boundary, before the + * compile. Bounded by the host's render timeout; a failed or late render + * materializes the admitted fallback (§5.3). + */ + async assemble(agentName: string): Promise { + if (this.suspended) return; + const work: Promise[] = []; + for (const subject of [...this.subjects.keys()]) { + // The handle is wrapped: a bare promise returned through `locked` would + // be flattened by `then`, holding the lock until the render settled — + // and the settlement itself needs the lock. + const started = await this.locked(() => this.freeze(subject, agentName)); + if (started) work.push(started.done); + } + await Promise.all(work); + this.persist(); + } + + /** + * Under the lock: decide whether `subject`'s batch renders for `agentName`, + * freeze it, make the render-start durable, re-check authority, and start + * the RPC. Returns the render's completion (shared by every assembly that + * waits on this batch, vector 25) or undefined when nothing was started. + */ + private async freeze(subject: string, agentName: string): Promise<{ done: Promise } | undefined> { + const state = this.subjects.get(subject); + if (!state || this.suspended) return undefined; + if (state.rendering) { + // Another assembly froze this batch; share its outcome (vector 25). + const rendering = state.rendering; + return (await this.host.audience(rendering.batch.latest)).includes(agentName) && state.rendering === rendering ? { done: rendering.done } : undefined; + } + const batch = state.batch; + if (!batch) return undefined; + // The audience lookup may spawn a fork. Under the lock no admission can + // move the subject on meanwhile; the identity check guards the host. + const audience = await this.host.audience(batch.latest); + if (!audience.includes(agentName) || state.batch !== batch || state.rendering) return undefined; + state.batch = undefined; + this.host.cancelWake(subject); + if (!this.host.authorized(batch.latest)) { + this.host.audit({ kind: 'revoked', subject, eventId: batch.latest.eventId }); + return undefined; + } + const rendering: Rendering = { batch, cancelled: false, done: Promise.resolve() }; + state.rendering = rendering; + // The render-start boundary is recoverable BEFORE the RPC: a restart + // then takes the fallback instead of asking the server a second time. + try { + if (this.host.saveNow) this.host.saveNow(this.snapshot()); else this.persist(); + await this.host.commit?.(); + } catch (error) { + this.host.audit({ kind: 'persist-failed', subject, eventId: batch.latest.eventId, error: String(error) }); + state.rendering = undefined; + batch.noRender = true; // cannot prove a render never started → fallback + state.batch = batch; + // The wake was withdrawn above; acknowledged work must not sleep until + // unrelated traffic happens by. Re-wake with backoff, so a persistent + // storage outage does not become a tight loop of model turns. + this.scheduleRecovery(subject, state); + return undefined; + } + this.clearRecovery(subject); + if (this.suspended) { state.rendering = undefined; state.batch = batch; return undefined; } + // Authority (grant, registration, binding) is re-checked immediately + // before dispatch: a server id reassigned to another endpoint during the + // commit must not receive this batch's notices. + if (!this.host.authorized(batch.latest)) { + this.host.audit({ kind: 'revoked', subject, eventId: batch.latest.eventId }); + state.rendering = undefined; + return undefined; + } + rendering.done = this.render(subject, state, rendering, agentName); + return { done: rendering.done }; + } + + private async render(subject: string, state: SubjectState, rendering: Rendering, assemblingFor: string): Promise { + const { batch } = rendering; + const occurrence = batch.latest; + let content: McplContentBlock[] = occurrence.content; + let timestamp = occurrence.timestamp; + let source: 'render' | 'fallback' = 'fallback'; + if (!batch.noRender) { + const params: PushRenderParams = { + featureSet: occurrence.featureSet ?? (occurrence.scope.kind === 'featureSet' ? occurrence.scope.id : ''), + ...(occurrence.scope.kind === 'channel' ? { channelId: occurrence.scope.id } : {}), + key: occurrence.key, eventId: occurrence.eventId, + notices: structuredClone(batch.notices), dropped: batch.dropped, + }; + this.rendersInFlight.set(occurrence.serverId, (this.rendersInFlight.get(occurrence.serverId) ?? 0) + 1); + try { + const result = await this.host.render(occurrence, params); + if (rendering.cancelled) { this.host.audit({ kind: 'late-render', subject, eventId: occurrence.eventId, discarded: true }); } + else validateCoalescedContent(result?.content, this.options.maxContentBytes); + if (rendering.cancelled) throw new CancelledRender(); + content = result.content; + timestamp = typeof result.timestamp === 'string' ? result.timestamp : new Date().toISOString(); + source = 'render'; + } catch (error) { + if (!(error instanceof CancelledRender)) this.host.audit({ kind: 'render-failed', subject, eventId: occurrence.eventId, error: String(error) }); + } finally { + const left = (this.rendersInFlight.get(occurrence.serverId) ?? 1) - 1; + if (left > 0) this.rendersInFlight.set(occurrence.serverId, left); else this.rendersInFlight.delete(occurrence.serverId); + } + } + // Settlement runs under the lock: between the RPC's return and the state + // update no admission can withdraw or replace what is being published. + await this.locked(async () => { + try { + if (rendering.cancelled || state.rendering !== rendering || this.suspended) return; + // Rule 4: authority is re-checked at response. + if (!this.host.authorized(occurrence)) { this.host.audit({ kind: 'revoked', subject, eventId: occurrence.eventId }); return; } + this.host.audit({ kind: 'rendered', subject, eventId: occurrence.eventId, source, empty: content.length === 0 }); + if (!content.length) return; // §5.2: nothing happened + const placement = await this.host.deliver({ ...occurrence, timestamp, content }, content, assemblingFor); + if (placement?.deferredId) { + // Landed in another agent's deferred queue (its turn is alive): still + // unread there, so it stays replaceable and withdrawable. + state.occupant = { eventId: occurrence.eventId, placement }; + state.audienceAgent = placement.agent; + return; + } + // The materialized occurrence is consumed by the request being assembled. + state.history = 'some'; + state.consumedEventId = occurrence.eventId; + state.occupant = undefined; + if (placement) state.audienceAgent = placement.agent; + } finally { + if (state.rendering === rendering) state.rendering = undefined; + } + }); + } + + // --------------------------------------------------------------------------- + // Lifecycle + // --------------------------------------------------------------------------- + + restore(snapshot: CoalescingSnapshot | null | undefined): void { + if (!snapshot || snapshot.version !== 1) return; + const now = this.now(); + for (const [key, entry] of snapshot.receipts) { + if (now - entry.at <= this.retryWindowMs) this.receipts.set(key, { ...entry, committed: true }); + } + for (const s of snapshot.subjects) { + const state: SubjectState = { + // A stored occupant is in context and will be in the next request: + // it counts as read from here on (the watermark restarts at the head). + history: s.occupant ? 'some' : s.history, + consumedEventId: s.occupant?.eventId ?? s.consumedEventId, + identity: s.identity, + audienceAgent: s.audienceAgent ?? s.occupant?.placement.agent, + touchedAt: now, + }; + if (s.batch) { + // An interrupted render is not replayed (§3.2): a batch that was + // RENDERING keeps its fallback; a batch that was only pending renders + // normally at the next assembly. + state.batch = { latest: s.batch.latest as CoalescedOccurrence, notices: s.batch.notices, dropped: s.batch.dropped, ...(s.batch.rendering ? { noRender: true } : {}) }; + } + this.subjects.set(s.subject, state); + } + this.settlePublished(); + } + + /** + * A batch restored as RENDERING may already have been published (its + * result or fallback is a durable context message; only the completion + * snapshot was lost). Recovery then marks it consumed instead of + * delivering the fallback a second time. + */ + private settlePublished(): void { + if (!this.host.wasPublished) return; + for (const [subject, state] of this.subjects) { + const batch = state.batch; + if (!batch?.noRender) continue; + if (this.host.wasPublished(subject, batch.latest.eventId)) { + state.batch = undefined; + state.history = 'some'; + state.consumedEventId = batch.latest.eventId; + } + } + } + + /** Apply receipts written since the last snapshot (crash-window bridge). */ + restoreReceipts(records: CoalescingReceiptRecord[]): void { + const now = this.now(); + for (const r of records) { + if (now - r.at > this.retryWindowMs) continue; + this.receipts.set(r.key, { result: r.result, at: r.at, committed: true }); + if (r.born && !this.subjects.has(r.subject)) this.subjects.set(r.subject, { history: r.born, touchedAt: now }); + // A birth fact is not proof the subject is still unread: the plain + // occurrence this record acknowledged is in context (durable before the + // record) and counts as read from recovery on, like a snapshot occupant. + let eventId: string | undefined; + try { eventId = JSON.parse(r.key)[2]; } catch { /* malformed key: no settlement */ } + if (eventId && this.host.wasPublished?.(r.subject, eventId)) { + const state = this.subjects.get(r.subject) ?? { history: 'unknown' as const, touchedAt: now }; + state.history = 'some'; + state.consumedEventId = eventId; + this.subjects.set(r.subject, state); + } + if (r.batch === undefined) continue; // pre-round-3 record: no batch information + // The subject's batch AFTER that acceptance, newer than the snapshot: + // the accepted deferred work itself, or null for a withdrawal / + // replacement that cleared it. It never began rendering before the + // record was written, but the snapshot cannot say whether it did + // afterwards: take the fallback. + const state = this.subjects.get(r.subject) ?? { history: 'unknown' as const, touchedAt: now }; + state.batch = r.batch + ? { latest: r.batch.latest as CoalescedOccurrence, notices: r.batch.notices, dropped: r.batch.dropped, noRender: true } + : undefined; + this.subjects.set(r.subject, state); + } + this.settlePublished(); + } + + snapshot(): CoalescingSnapshot { + return { + version: 1, + receipts: [...this.receipts], + subjects: [...this.subjects].map(([subject, s]) => ({ + subject, history: s.history, consumedEventId: s.consumedEventId, identity: s.identity, + ...(s.audienceAgent ? { audienceAgent: s.audienceAgent } : {}), + ...(s.occupant ? { occupant: s.occupant } : {}), + ...(s.batch ? { batch: { latest: s.batch.latest as CoalescedOccurrence, notices: s.batch.notices, dropped: s.batch.dropped, ...(s.batch.noRender ? { rendering: true } : {}) } } + : s.rendering ? { batch: { latest: s.rendering.batch.latest as CoalescedOccurrence, notices: s.rendering.batch.notices, dropped: s.rendering.batch.dropped, rendering: true } } + : {}), + })), + }; + } + + /** Batches whose wake should be re-queued after a restart (their fallback is pending). */ + pendingBatchOccurrences(): CoalescedOccurrence[] { + return [...this.subjects.values()].flatMap(s => s.batch ? [s.batch.latest] : []); + } + + suspend(): void { + this.suspended = true; + for (const subject of [...this.recovery.keys()]) this.clearRecovery(subject); + for (const [subject, state] of this.subjects) { + if (state.rendering) { state.rendering.cancelled = true; this.host.audit({ kind: 'render-cancelled', subject, eventId: state.rendering.batch.latest.eventId, reason: 'suspend' }); } + } + } + + private persist(): void { this.host.save(() => this.snapshot()); } + + private prune(): void { + const now = this.now(); + if (this.receipts.size > 4 * this.maxSubjects) { + for (const [key, entry] of this.receipts) if (now - entry.at > this.retryWindowMs) this.receipts.delete(key); + } + if (this.subjects.size <= this.maxSubjects) return; + // Evict the oldest idle subjects; their history degrades to `unknown` + // (an untracked subject) which is the conservative outcome (§3.3). + const idle = [...this.subjects].filter(([, s]) => !s.occupant && !s.batch && !s.rendering).sort((a, b) => a[1].touchedAt - b[1].touchedAt); + for (const [key] of idle.slice(0, Math.max(0, this.subjects.size - this.maxSubjects))) this.subjects.delete(key); + } +} diff --git a/src/mcpl/push-handler.ts b/src/mcpl/push-handler.ts index a9cde38b5..d85df767a 100644 --- a/src/mcpl/push-handler.ts +++ b/src/mcpl/push-handler.ts @@ -18,6 +18,7 @@ import type { import type { FeatureSetManager } from './feature-set-manager.js'; import { McplFeatureSetError } from './feature-set-manager.js'; import { expandCoreTags } from './tags.js'; +import { validateCoalescedContent } from './push-coalescer.js'; // ============================================================================ // McplPushEvent (the ProcessEvent shape pushed to the queue) @@ -42,6 +43,10 @@ export interface McplPushEvent { inferenceId: string; triggerInference?: boolean; targetAgents?: string[]; + /** RFC-006: set when the coalescer delivers; rides the wake it queues. */ + coalescingSubject?: string; + /** RFC-006 assembly: materialized for this agent's turn (store directly). */ + assemblingFor?: string; } // ============================================================================ @@ -171,6 +176,7 @@ export class PushHandler { pushEventFn: (event: McplPushEvent) => void, emitTraceFn: (event: { type: string; [key: string]: unknown }) => void, shouldTriggerInference?: (content: string, metadata: Record) => boolean, + private readonly handleCoalesced?: (serverId: string, params: PushEventParams, event: McplPushEvent) => Promise, ) { this.featureSetManager = featureSetManager; this.pushEventFn = pushEventFn; @@ -189,11 +195,11 @@ export class PushHandler { * 6. Emit trace * 7. Respond with accepted + inferenceId */ - handlePushEvent( + async handlePushEvent( serverId: string, params: PushEventParams, responder?: Responder, - ): void { + ): Promise { // §16.3: expand the normative chat:* core closure once, at entry, so // every downstream consumer (wake matching, metadata, the queued event) // sees the closed set. Producer `implies` edges are NOT consumed — @@ -223,8 +229,22 @@ export class PushHandler { return; } - // 2. Deduplicate by eventId - if (this.dedup.checkAndAdd(params.eventId)) { + // 2. Deduplicate by eventId. A coalesced occurrence is deduplicated by the + // coalescer's receipts instead (RFC-006 §3.1: a retry within the window + // gets its original result, which this set could not return). + const coalesced = params.coalesce !== undefined && !!this.handleCoalesced; + if (coalesced) { + // RFC-006 §13: malformed content is a -32602, checked before conversion. + try { + validateCoalescedContent(params.payload?.content); + } catch (error) { + const err = error as Error & { code?: number; field?: string }; + if (responder?.respondError) responder.respondError(err.code ?? -32602, err.message, { field: err.field }); + else responder?.respond({ accepted: false, reason: err.message }); + return; + } + } + if (!coalesced && this.dedup.checkAndAdd(params.eventId)) { console.error(`[push-event-rejected] server=${serverId} eventId=${params.eventId} reason=duplicate`); responder?.respond({ accepted: false, reason: 'duplicate' }); return; @@ -267,6 +287,22 @@ export class PushHandler { inferenceId, triggerInference, }; + if (coalesced) { + // RFC-006: the coalescer decides whether this occurrence replaces an + // unread one, appends, or withdraws; it delivers through the same event + // the ordinary path would have queued. + try { + const result = await this.handleCoalesced!(serverId, params, pushEvent); + this.emitTraceFn({ type: 'mcpl:push_event', serverId, eventId: params.eventId, featureSet: params.featureSet, coalesce: result.coalesce }); + responder?.respond(result); + } catch (error) { + const err = error as Error & { code?: number; field?: string }; + console.error(`[push-event-rejected] server=${serverId} eventId=${params.eventId} reason=${err.message}`); + if (responder?.respondError) responder.respondError(typeof err.code === 'number' ? err.code : -32603, err.message, { field: err.field }); + else responder?.respond({ accepted: false, reason: err.message }); + } + return; + } this.pushEventFn(pushEvent); // 7. Emit trace diff --git a/src/mcpl/server-connection.ts b/src/mcpl/server-connection.ts index c983b8b67..f29fea22d 100644 --- a/src/mcpl/server-connection.ts +++ b/src/mcpl/server-connection.ts @@ -207,7 +207,7 @@ export class McplServerConnection extends EventEmitter { private pendingRequests = new Map(); /** Metadata retained after selected request deadlines so a late response is * observable even though its original promise has already been rejected. */ - private orphanedRequests = new Map(); + private orphanedRequests = new Map }>(); private closed = false; /** Per-request timeout in ms (0 disables). See McplServerConfig.requestTimeoutMs. */ @@ -656,6 +656,11 @@ export class McplServerConnection extends EventEmitter { return this.sendRequest(McplMethod.BeforeInference, params as unknown as Record) as Promise; } + /** RFC-006 deferred push rendering; its own timeout bounds context assembly. */ + sendPushRender(params: import('./push-coalescer.js').PushRenderParams): Promise { + return this.sendRequest('push/render', params as unknown as Record, { timeoutMs: 5000, surfaceOrphanedResponse: true }) as Promise; + } + /** Send `featureSets/update` as a Notification. §6.7: valid ONLY for * purely descriptive metadata that does not alter the grant. Any grant @@ -986,7 +991,8 @@ export class McplServerConnection extends EventEmitter { pending.timer = setTimeout(() => { this.pendingRequests.delete(id); if (options.surfaceOrphanedResponse) { - this.orphanedRequests.set(id, { method }); + this.orphanedRequests.set(id, { method, ...(method === 'push/render' ? { renderParams: { featureSet: params.featureSet, key: params.key, eventId: params.eventId } } : {}) }); + if (this.orphanedRequests.size > 4096) this.orphanedRequests.delete(this.orphanedRequests.keys().next().value!); } reject(new Error( `MCPL server "${this.id}" did not respond to ${method} (id=${id}) ` + @@ -1178,6 +1184,10 @@ export class McplServerConnection extends EventEmitter { // re-inject either result because the original dispatch context is gone. const orphaned = this.orphanedRequests.get(response.id); this.orphanedRequests.delete(response.id); + if (orphaned?.renderParams) { + this.emit('orphaned-render-response', { params: orphaned.renderParams, result: response.result, error: response.error }); + return; // audit only: a timeout fallback can never be replaced + } const result = response.result; if (result && typeof result === 'object') { const r = result as { state?: unknown; checkpoint?: unknown }; diff --git a/src/mcpl/types.ts b/src/mcpl/types.ts index 503f2b683..c080af76d 100644 --- a/src/mcpl/types.ts +++ b/src/mcpl/types.ts @@ -165,6 +165,15 @@ export interface McplCapabilities { */ export interface McplHostCapabilities { version: string; + /** RFC-006 support, independent of capability grants. */ + eventCoalescing?: boolean | { + pushEvents?: boolean; + channelsIncoming?: boolean; + deferred?: boolean; + channelScopedPush?: boolean; + /** Retry-window guarantee in ms (integer ≥ 3,600,000; default 3,600,000). */ + retryWindowMs?: number; + }; pushEvents?: boolean; contextHooks?: { beforeInference?: boolean | { @@ -653,6 +662,21 @@ export interface StateRollbackResult { * Spec Section 9.1. */ export interface PushEventParams { + /** RFC-006 event coalescing (mcpl PR #5, revision 7). */ + coalesce?: { + /** Subject key, 1..256 UTF-8 bytes, opaque to the host. */ + key: string; + /** push/event only: address a registered channel's subject namespace. */ + channelId?: string; + /** push/event only: deferred mode — the host calls push/render at assembly. */ + deferred?: boolean; + /** The subject ceased to exist; `payload.content` is the deletion notice. */ + retract?: boolean; + /** No earlier occurrence of this subject was ever sent (§3.3). */ + initial?: boolean; + /** deferred only; ≤ 4 KiB serialized; server-private, echoed by push/render. */ + data?: unknown; + }; /** Declaring feature set */ featureSet: string; @@ -680,6 +704,10 @@ export interface PushEventParams { * Spec Section 9.3. */ export interface PushEventResult { + coalesce?: { + outcome: 'first' | 'replaced' | 'appended' | 'retracted' | 'noted' | 'consumed'; + priorEventId?: string; + }; /** Whether the event was accepted */ accepted: boolean; @@ -1116,6 +1144,8 @@ export interface ChannelsPublishResult { * Spec Section 14.3. */ export interface ChannelIncomingMessage { + eventId?: string; + coalesce?: Omit, 'channelId' | 'deferred'>; /** Channel this message came from */ channelId: string; @@ -1163,6 +1193,7 @@ export interface ChannelsIncomingResult { /** Result for a single incoming message. */ export interface ChannelIncomingMessageResult { + coalesce?: PushEventResult['coalesce']; messageId: string; accepted: boolean; conversationId?: string; diff --git a/src/types/agent.ts b/src/types/agent.ts index b5cd7471f..0b2c7c03c 100644 --- a/src/types/agent.ts +++ b/src/types/agent.ts @@ -296,6 +296,9 @@ export interface CompletedToolCall { * Inference request for an agent. */ export interface InferenceRequest { + /** Host-owned identity for withdrawal of an unconsumed coalesced wake. */ + coalescingSubject?: string; + coalescingEventId?: string; agentName: string; reason: string; source: string; diff --git a/src/types/trace.ts b/src/types/trace.ts index bd1394ffa..e4a6bdba9 100644 --- a/src/types/trace.ts +++ b/src/types/trace.ts @@ -209,6 +209,21 @@ export type TraceEvent = messageId: string; source: string; }) + /** An unread message withdrawn or replaced by its sender (RFC-006). */ + | (TraceEventBase & { + type: 'message:removed'; + messageId: string; + source: string; + }) + /** RFC-006 coalescing decision record (received / displaced / removed / + * rendered / revoked / render-cancelled / late-render / render-failed). */ + | (TraceEventBase & { + type: 'mcpl:coalescing'; + kind: string; + subject?: string; + eventId?: string; + [key: string]: unknown; + }) // EventGate lifecycle | (TraceEventBase & { diff --git a/test/helpers/coalescing-fixture.ts b/test/helpers/coalescing-fixture.ts new file mode 100644 index 000000000..2a2405683 --- /dev/null +++ b/test/helpers/coalescing-fixture.ts @@ -0,0 +1,127 @@ +/** + * Shared RFC-006 wire fixture: a loopback WebSocket MCPL server, a mock model + * provider, and the framework under test. Used by test/mcpl-coalescing.test.ts + * and by the subprocess-kill child (coalescing-kill-child.ts). + */ +import assert from 'node:assert/strict'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { once } from 'node:events'; +import { WebSocketServer, type WebSocket } from 'ws'; +import type { AddressInfo } from 'node:net'; +import type { FrameworkConfig } from '../../src/types/framework.js'; +import { AgentFramework } from '../../src/framework.js'; +import { MockMembrane, MockYieldingStream, createMockResponse } from './mock-membrane.js'; + +export const TS = '2026-09-30T00:00:00Z'; +export const ok = () => createMockResponse([{ type: 'text', text: 'ok' }]); + +export async function fixture(options: { server?: Record; framework?: Partial; agents?: unknown[]; dir?: string; port?: number } = {}) { + const wss = new WebSocketServer({ port: options.port ?? 0, host: '127.0.0.1' }); + await once(wss, 'listening'); + let online = true; + let socket: WebSocket; + let next = 1000; + const replies = new Map void>(); + const renders: Array> = []; + const hostCaps: Array> = []; + const published: Array> = []; + let renderer: (params: Record) => Promise = async () => ({ content: [{ type: 'text', text: 'document_diff' }] }); + wss.on('connection', (ws) => { + if (!online) { ws.close(); return; } + socket = ws; + ws.on('message', async (bytes) => { + const m = JSON.parse(String(bytes)); + if (!m.method) { replies.get(m.id)?.(m); replies.delete(m.id); return; } + const reply = (result: unknown) => ws.send(JSON.stringify({ jsonrpc: '2.0', id: m.id, result })); + if (m.method === 'initialize') { + hostCaps.push(m.params.capabilities.experimental.mcpl); + reply({ protocolVersion: '2024-11-05', capabilities: { tools: {}, experimental: { mcpl: { + version: '0.5', pushEvents: true, inferenceRequest: true, + channels: { incoming: true, register: true, lifecycle: true, publish: true }, + featureSets: { doc: { description: 'doc', uses: ['pushEvents'] } }, + } } }, serverInfo: { name: 'editor', version: '1' } }); + } else if (m.method === 'featureSets/update') reply({ accepted: true }); + else if (m.method === 'tools/list') reply({ tools: [] }); + else if (m.method === 'channels/publish') { published.push(m.params); if (m.id !== undefined) reply({ delivered: true }); } + else if (m.method === 'channels/close') reply({ closed: true }); + else if (m.method === 'channels/open') reply({ channel: { id: m.params.channelId, type: 'discord', label: m.params.channelId } }); + else if (m.method === 'push/render') { renders.push(m.params); reply(await renderer(m.params)); } + else if (m.id !== undefined) reply({}); + }); + }); + const dir = options.dir ?? mkdtempSync(join(tmpdir(), 'coalescing-')); + let framework: AgentFramework; + const membrane = new MockMembrane(); + membrane.pushResponse(ok()); + const create = async (grantPush = true) => { + framework = await AgentFramework.create({ + storePath: join(dir, 'store'), membrane: membrane.asMembrane(), + agents: (options.agents ?? [{ name: 'agent', model: 'test', systemPrompt: 'test' }]) as FrameworkConfig['agents'], + modules: [], ...options.framework, + mcplServers: [{ id: 'editor', url: `ws://127.0.0.1:${(wss.address() as AddressInfo).port}`, enabledFeatureSets: ['doc'], + ...options.server, ...(grantPush ? {} : { disabledCapabilities: ['pushEvents'] }) }], + } as FrameworkConfig); + return framework; + }; + const send = (method: string, params: unknown): Promise => new Promise((resolve, reject) => { + const id = next++; + const timer = setTimeout(() => reject(new Error(`no reply to ${method}`)), 3000); + replies.set(id, (value) => { clearTimeout(timer); resolve(value); }); + socket.send(JSON.stringify({ jsonrpc: '2.0', id, method, params })); + }); + await create(); + return { + get framework() { return framework; }, membrane, renders, hostCaps, published, create, send, dir, + port: (wss.address() as AddressInfo).port, + disconnect: () => socket.terminate(), + online: (value: boolean) => { online = value; }, + register: (id = 'chat') => send('channels/register', { channels: [{ id, type: 'discord', label: id, metadata: { channelType: 'guild_text' } }] }), + /** A coalesced channels/incoming message for platform message `m`. */ + channel: (eventId: string, text: string, flags: Record = {}, channelId = 'chat', messageId = 'm') => ({ + channelId, messageId, eventId, timestamp: TS, author: { id: 'u', name: 'User' }, tags: ['chat:mention'], + content: text ? [{ type: 'text', text }] : [], coalesce: { key: `message:${messageId}`, ...flags }, + }), + /** A coalesced feature-set push. */ + params: (eventId: string, text: string, flags: Record = {}) => ({ + featureSet: 'doc', eventId, timestamp: TS, coalesce: { key: 'document', ...flags }, + payload: { content: text ? [{ type: 'text', text }] : [] }, + }), + renderer: (fn: typeof renderer) => { renderer = fn; }, + /** Serialized context of the primary agent. */ + context: (agent = 'agent') => JSON.stringify(framework.getAgent(agent)!.getContextManager().getAllMessages()), + /** Serialized messages of the last model request (tool descriptions excluded). */ + lastRequest: () => JSON.stringify((membrane.calls.at(-1) as { messages?: unknown } | undefined)?.messages), + turn: async () => { membrane.pushResponse(ok()); await framework.runUntilIdle(); }, + /** Every subsequent model call gets a complete 'ok' response (multi-turn tests). */ + alwaysRespond: () => { + (membrane as unknown as { streamYielding: unknown }).streamYielding = (request: unknown) => { + membrane.calls.push(request as never); + return new MockYieldingStream([ok()]); + }; + }, + close: async () => { + await framework.stop(); + for (const client of wss.clients) client.terminate(); + await new Promise((r) => wss.close(() => r())); + rmSync(dir, { recursive: true, force: true }); + }, + }; +} + +export async function eventually(predicate: () => boolean, what = 'condition'): Promise { + const deadline = Date.now() + 3000; + while (!predicate()) { assert(Date.now() < deadline, `${what} did not settle`); await new Promise((r) => setTimeout(r, 10)); } +} + + +/** Simulate a crash: stop without the snapshot flush a clean stop performs. */ +export async function crash(f: Awaited>): Promise { + const internals = f.framework as unknown as { flushCoalescingSnapshot: () => void; coalescingSaveTimer: ReturnType | null }; + if (internals.coalescingSaveTimer) clearTimeout(internals.coalescingSaveTimer); + internals.coalescingSaveTimer = null; + internals.flushCoalescingSnapshot = () => {}; + await f.framework.stop(); +} + diff --git a/test/helpers/coalescing-kill-child.ts b/test/helpers/coalescing-kill-child.ts new file mode 100644 index 000000000..b8e679b93 --- /dev/null +++ b/test/helpers/coalescing-kill-child.ts @@ -0,0 +1,23 @@ +/** + * Subprocess for the SIGKILL durability tests (test/mcpl-coalescing.test.ts, + * round 5): runs the fixture on the given store directory, reaches the + * requested boundary, reports over IPC, and then waits to be killed. No + * stop(), no close(), no sync() of its own. + */ +import { fixture } from './coalescing-fixture.js'; + +const [dir, mode] = process.argv.slice(2); +const f = await fixture({ dir }); +if (mode === 'accept') { + const r = await f.send('push/event', f.params('1', 'killed_fallback', { deferred: true, data: { k: 'v' } })); + process.send!({ port: f.port, receipt: r.result }); +} else { + let reached!: () => void; + const started = new Promise((resolve) => { reached = resolve; }); + f.renderer(() => { reached(); return new Promise(() => { /* held until the kill */ }); }); + await f.send('push/event', f.params('1', 'render_fallback', { deferred: true })); + void f.framework.runUntilIdle(); + await started; + process.send!({ port: f.port, receipt: null }); +} +await new Promise(() => { /* wait for SIGKILL */ }); diff --git a/test/mcpl-coalescing.test.ts b/test/mcpl-coalescing.test.ts new file mode 100644 index 000000000..185562ba1 --- /dev/null +++ b/test/mcpl-coalescing.test.ts @@ -0,0 +1,1014 @@ +/** + * RFC-006 event coalescing (mcpl PR #5, revision 7) — wire-level tests over a + * real loopback WebSocket MCPL server and a mock model provider. + * + * Vector numbers refer to RFC-006 §14. + */ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { fork } from 'node:child_process'; +import { once } from 'node:events'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, dirname } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { createMockResponse } from './helpers/mock-membrane.js'; +import { fixture, eventually, crash, ok, TS } from './helpers/coalescing-fixture.js'; + +// --------------------------------------------------------------------------- +// Plain coalescing, feature-set scope (vectors 1–6, 35) +// --------------------------------------------------------------------------- + +test('host advertises eventCoalescing with the default retry window', async (t) => { + const f = await fixture(); t.after(f.close); + assert.deepEqual(f.hostCaps[0].eventCoalescing, { pushEvents: true, channelsIncoming: true, deferred: true, channelScopedPush: true, retryWindowMs: 3_600_000 }); +}); + +test('vectors 2, 6, 35: unread replacement keeps only the newest content; a retry returns the receipt', async (t) => { + const f = await fixture(); t.after(f.close); + const first = await f.send('push/event', f.params('1', 'edit_original', { initial: true })); + const second = await f.send('push/event', f.params('2', 'edit_middle')); + const last = await f.send('push/event', f.params('3', 'edit_latest')); + assert.equal(first.result.coalesce.outcome, 'first'); + assert.equal(second.result.coalesce.outcome, 'replaced'); + assert.deepEqual(last.result.coalesce, { outcome: 'replaced', priorEventId: '2' }); + // In context immediately (not staged), but only the newest occurrence. + assert(f.context().includes('edit_latest')); + assert(!f.context().includes('edit_original') && !f.context().includes('edit_middle')); + await f.framework.runUntilIdle(); + assert.equal(f.membrane.calls.length, 1); + assert(f.lastRequest().includes('edit_latest') && !f.lastRequest().includes('edit_original')); + const retry = await f.send('push/event', f.params('3', 'edit_latest')); + assert.deepEqual(retry.result, last.result); + await f.framework.runUntilIdle(); + assert.equal(f.membrane.calls.length, 1, 'a retry has no second effect'); +}); + +test('vector 3: a consumed occurrence is history; the next one appends', async (t) => { + const f = await fixture(); t.after(f.close); + await f.send('push/event', f.params('1', 'battery_79', { initial: true })); + await f.framework.runUntilIdle(); + const r = await f.send('push/event', f.params('2', 'battery_78')); + assert.deepEqual(r.result.coalesce, { outcome: 'appended', priorEventId: '1' }); + await f.turn(); + const req = f.lastRequest(); + assert(req.includes('battery_79') && req.includes('battery_78')); + assert(req.indexOf('battery_79') < req.indexOf('battery_78')); +}); + +test('vector 28: retraction of never-read content leaves no trace and no turn', async (t) => { + const f = await fixture(); t.after(f.close); + await f.send('push/event', f.params('1', 'unread_original', { initial: true })); + const r = await f.send('push/event', f.params('2', 'deletion_notice', { retract: true })); + assert.equal(r.result.coalesce.outcome, 'retracted'); + await f.framework.runUntilIdle(); + assert.equal(f.membrane.calls.length, 0); + assert(!f.context().includes('unread_original') && !f.context().includes('deletion_notice')); +}); + +test('vectors 29/34: read original → unread edit → delete keeps the original and appends only the notice', async (t) => { + const f = await fixture(); t.after(f.close); + await f.send('push/event', f.params('1', 'read_original', { initial: true })); + await f.framework.runUntilIdle(); + const edit = await f.send('push/event', f.params('2', 'unread_edit')); + assert.equal(edit.result.coalesce.outcome, 'appended'); + const del = await f.send('push/event', f.params('3', 'deletion_notice', { retract: true })); + assert.equal(del.result.coalesce.outcome, 'noted'); + await f.turn(); + const req = f.lastRequest(); + assert(req.includes('read_original') && req.includes('deletion_notice') && !req.includes('unread_edit')); + // History persists past the empty slot: a new create under the same key, then delete → noted. + await f.send('push/event', f.params('4', 'new_create')); + const again = await f.send('push/event', f.params('5', 'second_notice', { retract: true })); + assert.equal(again.result.coalesce.outcome, 'noted'); +}); + +test('vector 30/33: empty retraction of consumed content is "consumed" and appends nothing', async (t) => { + const f = await fixture(); t.after(f.close); await f.register(); + await f.send('channels/incoming', { messages: [f.channel('c', 'presence_line', { initial: true }, 'chat', 'p1')] }); + await f.framework.runUntilIdle(); + const count = f.framework.getAgent('agent')!.getContextManager().getMessageCount(); + const r = await f.send('push/event', { featureSet: 'doc', eventId: 'gone', timestamp: TS, coalesce: { channelId: 'chat', key: 'message:p1', retract: true }, payload: { content: [] } }); + assert.equal(r.result.coalesce.outcome, 'consumed'); + assert.equal(f.framework.getAgent('agent')!.getContextManager().getMessageCount(), count); +}); + +test('vectors 31/32/34a: history is unknown without `initial`; `initial` makes "retracted" reachable after a restart', async (t) => { + const f = await fixture(); t.after(f.close); + await f.framework.stop(); await f.create(); + const plainCreate = await f.send('push/event', f.params('a', 'no_birth_marker', { key: 'k1' })); + assert.equal(plainCreate.result.coalesce.outcome, 'appended'); + const plainDelete = await f.send('push/event', f.params('b', '[deleted]', { key: 'k1', retract: true })); + assert.equal(plainDelete.result.coalesce.outcome, 'noted', 'unknown history yields the notice'); + const born = await f.send('push/event', f.params('c', 'born_here', { key: 'k2', initial: true })); + assert.equal(born.result.coalesce.outcome, 'first'); + const gone = await f.send('push/event', f.params('d', '[deleted]', { key: 'k2', retract: true })); + assert.equal(gone.result.coalesce.outcome, 'retracted'); + assert(!f.context().includes('born_here')); + // vector 34b: initial cannot lower history. + await f.send('push/event', f.params('e', 'seen', { key: 'k3', initial: true })); + await f.framework.runUntilIdle(); + await f.send('push/event', f.params('f', 'again', { key: 'k3', initial: true })); + const later = await f.send('push/event', f.params('g', 'notice', { key: 'k3', retract: true })); + assert.equal(later.result.coalesce.outcome, 'noted'); +}); + +test('vector 9: feature set and key isolate subjects', async (t) => { + const f = await fixture(); t.after(f.close); + await f.send('push/event', f.params('1', 'doc_a', { key: 'A', initial: true })); + await f.send('push/event', f.params('2', 'doc_b', { key: 'B', initial: true })); + const r = await f.send('push/event', f.params('3', 'doc_a2', { key: 'A' })); + assert.equal(r.result.coalesce.outcome, 'replaced'); + assert(f.context().includes('doc_b') && f.context().includes('doc_a2') && !f.context().includes('doc_a"')); +}); + +test('vector 38: malformed coalesce is a -32602 on push/event and per-message on channels/incoming', async (t) => { + const f = await fixture(); t.after(f.close); await f.register(); + const bad = await f.send('push/event', f.params('1', 'x', { key: 'k'.repeat(257) })); + assert.equal(bad.error.code, -32602); + const both = await f.send('push/event', f.params('2', 'x', { deferred: true, retract: true })); + assert.equal(both.error.code, -32602); + const missing = { ...f.channel('bad', 'bad'), eventId: undefined }; + const deferred = f.channel('bad2', 'bad2', { deferred: true }); + const r = await f.send('channels/incoming', { messages: [missing, f.channel('good', 'good', { initial: true }), deferred] }); + assert.deepEqual(r.result.results.map((x: { accepted: boolean }) => x.accepted), [false, true, false]); + assert.equal(r.result.results[0].reason, 'coalesce_invalid'); + assert.equal(r.result.results[2].reason, 'coalesce_invalid'); + await f.framework.runUntilIdle(); + assert(f.lastRequest().includes('good')); +}); + +// --------------------------------------------------------------------------- +// Channel scope, mixed lanes (vectors 8, 10, 13, 14, 15a, 43) +// --------------------------------------------------------------------------- + +test('vector 10: channel create, push edit, push delete share one unread subject', async (t) => { + const f = await fixture(); t.after(f.close); await f.register(); + const first = await f.send('channels/incoming', { messages: [f.channel('c', 'chat_original', { initial: true })] }); + const edit = await f.send('push/event', f.params('e', 'chat_edit', { channelId: 'chat', key: 'message:m' })); + const del = await f.send('push/event', f.params('d', 'chat_deleted', { channelId: 'chat', key: 'message:m', retract: true })); + assert.equal(first.result.results[0].coalesce.outcome, 'first'); + assert.equal(edit.result.coalesce.outcome, 'replaced'); + assert.equal(del.result.coalesce.outcome, 'retracted'); + await f.framework.runUntilIdle(); + assert.equal(f.membrane.calls.length, 0); + for (const text of ['chat_original', 'chat_edit', 'chat_deleted']) assert(!f.context().includes(text)); +}); + +test('vector 8: repeated channel edits keep the platform message id and deduplicate a retried occurrence', async (t) => { + const f = await fixture(); t.after(f.close); await f.register(); + await f.send('channels/incoming', { messages: [f.channel('c', 'original', { initial: true })] }); + const edit = await f.send('channels/incoming', { messages: [f.channel('e', 'newest')] }); + const retry = await f.send('channels/incoming', { messages: [f.channel('e', 'newest')] }); + assert.deepEqual(retry.result, edit.result); + assert.equal(edit.result.results[0].coalesce.outcome, 'replaced'); + await f.framework.runUntilIdle(); + assert.equal(f.membrane.calls.length, 1); + assert(f.lastRequest().includes('newest') && !f.lastRequest().includes('original')); + const stored = f.framework.getAgent('agent')!.getContextManager().getAllMessages().find((m) => m.metadata?.eventId === 'e'); + assert.equal(stored?.metadata?.messageId, 'm'); +}); + +test('vector 43: a mixed-lane edit keeps the stable reply target', async (t) => { + const f = await fixture(); t.after(f.close); await f.register(); + await f.send('channels/incoming', { messages: [{ ...f.channel('c', 'first_version', { initial: true }), threadId: 'thread' }] }); + await f.framework.runUntilIdle(); + await f.send('push/event', { ...f.params('e', 'second_version', { channelId: 'chat', key: 'message:m' }), origin: { messageId: 'm', authorId: 'u', authorName: 'User', threadId: 'thread' } }); + await f.turn(); + assert.equal(f.published.at(-1)?.channelId, 'chat'); +}); + +test('vectors 13/15: a channel-scoped push needs current channel authority and a declared channel', async (t) => { + const f = await fixture(); t.after(f.close); + const unknown = await f.send('push/event', f.params('1', 'text', { channelId: 'chat' })); + assert.equal(unknown.error.code, -32023); + // A channel id that only ever appeared in a push's origin is not declared. + await f.send('push/event', { featureSet: 'doc', eventId: 'dm', timestamp: TS, payload: { content: [{ type: 'text', text: 'dm' }] }, + tags: ['chat:dm'], origin: { source: 'discord', channelId: '42', guildId: null, authorId: 'u', authorName: 'User' } }); + await f.framework.runUntilIdle(); + const forged = await f.send('push/event', f.params('2', 'text', { channelId: 'discord:dm:42' })); + assert.equal(forged.error.code, -32023); + await f.register(); + const okNow = await f.send('push/event', f.params('3', 'text', { channelId: 'chat', initial: true })); + assert.equal(okNow.result.coalesce.outcome, 'first'); +}); + +test('vector 14: scope is never inferred from origin', async (t) => { + const f = await fixture(); t.after(f.close); await f.register('chat:one'); + await f.send('channels/incoming', { messages: [f.channel('a', 'channel_one', { initial: true }, 'chat:one')] }); + await f.send('push/event', { ...f.params('feature', 'feature_scope', { key: 'message:m' }), origin: { channelId: 'chat:one' } }); + await f.framework.runUntilIdle(); + assert(f.lastRequest().includes('channel_one') && f.lastRequest().includes('feature_scope')); +}); + +test('vector 33 (channel lane): an ordinary channels/incoming message without coalesce is unchanged', async (t) => { + const f = await fixture(); t.after(f.close); await f.register(); + const r = await f.send('channels/incoming', { messages: [{ channelId: 'chat', messageId: 'p', timestamp: TS, author: { id: 'u', name: 'User' }, content: [{ type: 'text', text: 'plain' }] }] }); + assert.deepEqual(r.result.results[0], { messageId: 'p', accepted: true }); +}); + +// --------------------------------------------------------------------------- +// Wake treatment (vectors 36, 43 rev-6 behavioural) +// --------------------------------------------------------------------------- + +test('a retraction withdraws the unstarted wake its subject queued', async (t) => { + const f = await fixture({ server: { shouldTriggerInference: () => true } }); t.after(f.close); + const internals = f.framework as unknown as { pendingRequests: unknown[] }; + await f.send('push/event', f.params('1', 'withdrawn', { initial: true })); + assert.equal(internals.pendingRequests.length, 1); + await f.send('push/event', f.params('2', 'gone', { retract: true })); + assert.equal(internals.pendingRequests.length, 0); + await f.framework.runUntilIdle(); + assert.equal(f.membrane.calls.length, 0); +}); + +test('a replacement is evaluated by the gate as a fresh occurrence; a qualifying replacement wakes once', async (t) => { + const f = await fixture({ framework: { gate: { config: { policies: [{ name: 'attention', match: { tagsAny: ['doc:wake'] }, behavior: 'always' }], default: 'skip' } } } as never }); t.after(f.close); + const internals = f.framework as unknown as { pendingRequests: unknown[] }; + await f.send('push/event', { ...f.params('1', 'quiet', { initial: true }), tags: ['doc:quiet'] }); + assert.equal(internals.pendingRequests.length, 0); + await f.send('push/event', { ...f.params('2', 'loud', {}), tags: ['doc:wake'] }); + assert.equal(internals.pendingRequests.length, 1); + await f.send('push/event', { ...f.params('3', 'louder', {}), tags: ['doc:wake'] }); + assert.equal(internals.pendingRequests.length, 1, 'no second wake for the same subject'); + await f.send('push/event', { ...f.params('4', 'quiet_again', {}), tags: ['doc:quiet'] }); + assert.equal(internals.pendingRequests.length, 0, 'a non-qualifying replacement withdraws the wake'); + await f.framework.runUntilIdle(); + assert.equal(f.membrane.calls.length, 0); +}); + +// --------------------------------------------------------------------------- +// Deferred rendering (vectors 16–27) +// --------------------------------------------------------------------------- + +test('vectors 16/17/18: notices render once at assembly with their private data; never the fallback', async (t) => { + const f = await fixture(); t.after(f.close); + await f.send('push/event', f.params('1', 'fallback_unavailable', { deferred: true, data: { private: 'hidden_notice' } })); + const joined = await f.send('push/event', f.params('2', 'fallback_unavailable', { deferred: true, data: { private: 'hidden_notice_2' } })); + assert.equal(joined.result.coalesce.outcome, 'replaced'); + assert.equal(f.renders.length, 0); + assert(!f.context().includes('fallback_unavailable'), 'a batch is not model-visible'); + await f.framework.runUntilIdle(); + assert.equal(f.renders.length, 1); + assert.equal(f.renders[0].notices ? (f.renders[0].notices as unknown[]).length : 0, 2); + assert.deepEqual((f.renders[0].notices as Array<{ data: unknown }>).map((n) => n.data), [{ private: 'hidden_notice' }, { private: 'hidden_notice_2' }]); + const req = f.lastRequest(); + assert(req.includes('document_diff') && !req.includes('fallback_unavailable') && !req.includes('hidden_notice')); + // vector 18: a second inference issues no render; vector 19: a new notice does. + await f.turn(); + assert.equal(f.renders.length, 1); + const again = await f.send('push/event', f.params('3', 'fallback', { deferred: true })); + assert.equal(again.result.coalesce.outcome, 'first'); + await f.turn(); + assert.equal(f.renders.length, 2); +}); + +test('vector 20: an empty render appends nothing', async (t) => { + const f = await fixture(); t.after(f.close); + f.renderer(async () => ({ content: [] })); + await f.send('push/event', f.params('1', 'fallback', { deferred: true })); + await f.framework.runUntilIdle(); + assert.equal(f.renders.length, 1); + assert(!f.lastRequest().includes('fallback')); +}); + +test('vectors 21/23: a render past the deadline materializes the fallback; the late result is discarded', async (t) => { + const f = await fixture(); t.after(f.close); + f.renderer(async () => { await new Promise((r) => setTimeout(r, 5300)); return { content: [{ type: 'text', text: 'wire_late_payload' }] }; }); + await f.send('push/event', f.params('1', 'bounded_fallback', { deferred: true })); + await f.framework.runUntilIdle(); + assert(f.lastRequest().includes('bounded_fallback') && !f.lastRequest().includes('wire_late_payload')); + await new Promise((r) => setTimeout(r, 400)); + assert(!f.context().includes('wire_late_payload')); +}); + +test('vector 21: a render error uses the fallback', async (t) => { + const f = await fixture(); t.after(f.close); + f.renderer(async () => ({ content: [null] })); + await f.send('push/event', f.params('1', 'fallback_text', { deferred: true })); + await f.framework.runUntilIdle(); + assert(f.lastRequest().includes('fallback_text')); +}); + +test('vector 26: inference/request during push/render is refused', async (t) => { + const f = await fixture(); t.after(f.close); + let refused: unknown; + f.renderer(async () => { refused = (await f.send('inference/request', { featureSet: 'doc', messages: [] })).error?.code; return { content: [{ type: 'text', text: 'rendered' }] }; }); + await f.send('push/event', f.params('1', 'fallback', { deferred: true })); + await f.framework.runUntilIdle(); + assert.equal(refused, -32600); +}); + +test('vectors 27a/27c: retract or plain replacement during a render cancels the frozen batch', async (t) => { + for (const operation of ['retract', 'plain'] as const) { + const f = await fixture(); + try { + let release!: (value: unknown) => void; + let began!: () => void; + const started = new Promise((resolve) => { began = resolve; }); + f.renderer(() => { began(); return new Promise((resolve) => { release = resolve; }); }); + await f.send('push/event', f.params('1', 'fallback_one', { deferred: true, initial: true })); + const run = f.framework.runUntilIdle(); + await started; + const r = await f.send('push/event', operation === 'retract' + ? f.params('2', 'deletion_notice', { retract: true }) + : f.params('2', 'complete_snapshot')); + assert.equal(r.result.coalesce.outcome, operation === 'retract' ? 'retracted' : 'replaced'); + // The plain replacement arrived while the turn was alive: it lands at + // the turn boundary and wakes the agent again, so a second turn follows. + if (operation === 'plain') f.alwaysRespond(); + release({ content: [{ type: 'text', text: 'late_old_render' }] }); + await run; + const req = f.lastRequest(); + assert(!req.includes('late_old_render') && !req.includes('fallback_one') && !req.includes('deletion_notice')); + if (operation === 'plain') assert(req.includes('complete_snapshot')); + } finally { await f.close(); } + } +}); + +test('vector 24: revoked authority before assembly drops the batch without a render', async (t) => { + const f = await fixture(); t.after(f.close); + await f.send('push/event', f.params('1', 'previously_permitted', { deferred: true })); + await f.framework.stop(); await f.create(false); + await f.framework.runUntilIdle(); + assert.equal(f.renders.length, 0); + assert(!f.context().includes('previously_permitted')); +}); + +test('vector 27i: a deferred notice displaces an unread plain occurrence', async (t) => { + const f = await fixture(); t.after(f.close); + await f.send('push/event', f.params('1', 'plain_unread', { initial: true })); + const r = await f.send('push/event', f.params('2', 'fallback', { deferred: true })); + assert.equal(r.result.coalesce.outcome, 'replaced'); + assert(!f.context().includes('plain_unread')); + await f.framework.runUntilIdle(); + assert(f.lastRequest().includes('document_diff') && !f.lastRequest().includes('plain_unread')); +}); + +// --------------------------------------------------------------------------- +// Recovery (vectors 7, 39, 40, 41, 47) and the findings from PR #196 +// --------------------------------------------------------------------------- + +test('vector 41 (#196 finding 1): a transient reconnect does not reject ordinary channel traffic', async (t) => { + const f = await fixture({ server: { reconnect: true, reconnectIntervalMs: 20 } }); t.after(f.close); await f.register(); + const plain = (id: string) => ({ channelId: 'chat', messageId: id, timestamp: TS, author: { id: 'u', name: 'User' }, content: [{ type: 'text', text: 'plain_' + id }] }); + const before = await f.send('channels/incoming', { messages: [plain('1')] }); + f.disconnect(); + await eventually(() => f.hostCaps.length === 2, 'reconnect'); + await new Promise((r) => setTimeout(r, 100)); + const after = await f.send('channels/incoming', { messages: [plain('2')] }); + assert.equal(before.result.results[0].accepted, true); + assert.equal(after.result.results[0].accepted, true); +}); + +test('vector 39/40: a reconnect keeps accepted content and answers a retry with its receipt', async (t) => { + const f = await fixture({ server: { reconnect: true, reconnectIntervalMs: 20 } }); t.after(f.close); await f.register(); + const params = { messages: [f.channel('c', 'survives_disconnect', { initial: true })] }; + const accepted = await f.send('channels/incoming', params); + f.disconnect(); + await eventually(() => f.hostCaps.length === 2, 'reconnect'); + await f.register(); + const retry = await f.send('channels/incoming', params); + assert.deepEqual(retry.result, accepted.result); + await f.framework.runUntilIdle(); + assert.equal(f.membrane.calls.length, 1); + const req = f.lastRequest(); + assert.equal(req.split('survives_disconnect').length - 1, 1, 'exactly once'); +}); + +test('vectors 7/47: restart keeps accepted content, receipts, and appends after it', async (t) => { + const f = await fixture(); t.after(f.close); + const params = f.params('once', 'exactly_once', { initial: true }); + const original = await f.send('push/event', params); + await f.framework.stop(); await f.create(); + const retry = await f.send('push/event', params); + assert.deepEqual(retry.result, original.result); + const next = await f.send('push/event', f.params('two', 'after_restart')); + assert.equal(next.result.coalesce.outcome, 'appended'); + await f.framework.runUntilIdle(); + const req = f.lastRequest(); + assert(req.includes('exactly_once') && req.includes('after_restart')); + assert.equal(req.split('exactly_once').length - 1, 1); +}); + +test('#196 finding 2 / vector 34c: a busy channel never hits a subject cap', async (t) => { + const f = await fixture({ framework: { gate: { config: { policies: [{ name: 'attention', match: { tagsAny: ['chat:mention'] }, behavior: 'always' }], default: 'skip' } } } as never }); t.after(f.close); await f.register(); + const msg = (i: number | string, tags: string[]) => ({ channelId: 'chat', messageId: 'm' + i, eventId: 'e' + i, timestamp: TS, author: { id: 'u', name: 'User' }, tags, + content: [{ type: 'text', text: 'text_' + i }], coalesce: { key: 'message:m' + i, initial: true } }); + const t0 = Date.now(); + for (let i = 0; i < 300; i++) { + const r = await f.send('channels/incoming', { messages: [msg(i, [])] }); + assert.equal(r.result.results[0].accepted, true); + } + const elapsed = Date.now() - t0; + await f.framework.runUntilIdle(); + assert.equal(f.membrane.calls.length, 0); + const mention = await f.send('channels/incoming', { messages: [msg('mention', ['chat:mention'])] }); + assert.equal(mention.result.results[0].accepted, true); + await f.framework.runUntilIdle(); + assert.equal(f.membrane.calls.length, 1); + assert(f.lastRequest().includes('text_mention') && f.lastRequest().includes('text_0')); + // Each acceptance awaits a group-committed fsync (RFC-006 durability); serial + // sends cannot batch, so this is ~300 fsyncs. Bounded, and reported. + console.log(`# 300 serial coalesced channel messages: ${elapsed}ms (${(elapsed / 300).toFixed(1)} ms/msg incl. fsync)`); + assert(elapsed < 30_000, `300 coalesced messages took ${elapsed}ms`); +}); + +test('#196 finding 4: a context-budget restart does not assemble a pending batch', async (t) => { + const f = await fixture(); t.after(f.close); + await f.send('push/event', f.params('1', 'pending_batch', { deferred: true })); + const internals = f.framework as unknown as { startAgentStream(a: unknown, t: unknown): Promise; pendingRequests: unknown[] }; + internals.pendingRequests.length = 0; + await internals.startAgentStream(f.framework.getAgent('agent'), { agentName: 'agent', reason: 'context_budget_restart', source: 'probe', timestamp: Date.now() }); + await f.framework.runUntilIdle(); + assert.equal(f.renders.length, 0, 'not rendered mid-logical-turn'); + assert(!f.lastRequest().includes('document_diff')); + const joined = await f.send('push/event', f.params('2', 'pending_batch', { deferred: true })); + assert.equal(joined.result.coalesce.outcome, 'replaced', 'the batch survived the restart turn'); + await f.turn(); + assert.equal(f.renders.length, 1, 'rendered at the next fresh turn'); + assert.equal((f.renders[0].notices as unknown[]).length, 2); +}); + +test('#196 finding 5: undo of a turn keeps the coalesced input that preceded it', async (t) => { + const f = await fixture({ server: { shouldTriggerInference: () => true } }); t.after(f.close); + await f.send('push/event', f.params('a', 'turn_one', { key: 'a', initial: true })); + await f.framework.runUntilIdle(); + await f.send('push/event', f.params('b', 'unread_before_undo', { key: 'b', initial: true })); + await f.turn(); + assert(f.lastRequest().includes('unread_before_undo')); + f.framework.undoLastTurn('agent'); + assert(f.context().includes('unread_before_undo'), 'the input survives the undo of the turn that read it'); + const r = await f.send('push/event', f.params('c', 'after_undo', { key: 'b' })); + assert.equal(r.result.coalesce.outcome, 'appended', 'after a branch switch nothing is replaceable'); +}); + +test('a replacement while a turn is alive edits the deferred queue, not the live turn', async (t) => { + const f = await fixture(); t.after(f.close); + const internals = f.framework as unknown as { activeTurnTokens: Map; deferredMessages: unknown[] }; + internals.activeTurnTokens.set('agent', 100); + await f.send('push/event', f.params('1', 'queued_original', { initial: true })); + const r = await f.send('push/event', f.params('2', 'queued_replacement')); + assert.equal(r.result.coalesce.outcome, 'replaced'); + assert.equal(internals.deferredMessages.length, 1); + assert(!f.context().includes('queued_original') && !f.context().includes('queued_replacement')); + internals.activeTurnTokens.delete('agent'); + await f.turn(); + assert(f.lastRequest().includes('queued_replacement') && !f.lastRequest().includes('queued_original')); +}); + +test('compression consumes: a message folded into a summary is no longer replaceable', async (t) => { + const { AutobiographicalStrategy } = await import('@animalabs/context-manager'); + const strategy = new AutobiographicalStrategy({ + adaptiveResolution: true, foldingStrategy: 'kv-stable', recentWindowTokens: 1_000, targetChunkTokens: 300, + kvStableReachTokens: 300, autoTickOnNewMessage: false, compressionModel: 'summarizer', + } as never); + const f = await fixture({ agents: [{ name: 'agent', model: 'test', systemPrompt: 'test', strategy }] }); t.after(f.close); + (f.membrane as unknown as { complete: unknown }).complete = async (req: unknown) => { f.membrane.calls.push(req as never); return createMockResponse([{ type: 'text', text: 'SUMMARY' }]); }; + await f.send('push/event', f.params('1', 'folded_original', { initial: true })); + const cm = f.framework.getAgent('agent')!.getContextManager(); + for (let i = 0; i < 30; i++) cm.addMessage('user', [{ type: 'text', text: `filler_${i} ` + 'lorem ipsum '.repeat(40) }]); + for (let i = 0; i < 12; i++) await cm.tick(); + assert(cm.getMaxSummaryLevel() > 0, 'folding happened'); + const r = await f.send('push/event', f.params('2', 'after_fold')); + assert.equal(r.result.coalesce.outcome, 'appended'); + assert(f.context().includes('folded_original') && f.context().includes('after_fold')); +}); + +test('conversation forks: a channel-scoped edit follows the routed fork and pins its reply channel', async (t) => { + const f = await fixture({ framework: { conversations: { templateAgent: 'agent', bind: { channel: 'always' }, trigger: { channel: 'always' } } } as never }); t.after(f.close); await f.register(); + await f.send('channels/incoming', { messages: [f.channel('c', 'fork_original', { initial: true })] }); + const edit = await f.send('push/event', f.params('e', 'fork_latest', { channelId: 'chat', key: 'message:m' })); + assert.equal(edit.result.coalesce.outcome, 'replaced'); + f.alwaysRespond(); + await f.framework.runUntilIdle(); + assert.equal(f.membrane.calls.length, 1); + assert(f.lastRequest().includes('fork_latest') && !f.lastRequest().includes('fork_original')); + assert(!f.context().includes('fork_latest'), 'the trunk never sees channel traffic'); + assert.equal(f.published.at(-1)?.channelId, 'chat'); +}); + +// --------------------------------------------------------------------------- +// Review round 1 (#197): Greptile G1–G15, Codex C1–C3 +// --------------------------------------------------------------------------- + +test('C1: a channel-scoped render result is dropped when channels.incoming is revoked mid-render', async (t) => { + const f = await fixture(); t.after(f.close); await f.register(); + const { CapabilityGrant } = await import('../src/mcpl/capability-grant.js'); + let release!: (v: unknown) => void; + const started = new Promise((resolve) => { f.renderer(() => { resolve(); return new Promise((r) => { release = r; }); }); }); + await f.send('push/event', f.params('1', 'fallback', { deferred: true, channelId: 'chat', key: 'activity' })); + const run = f.framework.runUntilIdle(); + await started; + const conn = (f.framework as unknown as { mcplServerRegistry: { getServer(id: string): { establishGrant(g: unknown): void; grant: unknown } } }).mcplServerRegistry.getServer('editor'); + conn.establishGrant(new CapabilityGrant(new Set(['pushEvents']), [])); + release({ content: [{ type: 'text', text: 'revoked_render' }] }); + await run; + assert(!f.context().includes('revoked_render') && !f.context().includes('fallback')); +}); + +test('C2 / vector 42: reassigning the server id to another endpoint starts a new binding', async (t) => { + const f = await fixture(); t.after(f.close); + const a = await f.send('push/event', f.params('E', 'from_endpoint_a', { key: 'K', initial: true })); + assert.equal(a.result.coalesce.outcome, 'first'); + const config = (f.framework as unknown as { mcplServerConfigs: Map }).mcplServerConfigs.get('editor')!; + await f.framework.restartMcplServer('editor', { ...config, url: `${config.url}/reassigned` } as never); + await eventually(() => f.hostCaps.length === 2, 'reconnect to the new endpoint'); + const b = await f.send('push/event', f.params('E', 'from_endpoint_b', { key: 'K', initial: true })); + assert.equal(b.result.coalesce.outcome, 'first', "B's reuse of A's event id is a new occurrence"); + const b2 = await f.send('push/event', f.params('E2', 'from_endpoint_b_2', { key: 'K' })); + assert.equal(b2.result.coalesce.outcome, 'replaced'); + assert.equal(b2.result.coalesce.priorEventId, 'E'); + assert(f.context().includes('from_endpoint_a'), "A's unread content is not touched by B"); + assert(f.context().includes('from_endpoint_b_2') && !f.context().includes('from_endpoint_b"')); +}); + +test('C3: a replacement follows the prior delivery into its fork, never into a fresh one', async (t) => { + const f = await fixture({ framework: { conversations: { templateAgent: 'agent', bind: { channel: 'always' }, trigger: { channel: 'always' } } } as never }); t.after(f.close); await f.register(); + await f.send('channels/incoming', { messages: [f.channel('c', 'fork_original', { initial: true })] }); + const router = (f.framework as unknown as { conversationRouter: { getBinding(id: string): { agentName: string } | undefined; unbind(id: string): void } }).conversationRouter; + const g1 = router.getBinding('chat')!.agentName; + router.unbind('chat'); + const edit = await f.send('push/event', f.params('e', 'fork_edit', { channelId: 'chat', key: 'message:m' })); + assert.equal(edit.result.coalesce.outcome, 'replaced'); + assert.equal(router.getBinding('chat'), undefined, 'no fresh fork was spawned for the edit'); + const g1ctx = JSON.stringify(f.framework.getAgent(g1)!.getContextManager().getAllMessages()); + assert(g1ctx.includes('fork_edit') && !g1ctx.includes('fork_original')); +}); + +test('G1: malformed content on a coalesced channel item fails that item only', async (t) => { + const f = await fixture(); t.after(f.close); await f.register(); + const bad = { ...f.channel('bad', 'x', { initial: true }, 'chat', 'b'), content: 'not-an-array' }; + const r = await f.send('channels/incoming', { messages: [bad, f.channel('good', 'good_sibling', { initial: true }, 'chat', 'g')] }); + assert.deepEqual(r.result.results.map((x: { accepted: boolean }) => x.accepted), [false, true]); + assert.equal(r.result.results[0].reason, 'coalesce_invalid'); + const badPush = await f.send('push/event', { ...f.params('p', 'x'), payload: { content: [{ type: 'text' }] } }); + assert.equal(badPush.error.code, -32602); +}); + +test('G3/G5: a deferred channel-scoped push renders with its feature set and wakes with its channel', async (t) => { + const f = await fixture({ server: { shouldTriggerInference: () => true } }); t.after(f.close); await f.register(); + await f.send('push/event', { ...f.params('1', 'fallback', { deferred: true, channelId: 'chat', key: 'activity' }), origin: { authorId: 'u', authorName: 'User' } }); + const internals = f.framework as unknown as { pendingRequests: Array<{ channelId?: string; counterparty?: string }> }; + assert.equal(internals.pendingRequests[0]?.channelId, 'chat'); + assert.equal(internals.pendingRequests[0]?.counterparty, 'editor:user:u'); + await f.framework.runUntilIdle(); + assert.equal(f.renders[0]?.featureSet, 'doc'); + assert.equal(f.renders[0]?.channelId, 'chat'); + assert.equal(f.published.at(-1)?.channelId, 'chat', 'the reply goes to the batch\'s channel'); +}); + +test('G4: a deferred channel-scoped push spawns the conversation fork it needs', async (t) => { + const f = await fixture({ framework: { conversations: { templateAgent: 'agent', bind: { channel: 'always' }, trigger: { channel: 'always' } } } as never, server: { shouldTriggerInference: () => true } }); t.after(f.close); await f.register(); + await f.send('push/event', f.params('1', 'fallback', { deferred: true, channelId: 'chat', key: 'activity' })); + f.alwaysRespond(); + await f.framework.runUntilIdle(); + assert.equal(f.renders.length, 1); + const router = (f.framework as unknown as { conversationRouter: { getBinding(id: string): { agentName: string } | undefined } }).conversationRouter; + const fork = router.getBinding('chat')?.agentName; + assert(fork, 'a fork was bound'); + assert(JSON.stringify(f.framework.getAgent(fork!)!.getContextManager().getAllMessages()).includes('document_diff')); + assert(!f.context().includes('document_diff')); +}); + +test('G6: a replacement finds its prior after the deferred flush stored it', async (t) => { + const f = await fixture(); t.after(f.close); + const internals = f.framework as unknown as { activeTurnTokens: Map; flushDeferredWrites(label: string): Promise }; + internals.activeTurnTokens.set('agent', 100); + await f.send('push/event', f.params('1', 'queued_then_stored', { initial: true })); + internals.activeTurnTokens.delete('agent'); + await internals.flushDeferredWrites('test'); + assert(f.context().includes('queued_then_stored'), 'flushed into context, unread'); + const r = await f.send('push/event', f.params('2', 'replacement_after_flush')); + assert.equal(r.result.coalesce.outcome, 'replaced'); + assert(!f.context().includes('queued_then_stored') && f.context().includes('replacement_after_flush')); + const del = await f.send('push/event', f.params('3', 'notice', { retract: true })); + assert.equal(del.result.coalesce.outcome, 'retracted'); + assert(!f.context().includes('replacement_after_flush')); +}); + +test('G9: a receipt is durable before the acceptance is acknowledged', async (t) => { + const f = await fixture(); t.after(f.close); + await f.send('push/event', f.params('durable', 'x', { initial: true })); + const store = (f.framework as unknown as { store: { getStateJson(id: string): unknown } }).store; + const recent = store.getStateJson('mcpl/coalescing-recent') as Array<{ key: string; born?: string }>; + assert.equal(recent.length, 1); + assert(recent[0].key.includes('"durable"')); + assert.equal(recent[0].born, 'none'); +}); + +test('G10: a batch that never began rendering renders after a clean restart', async (t) => { + const f = await fixture(); t.after(f.close); + await f.send('push/event', f.params('1', 'fallback_text', { deferred: true })); + await f.framework.stop(); await f.create(); + await f.framework.runUntilIdle(); + assert.equal(f.renders.length, 1, 'push/render issued after restart'); + assert(f.lastRequest().includes('document_diff') && !f.lastRequest().includes('fallback_text')); +}); + +test('G11: a push edit without origin identity keeps the channel message identity', async (t) => { + const f = await fixture(); t.after(f.close); await f.register(); + await f.send('channels/incoming', { messages: [{ ...f.channel('c', 'first_version', { initial: true }), threadId: 'thread' }] }); + await f.send('push/event', f.params('e', 'second_version', { channelId: 'chat', key: 'message:m' })); + const stored = f.framework.getAgent('agent')!.getContextManager().getAllMessages().find((m) => m.metadata?.eventId === 'e'); + assert.equal(stored?.metadata?.messageId, 'm'); + assert.equal(stored?.metadata?.threadId, 'thread'); + assert.deepEqual(stored?.metadata?.author, { id: 'u', name: 'User' }); +}); + +test('G13: modules see coalesced deliveries', async (t) => { + const seen: string[] = []; + const spy = { name: 'spy', async start() {}, async stop() {}, async onProcess(e: { type: string }) { seen.push(e.type); return {}; } }; + const f = await fixture({ framework: { modules: [spy] } as never }); t.after(f.close); await f.register(); + await f.send('push/event', f.params('1', 'x', { initial: true })); + await f.send('channels/incoming', { messages: [f.channel('c', 'y', { initial: true })] }); + assert.deepEqual(seen.filter((x) => x.startsWith('mcpl:')), ['mcpl:push-event', 'mcpl:channel-incoming']); +}); + +test('G15: inference/request is refused while a cancelled render is still outstanding', async (t) => { + const f = await fixture(); t.after(f.close); + let refused: unknown; + f.renderer(async () => { + await f.send('push/event', f.params('2', 'notice', { retract: true })); + refused = (await f.send('inference/request', { featureSet: 'doc', messages: [] })).error?.code; + return { content: [{ type: 'text', text: 'late' }] }; + }); + await f.send('push/event', f.params('1', 'fallback', { deferred: true, initial: true })); + await f.framework.runUntilIdle(); + assert.equal(refused, -32600); + assert(!f.lastRequest().includes('late') && !f.lastRequest().includes('fallback')); +}); + +// --------------------------------------------------------------------------- +// Review round 2 (#197): Codex R1–R4 on d86afc1 +// --------------------------------------------------------------------------- + +test('R1: a deferred acceptance survives a crash before the snapshot flush, with its receipt', async (t) => { + const f = await fixture(); t.after(f.close); + const params = f.params('1', 'crash_fallback', { deferred: true, data: { k: 'v' } }); + const original = await f.send('push/event', params); + await crash(f); await f.create(); + const retry = await f.send('push/event', params); + assert.deepEqual(retry.result, original.result, 'the receipt survived'); + await f.framework.runUntilIdle(); + assert.equal(f.membrane.calls.length, 1, 'the accepted work was delivered'); + assert(f.lastRequest().includes('crash_fallback'), 'as its fallback: the host cannot prove no render started'); + assert.equal(f.renders.length, 0); +}); + +test('R2: work admitted under a former binding never reaches the replacement endpoint', async (t) => { + const f = await fixture(); t.after(f.close); + await f.send('push/event', f.params('1', 'old_binding_fallback', { deferred: true, data: { secret: 'private_notice_data' } })); + const config = (f.framework as unknown as { mcplServerConfigs: Map }).mcplServerConfigs.get('editor')!; + await f.framework.restartMcplServer('editor', { ...config, url: `${config.url}/reassigned` } as never); + await eventually(() => f.hostCaps.length === 2, 'reconnect to the new endpoint'); + await f.framework.runUntilIdle(); + assert.equal(f.renders.length, 0, 'no push/render to the new peer'); + assert(!JSON.stringify(f.renders).includes('private_notice_data')); + assert(!f.context().includes('old_binding_fallback')); +}); + +test('R3: a deferred notice displacing an unread plain occurrence stays in its fork', async (t) => { + const f = await fixture({ framework: { conversations: { templateAgent: 'agent', bind: { channel: 'always' }, trigger: { channel: 'always' } } } as never, server: { shouldTriggerInference: () => true } }); t.after(f.close); await f.register(); + await f.send('channels/incoming', { messages: [f.channel('c', 'fork_plain', { initial: true })] }); + const router = (f.framework as unknown as { conversationRouter: { getBinding(id: string): { agentName: string } | undefined; unbind(id: string): void } }).conversationRouter; + const g1 = router.getBinding('chat')!.agentName; + router.unbind('chat'); + const r = await f.send('push/event', f.params('n', 'fallback', { deferred: true, channelId: 'chat', key: 'message:m' })); + assert.equal(r.result.coalesce.outcome, 'replaced'); + f.alwaysRespond(); + await f.framework.runUntilIdle(); + assert.equal(router.getBinding('chat'), undefined, 'no fresh fork'); + const g1ctx = JSON.stringify(f.framework.getAgent(g1)!.getContextManager().getAllMessages()); + assert(g1ctx.includes('document_diff') && !g1ctx.includes('fork_plain')); +}); + +test('R4: the render-start boundary is persisted before the RPC is issued', async (t) => { + const f = await fixture(); t.after(f.close); + let release!: (v: unknown) => void; + const started = new Promise((resolve) => { f.renderer(() => { resolve(); return new Promise((r) => { release = r; }); }); }); + await f.send('push/event', f.params('1', 'fallback', { deferred: true })); + const run = f.framework.runUntilIdle(); + await started; + const store = (f.framework as unknown as { store: { getStateJson(id: string): { subjects: Array<{ batch?: { rendering?: boolean } }> } } }).store; + assert.equal(store.getStateJson('mcpl/coalescing').subjects[0]?.batch?.rendering, true, 'persisted as RENDERING while the RPC is outstanding'); + release({ content: [{ type: 'text', text: 'rendered' }] }); + await run; + assert(f.lastRequest().includes('rendered')); +}); + +// --------------------------------------------------------------------------- +// Review round 3 (#197): Codex on c03b23b — recovery from a mid-window image +// --------------------------------------------------------------------------- + +test('R5: an acknowledged retraction is recoverable; a retry cannot resurrect the batch', async (t) => { + const f = await fixture(); t.after(f.close); + await f.send('push/event', f.params('1', 'withdrawn_fallback', { deferred: true, initial: true })); + const retract = f.params('2', 'notice', { retract: true }); + const first = await f.send('push/event', retract); + assert.equal(first.result.coalesce.outcome, 'retracted'); + await crash(f); await f.create(); + const retry = await f.send('push/event', retract); + assert.deepEqual(retry.result, first.result); + await f.framework.runUntilIdle(); + assert.equal(f.membrane.calls.length, 0); + assert(!f.context().includes('withdrawn_fallback') && !f.context().includes('notice')); +}); + +test('R6: a completed render is not delivered again as fallback after recovery', async (t) => { + const f = await fixture(); t.after(f.close); + await f.send('push/event', f.params('1', 'stale_fallback', { deferred: true })); + await f.framework.runUntilIdle(); + assert(f.lastRequest().includes('document_diff')); + await crash(f); await f.create(); + await f.framework.runUntilIdle(); + const ctx = f.context(); + assert.equal(ctx.split('document_diff').length - 1, 1, 'rendered content exactly once'); + assert(!ctx.includes('stale_fallback'), 'no second delivery as fallback'); + assert.equal(f.renders.length, 1); + // History is known: a retraction now appends the notice rather than withdrawing. + const r = await f.send('push/event', f.params('2', 'deleted_notice', { retract: true })); + assert.equal(r.result.coalesce.outcome, 'noted'); +}); + +test('R7: a read plain occurrence recovered from a mid-window image still gets its deletion notice', async (t) => { + const f = await fixture(); t.after(f.close); + await f.send('push/event', f.params('1', 'read_before_crash', { initial: true })); + await f.framework.runUntilIdle(); + assert(f.lastRequest().includes('read_before_crash')); + await crash(f); await f.create(); + const r = await f.send('push/event', f.params('2', 'deletion_notice_after_crash', { retract: true })); + assert.equal(r.result.coalesce.outcome, 'noted'); + assert(f.context().includes('read_before_crash') && f.context().includes('deletion_notice_after_crash')); + // And the unread variant stays traceless: born, never read, crash, retract. + await f.send('push/event', f.params('3', 'never_read', { key: 'k2', initial: true })); + await crash(f); await f.create(); + const r2 = await f.send('push/event', f.params('4', 'notice2', { key: 'k2', retract: true })); + assert.equal(r2.result.coalesce.outcome, 'noted', 'a stored occurrence counts as read after recovery (watermark at head)'); + assert(f.context().includes('never_read') && f.context().includes('notice2')); +}); + +// --------------------------------------------------------------------------- +// Review round 5 (#197): durability under SIGKILL (no stop, no close, no sync) +// --------------------------------------------------------------------------- + +async function killChild(mode: 'accept' | 'render'): Promise<{ dir: string; port: number; receipt: unknown }> { + const dir = mkdtempSync(join(tmpdir(), 'coalescing-kill-')); + const child = fork(join(dirname(fileURLToPath(import.meta.url)), 'helpers', 'coalescing-kill-child.js'), [dir, mode], { stdio: ['ignore', 'ignore', 'inherit', 'ipc'] }); + const [msg] = await once(child, 'message') as [{ port: number; receipt: unknown }]; + child.kill('SIGKILL'); + await once(child, 'exit'); + return { dir, port: msg.port, receipt: msg.receipt }; +} + +test('R8 (kill, acceptance): the receipt and the accepted work survive SIGKILL after the wire ack', async (t) => { + const { dir, port, receipt } = await killChild('accept'); + const f = await fixture({ dir, port }); t.after(async () => { await f.close(); rmSync(dir, { recursive: true, force: true }); }); + const retry = await f.send('push/event', f.params('1', 'killed_fallback', { deferred: true, data: { k: 'v' } })); + assert.deepEqual(retry.result, receipt, 'the receipt survived the kill'); + await f.framework.runUntilIdle(); + assert.equal(f.membrane.calls.length, 1, 'the accepted work survived the kill'); + assert(f.lastRequest().includes('killed_fallback')); +}); + +test('R9 (kill, render-start): a render that may have run is not issued again after SIGKILL', async (t) => { + const { dir, port } = await killChild('render'); + const f = await fixture({ dir, port }); t.after(async () => { await f.close(); rmSync(dir, { recursive: true, force: true }); }); + await f.framework.runUntilIdle(); + assert.equal(f.renders.length, 0, 'no second push/render'); + assert(f.lastRequest().includes('render_fallback'), 'the fallback is delivered instead'); +}); + +// --------------------------------------------------------------------------- +// Review round 6 (#197): the asynchronous barrier's failure and interleaving paths +// --------------------------------------------------------------------------- + +test('R10: a retry after a failed commit is acknowledged only once the commit succeeds', async (t) => { + const f = await fixture(); t.after(f.close); + const store = (f.framework as unknown as { store: { sync: () => void } }).store; + const orig = store.sync.bind(store); + let syncs = 0; let failing = 1; + store.sync = () => { syncs++; if (failing > 0) { failing--; throw new Error('EIO simulated'); } orig(); }; + const params = f.params('1', 'fallback', { deferred: true, initial: true }); + const first = await f.send('push/event', params); + assert.equal(first.error?.code, -32000); + const before = syncs; + const retry = await f.send('push/event', params); + assert.equal(retry.result?.coalesce?.outcome, 'first', 'acknowledged on the retry'); + assert(syncs > before, 'the retry committed; it did not just return the pending receipt'); + // Still failing: the retry fails again rather than acknowledging. + failing = 1; + const again = await f.send('push/event', f.params('2', 'other', { key: 'k2', initial: true })); + assert.equal(again.error?.code, -32000); + const again2 = await f.send('push/event', f.params('2', 'other', { key: 'k2', initial: true })); + assert.equal(again2.result?.coalesce?.outcome, 'first'); + assert.equal(f.framework.getAgent('agent')!.getContextManager().getAllMessages().filter((m) => JSON.stringify(m.content).includes('other')).length, 1, 'plain: delivered exactly once across the failed and successful acknowledgement'); +}); + +/** Hold the host's next durability commit; returns a release function once it is held. */ +function holdNextCommit(f: Awaited>): Promise<() => void> { + const internals = f.framework as unknown as { commitCoalescingDurable: () => Promise }; + const orig = internals.commitCoalescingDurable.bind(internals); + return new Promise((held) => { + internals.commitCoalescingDurable = () => { + internals.commitCoalescingDurable = orig; + return new Promise((resolve) => { held(() => orig().then(resolve, resolve)); }); + }; + }); +} + +test('R11: a retraction during the render-start commit wait is final; the batch is not restored', async (t) => { + const f = await fixture(); t.after(f.close); + await f.send('push/event', f.params('1', 'withdrawn_fallback', { deferred: true, initial: true })); + const held = holdNextCommit(f); + const run = f.framework.runUntilIdle(); + const release = await held; + // The freeze (including this commit wait) is one critical section with + // admission: the retraction is admitted only after it, so it is sent + // without awaiting here and settles either way as a cancellation. + const pending = f.send('push/event', f.params('2', 'notice', { retract: true })); + release(); + const r = await pending; + assert.equal(r.result.coalesce.outcome, 'retracted'); + await run; + assert.equal((f.framework as unknown as { pushCoalescer: { pendingBatches(): number } }).pushCoalescer.pendingBatches(), 0); + assert(!f.context().includes('withdrawn_fallback') && !f.lastRequest().includes('document_diff'), 'nothing of the withdrawn batch was published'); + const renders = f.renders.length; + await f.turn(); + assert.equal(f.renders.length, renders, 'and nothing renders later'); + assert(!f.context().includes('withdrawn_fallback') && !f.context().includes('document_diff')); +}); + +test('R12: endpoint reassignment during the render-start commit wait never sends the batch to the new peer', async (t) => { + const f = await fixture(); t.after(f.close); + await f.send('push/event', f.params('1', 'old_fallback', { deferred: true, data: { secret: 'private_notice_data' } })); + const held = holdNextCommit(f); + const run = f.framework.runUntilIdle(); + const release = await held; + const config = (f.framework as unknown as { mcplServerConfigs: Map }).mcplServerConfigs.get('editor')!; + await f.framework.restartMcplServer('editor', { ...config, url: `${config.url}/reassigned` } as never); + await eventually(() => f.hostCaps.length === 2, 'reconnect to the new endpoint'); + release(); + await run; + assert.equal(f.renders.length, 0, 'no push/render to the new peer'); + assert(!JSON.stringify(f.renders).includes('private_notice_data')); + assert(!f.context().includes('old_fallback')); +}); + +// --------------------------------------------------------------------------- +// Review round 7 (#197): neighbours of the round-6 fixes +// --------------------------------------------------------------------------- + +test('R13: a retraction during the audience lookup is final; the stale batch is not frozen', async (t) => { + const f = await fixture(); t.after(f.close); + await f.send('push/event', f.params('1', 'withdrawn_fallback', { deferred: true, initial: true })); + const internals = f.framework as unknown as { coalescedAudience: (occ: unknown) => Promise }; + const orig = internals.coalescedAudience.bind(internals); + let release!: () => void; + const held = new Promise((resolve) => { + internals.coalescedAudience = (occ) => { internals.coalescedAudience = orig; return new Promise((res) => { release = () => orig(occ).then(res); resolve(); }); }; + }); + const run = f.framework.runUntilIdle(); + await held; + const pending = f.send('push/event', f.params('2', 'notice', { retract: true })); + release(); + const r = await pending; + assert.equal(r.result.coalesce.outcome, 'retracted'); + await run; + assert(!f.context().includes('withdrawn_fallback') && !f.context().includes('document_diff'), 'nothing of the withdrawn batch was published'); + await f.turn(); + assert(!f.context().includes('document_diff')); +}); + +test('R14: two consecutive crash windows with an acceptance in between keep both recovered batches', async (t) => { + const f = await fixture(); t.after(f.close); + await f.send('push/event', f.params('a', 'first_window_fallback', { deferred: true, key: 'A' })); + await crash(f); await f.create(); + await f.send('push/event', f.params('b', 'second_window_fallback', { deferred: true, key: 'B' })); + await crash(f); await f.create(); + const retryA = await f.send('push/event', f.params('a', 'first_window_fallback', { deferred: true, key: 'A' })); + assert.equal(retryA.result?.coalesce?.outcome, 'first', "A's receipt survived the second window"); + await f.framework.runUntilIdle(); + const req = f.lastRequest(); + assert(req.includes('first_window_fallback') && req.includes('second_window_fallback'), 'both recovered batches delivered'); +}); + +test('R15: a retraction arriving between the render result and its publication waits for the settlement', async (t) => { + const f = await fixture(); t.after(f.close); + await f.send('push/event', f.params('1', 'fallback', { deferred: true, initial: true })); + const internals = f.framework as unknown as { deliverCoalesced: (...a: unknown[]) => Promise }; + const orig = internals.deliverCoalesced.bind(internals); + let release!: () => void; + const held = new Promise((resolve) => { + internals.deliverCoalesced = (...a) => { internals.deliverCoalesced = orig; return new Promise((res) => { release = () => orig(...a).then(res); resolve(); }); }; + }); + f.alwaysRespond(); // the notice wakes a second turn + const run = f.framework.runUntilIdle(); + await held; // the render result is in hand; publication is being settled + const pending = f.send('push/event', f.params('2', 'deleted_notice', { retract: true })); + release(); + const r = await pending; + assert.equal(r.result.coalesce.outcome, 'noted', 'the retraction saw the published render as read'); + await run; + const ctx = f.context(); + assert.equal(ctx.split('document_diff').length - 1, 1, 'rendered exactly once'); + assert(ctx.includes('deleted_notice'), 'and corrected by the notice'); +}); + +// --------------------------------------------------------------------------- +// Review round 8 (#197): liveness after a failed render-start commit +// --------------------------------------------------------------------------- + +test('R16: a batch whose render-start commit failed is re-woken once storage recovers', async (t) => { + const f = await fixture(); t.after(f.close); + const coalescer = (f.framework as unknown as { pushCoalescer: { options: { recoveryBackoffMs?: number }; pendingBatches(): number } }).pushCoalescer; + coalescer.options.recoveryBackoffMs = 50; + const internals = f.framework as unknown as { pendingRequests: unknown[] }; + const store = (f.framework as unknown as { store: { sync: () => void } }).store; + const orig = store.sync.bind(store); + let failing = 0; + store.sync = () => { if (failing > 0) { failing--; throw new Error('EIO simulated'); } orig(); }; + await f.send('push/event', f.params('1', 'asleep_fallback', { deferred: true, initial: true })); + failing = 1; // the render-start commit fails; storage then recovers + await f.framework.runUntilIdle(); + assert.equal(f.renders.length, 0); + assert.equal(coalescer.pendingBatches(), 1, 'the batch is kept'); + assert(!f.lastRequest().includes('asleep_fallback')); + await eventually(() => internals.pendingRequests.length > 0, 'recovery wake'); + await f.turn(); + assert(f.lastRequest().includes('asleep_fallback'), 'delivered without unrelated traffic'); + assert.equal(coalescer.pendingBatches(), 0); +}); + +test('R16b: a producer retry re-wakes a batch left asleep by a failed freeze', async (t) => { + const f = await fixture(); t.after(f.close); + const coalescer = (f.framework as unknown as { pushCoalescer: { options: { recoveryBackoffMs?: number }; pendingBatches(): number } }).pushCoalescer; + coalescer.options.recoveryBackoffMs = 600_000; // park the timer; the retry must do it + const internals = f.framework as unknown as { pendingRequests: unknown[] }; + const store = (f.framework as unknown as { store: { sync: () => void } }).store; + const orig = store.sync.bind(store); + let failing = 0; + store.sync = () => { if (failing > 0) { failing--; throw new Error('EIO simulated'); } orig(); }; + const params = f.params('1', 'asleep_fallback', { deferred: true, initial: true }); + const first = await f.send('push/event', params); + failing = 1; + await f.framework.runUntilIdle(); + assert.equal(internals.pendingRequests.length, 0, 'asleep'); + const retry = await f.send('push/event', params); + assert.deepEqual(retry.result, first.result); + assert.equal(internals.pendingRequests.length, 1, 'the retry re-woke it'); + await f.turn(); + assert(f.lastRequest().includes('asleep_fallback')); +}); + +test('R17: a persistent storage outage backs off (1×, 2×, 4× …) and the series resets on recovery', async (t) => { + const f = await fixture(); t.after(f.close); + const coalescer = (f.framework as unknown as { pushCoalescer: { options: { recoveryBackoffMs?: number }; pendingBatches(): number; recovery: Map } }).pushCoalescer; + coalescer.options.recoveryBackoffMs = 40; + const internals = f.framework as unknown as { pendingRequests: unknown[] }; + const store = (f.framework as unknown as { store: { sync: () => void } }).store; + const orig = store.sync.bind(store); + let failing = false; + store.sync = () => { if (failing) throw new Error('EIO persistent'); orig(); }; + const wakes: Array<{ attempt: number; delayMs: number }> = []; + f.framework.onTrace((e) => { if (e.type === 'mcpl:coalescing' && (e as { kind?: string }).kind === 'recovery-wake') wakes.push(e as never); }); + await f.send('push/event', f.params('1', 'outage_fallback', { deferred: true, initial: true })); + failing = true; + f.alwaysRespond(); + for (let i = 0; i < 3; i++) { + await f.framework.runUntilIdle(); // freeze fails → recovery scheduled + await eventually(() => internals.pendingRequests.length > 0, `recovery wake ${i + 1}`); + } + assert.deepEqual(wakes.map((w) => w.attempt), [1, 2, 3], 'consecutive failures count up'); + assert.deepEqual(wakes.map((w) => w.delayMs), [40, 80, 160], 'and the delay doubles'); + assert.equal(coalescer.pendingBatches(), 1); + assert(!f.context().includes('outage_fallback')); + failing = false; // storage recovers + await f.framework.runUntilIdle(); + assert(f.lastRequest().includes('outage_fallback'), 'delivered once storage recovered'); + assert.equal(coalescer.pendingBatches(), 0); + assert.equal(coalescer.recovery.size, 0, 'the failure series is forgotten on success'); +}); + +// --------------------------------------------------------------------------- +// Review round 10 (#197): residents share one message slot +// --------------------------------------------------------------------------- + +for (const reader of ['other', 'agent'] as const) test(`R18 (${reader} read it): a message read by any resident of the shared slot is history for all of them`, async (t) => { + const f = await fixture({ agents: [{ name: 'agent', model: 'test', systemPrompt: 'test' }, { name: 'other', model: 'test', systemPrompt: 'test' }], server: { shouldTriggerInference: () => true } }); t.after(f.close); + await f.send('push/event', f.params('1', 'read_by_one', { initial: true })); + const internals = f.framework as unknown as { pendingRequests: Array<{ agentName: string }> }; + internals.pendingRequests = internals.pendingRequests.filter((r) => r.agentName === reader); + f.alwaysRespond(); + await f.framework.runUntilIdle(); + assert(f.lastRequest().includes('read_by_one'), `${reader} read it`); + const edit = await f.send('push/event', f.params('2', 'later_edit')); + assert.equal(edit.result.coalesce.outcome, 'appended'); + for (const name of ['agent', 'other']) { + const ctx = JSON.stringify(f.framework.getAgent(name)!.getContextManager().getAllMessages()); + assert(ctx.includes('read_by_one') && ctx.includes('later_edit'), `${name} keeps the consumed original`); + } + const del = await f.send('push/event', f.params('3', 'deletion_notice', { retract: true })); + assert.equal(del.result.coalesce.outcome, 'noted'); + assert(f.context().includes('read_by_one') && f.context().includes('deletion_notice') && !f.context().includes('later_edit')); +});