From 65201839bb48de881f91fbd832451900ab2d9680 Mon Sep 17 00:00:00 2001 From: "(CJ) Chukwudi Nwobodo" <142016413+chuks-qua@users.noreply.github.com> Date: Thu, 24 Sep 2026 19:24:35 +0100 Subject: [PATCH 001/136] perf(server): add bounded server work tracing --- .../canonical/canonical-agent-boundary.ts | 11 +- .../__tests__/server-work-trace.test.ts | 40 ++++++ .../agents/diagnostics/server-work-trace.ts | 128 ++++++++++++++++++ .../agent-event-publication-service.ts | 9 ++ .../features/agents/transport/agent-rpc.ts | 17 ++- .../agents/turns/turn-event-pipeline.ts | 10 +- .../transport/workspace-thread-rpc.ts | 27 ++-- .../composition/provider-event-ingress.ts | 35 ++++- .../terminal/transport/terminal-rpc.ts | 12 +- 9 files changed, 268 insertions(+), 21 deletions(-) create mode 100644 apps/server/src/features/agents/diagnostics/__tests__/server-work-trace.test.ts create mode 100644 apps/server/src/features/agents/diagnostics/server-work-trace.ts diff --git a/apps/server/src/features/agents/canonical/canonical-agent-boundary.ts b/apps/server/src/features/agents/canonical/canonical-agent-boundary.ts index 0d86a14e2..7f67102d4 100644 --- a/apps/server/src/features/agents/canonical/canonical-agent-boundary.ts +++ b/apps/server/src/features/agents/canonical/canonical-agent-boundary.ts @@ -41,6 +41,7 @@ import { type TurnOutcome, } from "@mcode/contracts"; import { broadcast } from "../../../application/transport/push.js"; +import { serverWorkTrace } from "../diagnostics/server-work-trace.js"; import { canonicalAgentEvents, canonicalAgentIngestCheckpoints, @@ -480,11 +481,17 @@ export class CanonicalAgentBoundary implements ParentTurnDurability, CodexCollab /** Commit one semantic batch and publish only the applied events after SQLite commits. */ commit(input: CanonicalAgentCommitInput): CanonicalAgentCommitResult { - return this.eventStore.commit(this.toEventStoreCommitInput(input)); + return serverWorkTrace + ? serverWorkTrace.measure("canonical-write", input.threadId, input.executionId, + () => this.eventStore.commit(this.toEventStoreCommitInput(input))) + : this.eventStore.commit(this.toEventStoreCommitInput(input)); } private commitInsideTransaction(input: CanonicalAgentCommitInput): CanonicalAgentCommitResult { - return this.eventStore.applyWithinTransaction(this.toEventStoreCommitInput(input)); + return serverWorkTrace + ? serverWorkTrace.measure("canonical-write", input.threadId, input.executionId, + () => this.eventStore.applyWithinTransaction(this.toEventStoreCommitInput(input))) + : this.eventStore.applyWithinTransaction(this.toEventStoreCommitInput(input)); } private toEventStoreCommitInput(input: CanonicalAgentCommitInput): CanonicalAgentEventStoreInput { diff --git a/apps/server/src/features/agents/diagnostics/__tests__/server-work-trace.test.ts b/apps/server/src/features/agents/diagnostics/__tests__/server-work-trace.test.ts new file mode 100644 index 000000000..741c6ec6a --- /dev/null +++ b/apps/server/src/features/agents/diagnostics/__tests__/server-work-trace.test.ts @@ -0,0 +1,40 @@ +import { describe, expect, it } from "vitest"; +import { ServerWorkTrace, type ServerWorkTraceReport } from "../server-work-trace.js"; + +describe("ServerWorkTrace", () => { + it("attributes aggregate work in a stalled tick and bounds content-free output", () => { + const reports: ServerWorkTraceReport[] = []; + const trace = new ServerWorkTrace((report) => reports.push(report)); + trace.tick(1_000); + for (let index = 0; index < 40; index += 1) { + trace.record("event-apply", "thread-a", "execution-a", 12); + } + trace.record("canonical-write", "thread-b", "execution-b", 30); + trace.record("event-apply", "secret prompt with spaces", "secret output with spaces", 1); + for (let index = 0; index < 80; index += 1) { + trace.record("worker-wait", `thread-${index}`, "execution-c", 1); + } + trace.tick(1_240); + + expect(reports).toHaveLength(1); + const report = reports[0]; + expect(report?.kind).toBe("server-work-stall"); + if (report?.kind !== "server-work-stall") return; + expect(report.delayMs).toBe(220); + expect(report.samples).toHaveLength(64); + expect(report.overflowCount).toBe(19); + expect(report.samples[0]).toEqual({ + phase: "event-apply", threadId: "thread-a", executionId: "execution-a", + count: 40, totalMs: 480, maxMs: 12, + }); + expect(report.samples[1]).toEqual({ + phase: "canonical-write", threadId: "thread-b", executionId: "execution-b", + count: 1, totalMs: 30, maxMs: 30, + }); + expect(report.samples[2]).toMatchObject({ threadId: null, executionId: null }); + expect(JSON.stringify(report)).not.toMatch(/prompt|toolInput|output|secret/); + + trace.tick(1_260); + expect(reports).toHaveLength(1); + }); +}); diff --git a/apps/server/src/features/agents/diagnostics/server-work-trace.ts b/apps/server/src/features/agents/diagnostics/server-work-trace.ts new file mode 100644 index 000000000..7ea48f471 --- /dev/null +++ b/apps/server/src/features/agents/diagnostics/server-work-trace.ts @@ -0,0 +1,128 @@ +import * as NodePerfHooks from "node:perf_hooks"; +import { logger } from "@mcode/shared"; + +/** Fixed names keep trace output free of event bodies and provider text. */ +export type ServerWorkPhase = + | "provider-callback" | "worker-admission" | "worker-wait" | "mailbox-wait" + | "canonical-write" | "event-apply" | "finalization" | "publication" + | "terminal-create" | "thread-create" | "agent-send"; + +interface WorkSample { + phase: ServerWorkPhase; + threadId: string | null; + executionId: string | null; + count: number; + totalMs: number; + maxMs: number; +} + +/** One bounded report for work accumulated while the server loop could not run. */ +export type ServerWorkTraceReport = { + kind: "server-work-stall"; + at: number; + delayMs: number; + windowMs: number; + samples: WorkSample[]; + overflowCount: number; +} | { + kind: "server-work-slow-operation"; + at: number; + sample: WorkSample; +}; + +const MAX_KEYS = 64; +const INTERVAL_MS = 20; +const REPORT_DELAY_MS = 100; +const TRACE_ID = /^[A-Za-z0-9_-]{1,128}$/; + +function safeId(value: string | undefined): string | null { + return value && TRACE_ID.test(value) ? value : null; +} + +/** Aggregates fixed-phase timings in memory and emits only delayed loop windows. */ +export class ServerWorkTrace { + private readonly samples = new Map(); + private overflowCount = 0; + private lastTick = NodePerfHooks.performance.now(); + private timer: ReturnType | undefined; + + constructor(private readonly emit: (report: ServerWorkTraceReport) => void) {} + + /** Begin the lightweight event-loop probe. */ + start(): void { + if (this.timer) return; + this.lastTick = NodePerfHooks.performance.now(); + this.timer = setInterval(() => this.tick(), INTERVAL_MS); + this.timer.unref(); + } + + /** Stop the probe and discard its pending window. */ + stop(): void { + if (this.timer) clearInterval(this.timer); + this.timer = undefined; + this.samples.clear(); + this.overflowCount = 0; + } + + /** Measure synchronous work without retaining its input or result. */ + measure(phase: ServerWorkPhase, threadId: string | undefined, executionId: string | undefined, work: () => T): T { + const started = NodePerfHooks.performance.now(); + try { + return work(); + } finally { + this.record(phase, threadId, executionId, NodePerfHooks.performance.now() - started); + } + } + + /** Record an asynchronous wait or a measured operation after completion. */ + record(phase: ServerWorkPhase, threadId: string | undefined, executionId: string | undefined, durationMs: number): void { + const safeThreadId = safeId(threadId); + const safeExecutionId = safeId(executionId); + if (this.isSlowOperation(phase, durationMs)) { + this.emit({ kind: "server-work-slow-operation", at: Date.now(), sample: { + phase, threadId: safeThreadId, executionId: safeExecutionId, + count: 1, totalMs: durationMs, maxMs: durationMs, + } }); + } + const key = JSON.stringify([phase, safeThreadId, safeExecutionId]); + const sample = this.samples.get(key); + if (sample) { + sample.count += 1; + sample.totalMs += durationMs; + sample.maxMs = Math.max(sample.maxMs, durationMs); + return; + } + if (this.samples.size === MAX_KEYS) { + this.overflowCount += 1; + return; + } + this.samples.set(key, { + phase, threadId: safeThreadId, executionId: safeExecutionId, + count: 1, totalMs: durationMs, maxMs: durationMs, + }); + } + + private isSlowOperation(phase: ServerWorkPhase, durationMs: number): boolean { + return durationMs >= REPORT_DELAY_MS + && (phase === "terminal-create" || phase === "thread-create" || phase === "agent-send"); + } + + /** Check a single probe interval; exposed to drive deterministic stall tests. */ + tick(now: number = NodePerfHooks.performance.now()): void { + const windowMs = now - this.lastTick; + this.lastTick = now; + const delayMs = Math.max(0, windowMs - INTERVAL_MS); + if (delayMs >= REPORT_DELAY_MS) { + this.emit({ kind: "server-work-stall", at: Date.now(), delayMs, windowMs, samples: [...this.samples.values()], overflowCount: this.overflowCount }); + } + this.samples.clear(); + this.overflowCount = 0; + } +} + +/** Null unless explicitly enabled before server startup. */ +export const serverWorkTrace = process.env.MCODE_SERVER_WORK_TRACE === "1" + ? new ServerWorkTrace((report) => logger.warn("Server work trace", report)) + : null; + +serverWorkTrace?.start(); diff --git a/apps/server/src/features/agents/orchestration/agent-event-publication-service.ts b/apps/server/src/features/agents/orchestration/agent-event-publication-service.ts index 65d5e9ef0..ed3444991 100644 --- a/apps/server/src/features/agents/orchestration/agent-event-publication-service.ts +++ b/apps/server/src/features/agents/orchestration/agent-event-publication-service.ts @@ -5,6 +5,7 @@ import type { NarrativeStore } from "../conversation/narrative/narrative-store.j import { publishParentProviderEvent } from "../events/provider-event-publication.js"; import { publishAgentPermissionEvents } from "../permissions/permission-publication.js"; import { sanitizePublicToolInput } from "../tools/input/public-tool-input.js"; +import { serverWorkTrace } from "../diagnostics/server-work-trace.js"; /** The runtime read surface needed to prepare renderer-facing provider events. */ export interface AgentEventPublicationRuntime { @@ -42,6 +43,14 @@ export class AgentEventPublicationService { /** Publish one provider event after its synchronous persistence application completed. */ publish(event: AgentEvent): void { + if (serverWorkTrace) { + serverWorkTrace.measure("publication", event.threadId, event.turnExecutionId, () => this.publishCore(event)); + return; + } + this.publishCore(event); + } + + private publishCore(event: AgentEvent): void { if (this.shouldSuppress(event)) return; const enriched = this.sanitize(this.enrich(event)); const published = publishParentProviderEvent(event, enriched, { diff --git a/apps/server/src/features/agents/transport/agent-rpc.ts b/apps/server/src/features/agents/transport/agent-rpc.ts index f67920928..f3b7719b4 100644 --- a/apps/server/src/features/agents/transport/agent-rpc.ts +++ b/apps/server/src/features/agents/transport/agent-rpc.ts @@ -6,6 +6,7 @@ import type { } from "@mcode/contracts"; import { WS_METHODS } from "@mcode/contracts"; import { logger } from "@mcode/shared"; +import * as NodePerfHooks from "node:perf_hooks"; import type { z } from "zod"; import type { GitWatcherService } from "../../projects/git/git-watcher-service.js"; import type { ThreadControlService } from "../../thread-control/authority/thread-control-service.js"; @@ -21,6 +22,7 @@ import type { ThoughtSegmentRepo } from "../conversation/narrative/persistence/t import type { NarrativeStore } from "../conversation/narrative/narrative-store.js"; import type { HookExecutionRepo } from "../events/persistence/hook-execution-repo.js"; import type { AgentService } from "../orchestration/agent-service.js"; +import { serverWorkTrace } from "../diagnostics/server-work-trace.js"; import type { AgentTurnContinuationPort } from "../orchestration/agent-runtime-internal-ports.js"; import type { TaskRepo } from "../orchestration/persistence/task-repo.js"; import type { AgentPermissionService } from "../permissions/agent-permission-service.js"; @@ -107,11 +109,16 @@ const PREVIEW_ANNOTATION_FENCE_END = "mcode-preview-annotations:end -->"; const agentHandlers: AgentRpcHandlerMap = { "agent.send": async (deps, params) => { - await deps.agentService.sendMessage({ - ...params, - content: appendPreviewAnnotations(params.content, params.previewAnnotations), - displayContent: params.displayContent ?? params.content, - }); + const started = serverWorkTrace ? NodePerfHooks.performance.now() : 0; + try { + await deps.agentService.sendMessage({ + ...params, + content: appendPreviewAnnotations(params.content, params.previewAnnotations), + displayContent: params.displayContent ?? params.content, + }); + } finally { + if (serverWorkTrace) serverWorkTrace.record("agent-send", params.threadId, undefined, NodePerfHooks.performance.now() - started); + } }, "agent.recoveryIncident": (deps) => deps.turnRecoveryService.currentRecoveryIncident(), "agent.retry": async (deps, params) => { diff --git a/apps/server/src/features/agents/turns/turn-event-pipeline.ts b/apps/server/src/features/agents/turns/turn-event-pipeline.ts index 5df79c552..07c874fa2 100644 --- a/apps/server/src/features/agents/turns/turn-event-pipeline.ts +++ b/apps/server/src/features/agents/turns/turn-event-pipeline.ts @@ -1,5 +1,7 @@ import type { AgentEvent, ProviderFileMutationStart, ProviderTurnDiffUpdate } from "@mcode/contracts"; import type { TurnDiffService } from "./turn-diff-service.js"; +import * as NodePerfHooks from "node:perf_hooks"; +import { serverWorkTrace } from "../diagnostics/server-work-trace.js"; import type { ProviderEventIngressConsumer, @@ -223,7 +225,13 @@ export class TurnEventPipeline implements ProviderEventIngressConsumer { } private startFinalization(command: FinalizeTurnCommand): Promise { - return this.lifecycle.finalize(command) ?? Promise.resolve(false); + const trace = serverWorkTrace; + if (!trace) return this.lifecycle.finalize(command) ?? Promise.resolve(false); + const started = NodePerfHooks.performance.now(); + const result = this.lifecycle.finalize(command) ?? Promise.resolve(false); + return result.finally(() => trace.record( + "finalization", command.threadId, command.executionId, NodePerfHooks.performance.now() - started, + )); } private decrementQueuedBytes(threadId: string, byteLength: number): void { diff --git a/apps/server/src/features/projects/lifecycle/transport/workspace-thread-rpc.ts b/apps/server/src/features/projects/lifecycle/transport/workspace-thread-rpc.ts index 227714c07..7505b7557 100644 --- a/apps/server/src/features/projects/lifecycle/transport/workspace-thread-rpc.ts +++ b/apps/server/src/features/projects/lifecycle/transport/workspace-thread-rpc.ts @@ -1,7 +1,9 @@ import { WS_METHODS, type Thread, type Workspace, type WsMethodName } from "@mcode/contracts"; import { logger } from "@mcode/shared"; +import * as NodePerfHooks from "node:perf_hooks"; import type { z } from "zod"; import type { GoalLifecycleService } from "../../../agents/goals/goal-lifecycle-service.js"; +import { serverWorkTrace } from "../../../agents/diagnostics/server-work-trace.js"; import { broadcast } from "../../../../application/transport/push.js"; import type { GithubService } from "../../../pull-requests/github/github-service.js"; import type { CiWatcherService } from "../../../pull-requests/status/ci-watcher.js"; @@ -224,15 +226,22 @@ async function createThread( deps: WorkspaceThreadRouterDeps, params: { workspaceId: string; title: string; mode: string; branch: string }, ): Promise { - const thread = await deps.threadService.create( - params.workspaceId, - params.title, - params.mode, - params.branch, - { branchless: params.mode === "worktree" }, - ); - watchReturnedThreadWorktree(deps, thread); - return thread; + const started = serverWorkTrace ? NodePerfHooks.performance.now() : 0; + let threadId: string | undefined; + try { + const thread = await deps.threadService.create( + params.workspaceId, + params.title, + params.mode, + params.branch, + { branchless: params.mode === "worktree" }, + ); + threadId = thread.id; + watchReturnedThreadWorktree(deps, thread); + return thread; + } finally { + if (serverWorkTrace) serverWorkTrace.record("thread-create", threadId, undefined, NodePerfHooks.performance.now() - started); + } } function watchReturnedThreadWorktree( diff --git a/apps/server/src/features/providers/composition/provider-event-ingress.ts b/apps/server/src/features/providers/composition/provider-event-ingress.ts index 4261ef24e..da7a02247 100644 --- a/apps/server/src/features/providers/composition/provider-event-ingress.ts +++ b/apps/server/src/features/providers/composition/provider-event-ingress.ts @@ -1,4 +1,5 @@ import { logger } from "@mcode/shared"; +import * as NodePerfHooks from "node:perf_hooks"; import { AgentEventType, isTurnDiffSource, @@ -30,6 +31,7 @@ import { type ProviderEventWorkerTask, } from "./provider-event-worker-protocol.js"; import { normalizeProviderError } from "./provider-error-normalize.js"; +import { serverWorkTrace } from "../../agents/diagnostics/server-work-trace.js"; const MAX_PENDING_NON_TERMINAL_EVENTS = 8_192; const MAX_PENDING_NON_TERMINAL_EVENTS_PER_THREAD = 2_048; @@ -128,6 +130,7 @@ export interface ProviderEventIngressQueueMetrics { interface QueuedProviderEvent { event: ProviderEventIngressEvent; queuedAt: number; + traceQueuedAt: number; byteLength: number; terminal: boolean; } @@ -224,6 +227,15 @@ export class ProviderEventIngress { /** Accept one provider-originated runtime event that has no canonical receipt. */ acceptProviderRuntime(providerId: ProviderId, runtimeEvent: unknown): void { + if (serverWorkTrace) { + const threadId = providerEventWorkerThreadId({ kind: "provider-runtime", providerId, runtimeEvent }); + serverWorkTrace.measure("provider-callback", threadId, undefined, () => this.acceptProviderRuntimeCore(providerId, runtimeEvent)); + return; + } + this.acceptProviderRuntimeCore(providerId, runtimeEvent); + } + + private acceptProviderRuntimeCore(providerId: ProviderId, runtimeEvent: unknown): void { const parsedProviderId = ProviderIdSchema.safeParse(providerId); if (!parsedProviderId.success) { this.report({ reason: "provider-identity-mismatch", sourceKind: "provider-runtime", providerId: String(providerId) }); @@ -299,7 +311,19 @@ export class ProviderEventIngress { this.report({ reason: "worker-shutdown", sourceKind: task.kind, eventId: canonicalEventIdentity(task) }); return false; } - const accepted = this.workerPool.submit(providerEventWorkerThreadId(task), task, { onOutcome }); + const threadId = providerEventWorkerThreadId(task); + let accepted: boolean; + const trace = serverWorkTrace; + if (trace) { + const started = NodePerfHooks.performance.now(); + accepted = this.workerPool.submit(threadId, task, { onOutcome: (outcome) => { + trace.record("worker-wait", threadId, + outcome.status === "accepted" ? outcome.event.event.turnExecutionId : undefined, + NodePerfHooks.performance.now() - started); + onOutcome(outcome); + } }); + trace.record("worker-admission", threadId, undefined, NodePerfHooks.performance.now() - started); + } else accepted = this.workerPool.submit(threadId, task, { onOutcome }); if (!accepted) this.rejectForWorkerCapacity(task); return accepted; } @@ -392,7 +416,7 @@ export class ProviderEventIngress { this.pendingByThread.set(threadId, queued); this.readyThreadIds.push(threadId); } - queued.push({ event, queuedAt: Date.now(), byteLength, terminal }); + queued.push({ event, queuedAt: Date.now(), traceQueuedAt: serverWorkTrace ? NodePerfHooks.performance.now() : 0, byteLength, terminal }); this.pendingEventCount += 1; this.pendingByteCount += byteLength; this.pendingBytesByThread.set(threadId, (this.pendingBytesByThread.get(threadId) ?? 0) + byteLength); @@ -431,7 +455,12 @@ export class ProviderEventIngress { for (let delivered = 0; delivered < MAX_PROVIDER_EVENTS_PER_DRAIN; delivered += 1) { const queued = this.takeNextPendingEvent(); if (!queued) break; - this.consumer.handleProviderEvent(queued.event); + if (serverWorkTrace) { + serverWorkTrace.record("mailbox-wait", queued.event.event.threadId, + queued.event.event.turnExecutionId, NodePerfHooks.performance.now() - queued.traceQueuedAt); + serverWorkTrace.measure("event-apply", queued.event.event.threadId, + queued.event.event.turnExecutionId, () => this.consumer?.handleProviderEvent(queued.event)); + } else this.consumer.handleProviderEvent(queued.event); } if (this.pendingEventCount > 0) this.scheduleYieldedDrain(); } diff --git a/apps/server/src/features/terminal/transport/terminal-rpc.ts b/apps/server/src/features/terminal/transport/terminal-rpc.ts index 55767acd3..baffcef29 100644 --- a/apps/server/src/features/terminal/transport/terminal-rpc.ts +++ b/apps/server/src/features/terminal/transport/terminal-rpc.ts @@ -8,6 +8,8 @@ import { TerminalBackendError, type TerminalBackend } from "../backends/terminal import type { TerminalProfileService } from "../profiles/terminal-profile-service.js"; import type { WorkspaceTerminalPreferencesService } from "../preferences/workspace-terminal-preferences-service.js"; import type { SettingsService } from "../../settings/settings-service.js"; +import * as NodePerfHooks from "node:perf_hooks"; +import { serverWorkTrace } from "../../agents/diagnostics/server-work-trace.js"; type TerminalManagementMethod = | "terminal.profile.list" @@ -111,7 +113,15 @@ const terminalManagementHandlers: TerminalManagementHandlers = { const terminalClassicHandlers: TerminalClassicHandlers = { "terminal.capabilities": (deps) => deps.terminalService.capabilities(), - "terminal.create": (deps, params) => deps.terminalService.create(params.threadId), + "terminal.create": async (deps, params) => { + if (!serverWorkTrace) return deps.terminalService.create(params.threadId); + const started = NodePerfHooks.performance.now(); + try { + return await deps.terminalService.create(params.threadId); + } finally { + serverWorkTrace.record("terminal-create", params.threadId, undefined, NodePerfHooks.performance.now() - started); + } + }, "terminal.write": async (deps, params) => { await deps.terminalService.write(params.ptyId, params.data); }, From 55cd8aee6b966db62b6af574ea6c5da0a12e95fc Mon Sep 17 00:00:00 2001 From: "(CJ) Chukwudi Nwobodo" <142016413+chuks-qua@users.noreply.github.com> Date: Thu, 24 Sep 2026 19:29:59 +0100 Subject: [PATCH 002/136] feat(server): add execution-scoped worker mailbox scheduler --- .../execution-mailbox-scheduler.test.ts | 277 ++++++++++++ .../execution/execution-mailbox-protocol.ts | 46 ++ .../execution/execution-mailbox-scheduler.ts | 404 ++++++++++++++++++ 3 files changed, 727 insertions(+) create mode 100644 apps/server/src/features/agents/execution/__tests__/execution-mailbox-scheduler.test.ts create mode 100644 apps/server/src/features/agents/execution/execution-mailbox-protocol.ts create mode 100644 apps/server/src/features/agents/execution/execution-mailbox-scheduler.ts diff --git a/apps/server/src/features/agents/execution/__tests__/execution-mailbox-scheduler.test.ts b/apps/server/src/features/agents/execution/__tests__/execution-mailbox-scheduler.test.ts new file mode 100644 index 000000000..f952d9190 --- /dev/null +++ b/apps/server/src/features/agents/execution/__tests__/execution-mailbox-scheduler.test.ts @@ -0,0 +1,277 @@ +import { describe, expect, it } from "vitest"; + +import { + ExecutionMailboxScheduler, + type ExecutionMailboxLimits, +} from "../execution-mailbox-scheduler.js"; +import type { + ExecutionIdentity, + ExecutionWorkerPort, + ExecutionWorkerReply, + ExecutionWorkerRequest, +} from "../execution-mailbox-protocol.js"; + +type Work = { readonly kind: "start" } | { readonly kind: "event"; readonly sequence: number }; +type Command = Work | { readonly kind: "stop"; readonly requestId: string }; +type Result = { readonly revision: number }; + +const LIMITS: ExecutionMailboxLimits = { + maxPending: 8, + maxPendingBytes: 800, + reservedStop: 2, + reservedStopBytes: 200, + maxPerExecutionPending: 4, + maxPerExecutionBytes: 400, + reservedPerExecutionStop: 1, + reservedPerExecutionStopBytes: 100, +}; + +class FakeWorker implements ExecutionWorkerPort { + onmessage: ((event: MessageEvent>) => void) | null = null; + onerror: ((event: ErrorEvent) => void) | null = null; + readonly requests: ExecutionWorkerRequest[] = []; + terminated = false; + + postMessage(request: ExecutionWorkerRequest): void { + this.requests.push(request); + } + + terminate(): void { + this.terminated = true; + } + + reply(request: ExecutionWorkerRequest, revision: number): void { + this.onmessage?.(new MessageEvent("message", { + data: { ...request, result: { revision } }, + })); + } + + crash(): void { + this.onerror?.(new ErrorEvent("error")); + } +} + +function execution(threadId: string, executionId = `execution-${threadId}`): ExecutionIdentity { + return { threadId, turnId: `turn-${threadId}`, executionId }; +} + +function fixture(workerCount = 1, limits = LIMITS) { + const workers: FakeWorker[] = []; + const lost: ExecutionIdentity[][] = []; + const scheduler = new ExecutionMailboxScheduler({ + workerCount, + limits, + createWorker: () => { + const worker = new FakeWorker(); + workers.push(worker); + return worker; + }, + onWorkerLost: (affected) => lost.push([...affected]), + }); + return { scheduler, workers, lost }; +} + +function claimed(scheduler: ExecutionMailboxScheduler, identity: ExecutionIdentity, epoch = 1) { + const claim = scheduler.claim(identity, epoch); + if (claim.kind !== "claimed") throw new Error(`Claim failed: ${claim.kind}`); + return claim.lease; +} + +function admitted( + scheduler: ExecutionMailboxScheduler, + identity: ExecutionIdentity, + lease: ReturnType, + command: Command, + byteLength = 100, +) { + const admission = scheduler.submit({ execution: identity, lease, command, byteLength }); + if (admission.kind !== "admitted") throw new Error(`Admission failed: ${admission.kind}`); + return admission; +} + +function request(worker: FakeWorker, index: number): ExecutionWorkerRequest { + const next = worker.requests[index]; + if (!next) throw new Error(`Missing worker request ${index}`); + return next; +} + +describe("ExecutionMailboxScheduler", () => { + it("keeps an execution on one worker and processes each mailbox in order", async () => { + const { scheduler, workers } = fixture(2); + const first = execution("first"); + const second = execution("second"); + const third = execution("third"); + const firstLease = claimed(scheduler, first); + const secondLease = claimed(scheduler, second); + const thirdLease = claimed(scheduler, third); + expect(firstLease.workerIndex).toBe(0); + expect(secondLease.workerIndex).toBe(1); + expect(thirdLease.workerIndex).toBe(0); + + const firstStart = admitted(scheduler, first, firstLease, { kind: "start" }); + const firstEvent = admitted(scheduler, first, firstLease, { kind: "event", sequence: 1 }); + const thirdStart = admitted(scheduler, third, thirdLease, { kind: "start" }); + const secondStart = admitted(scheduler, second, secondLease, { kind: "start" }); + expect(workers[0]?.requests).toHaveLength(1); + expect(workers[1]?.requests).toHaveLength(1); + + workers[0]?.reply(request(workers[0], 0), 1); + expect(request(workers[0]!, 1).execution).toEqual(first); + workers[0]?.reply(request(workers[0]!, 1), 2); + expect(request(workers[0]!, 2).execution).toEqual(third); + workers[0]?.reply(request(workers[0]!, 2), 1); + workers[1]?.reply(request(workers[1]!, 0), 1); + + expect([firstStart.ordinal, firstEvent.ordinal, thirdStart.ordinal]).toEqual([1, 2, 1]); + expect(await firstEvent.completion).toEqual({ kind: "reply", result: { revision: 2 } }); + expect(await thirdStart.completion).toEqual({ kind: "reply", result: { revision: 1 } }); + expect(await secondStart.completion).toEqual({ kind: "reply", result: { revision: 1 } }); + expect(scheduler.release(first, firstLease)).toBe(true); + expect(scheduler.depth().pending).toBe(0); + scheduler.shutdown(); + }); + + it("reserves count and byte credits for Stop, including within one execution", async () => { + const { scheduler, workers } = fixture(); + const identity = execution("busy"); + const lease = claimed(scheduler, identity); + const first = admitted(scheduler, identity, lease, { kind: "event", sequence: 1 }); + const second = admitted(scheduler, identity, lease, { kind: "event", sequence: 2 }); + const third = admitted(scheduler, identity, lease, { kind: "event", sequence: 3 }); + expect(scheduler.submit({ execution: identity, lease, command: { kind: "event", sequence: 4 }, byteLength: 100 })) + .toEqual({ kind: "overloaded" }); + const stop = admitted(scheduler, identity, lease, { kind: "stop", requestId: "stop-1" }); + expect(scheduler.depth()).toMatchObject({ pending: 4, pendingBytes: 400 }); + expect(scheduler.submit({ execution: identity, lease, command: { kind: "stop", requestId: "stop-2" }, byteLength: 100 })) + .toEqual({ kind: "stop-pending" }); + + for (let index = 0; index < 4; index += 1) workers[0]?.reply(request(workers[0]!, index), index + 1); + expect((await stop.completion).kind).toBe("reply"); + expect((await first.completion).kind).toBe("reply"); + expect((await second.completion).kind).toBe("reply"); + expect((await third.completion).kind).toBe("reply"); + expect(workers[0]?.requests.map((item) => item.command.kind)).toEqual(["event", "event", "event", "stop"]); + scheduler.shutdown(); + }); + + it("keeps a global byte reserve when several executions fill normal capacity", async () => { + const limits: ExecutionMailboxLimits = { + ...LIMITS, + maxPending: 10, + maxPendingBytes: 500, + reservedStop: 1, + reservedStopBytes: 100, + maxPerExecutionBytes: 500, + }; + const { scheduler } = fixture(1, limits); + const first = execution("first"); + const second = execution("second"); + const third = execution("third"); + const firstLease = claimed(scheduler, first); + const secondLease = claimed(scheduler, second); + const thirdLease = claimed(scheduler, third); + admitted(scheduler, first, firstLease, { kind: "event", sequence: 1 }, 200); + admitted(scheduler, second, secondLease, { kind: "event", sequence: 1 }, 200); + expect(scheduler.submit({ execution: third, lease: thirdLease, command: { kind: "start" }, byteLength: 100 })) + .toEqual({ kind: "overloaded" }); + admitted(scheduler, first, firstLease, { kind: "stop", requestId: "reserved" }, 100); + expect(scheduler.depth()).toMatchObject({ pending: 3, pendingBytes: 500 }); + expect(scheduler.submit({ execution: second, lease: secondLease, command: { kind: "stop", requestId: "full" }, byteLength: 100 })) + .toEqual({ kind: "overloaded" }); + scheduler.shutdown(); + }); + + it("lets another execution progress while Stop remains behind its own earlier events", async () => { + const { scheduler, workers } = fixture(); + const chatty = execution("chatty"); + const quiet = execution("quiet"); + const chattyLease = claimed(scheduler, chatty); + const quietLease = claimed(scheduler, quiet); + admitted(scheduler, chatty, chattyLease, { kind: "event", sequence: 1 }); + admitted(scheduler, chatty, chattyLease, { kind: "event", sequence: 2 }); + const stop = admitted(scheduler, chatty, chattyLease, { kind: "stop", requestId: "stop-chatty" }); + const quietEvent = admitted(scheduler, quiet, quietLease, { kind: "event", sequence: 1 }); + for (let index = 0; index < 4; index += 1) workers[0]?.reply(request(workers[0]!, index), index + 1); + expect(workers[0]?.requests.map((item) => `${item.execution.threadId}:${item.command.kind}`)) + .toEqual(["chatty:event", "chatty:event", "quiet:event", "chatty:stop"]); + expect((await quietEvent.completion).kind).toBe("reply"); + expect((await stop.completion).kind).toBe("reply"); + scheduler.shutdown(); + }); + + it("revokes a crashed worker without replay and fences late replies from its generation", async () => { + const { scheduler, workers, lost } = fixture(); + const oldExecution = execution("same-thread", "old-execution"); + const oldLease = claimed(scheduler, oldExecution); + const first = admitted(scheduler, oldExecution, oldLease, { kind: "event", sequence: 1 }); + const queued = admitted(scheduler, oldExecution, oldLease, { kind: "event", sequence: 2 }); + const oldRequest = request(workers[0]!, 0); + workers[0]?.crash(); + expect(await first.completion).toEqual({ kind: "worker-lost" }); + expect(await queued.completion).toEqual({ kind: "worker-lost" }); + expect(lost).toEqual([[oldExecution]]); + expect(workers[0]?.terminated).toBe(true); + expect(scheduler.claim(execution("unavailable"), 2)).toEqual({ kind: "worker-unavailable" }); + expect(scheduler.submit({ execution: oldExecution, lease: oldLease, command: { kind: "stop", requestId: "stale" }, byteLength: 100 })) + .toEqual({ kind: "stale-execution" }); + + expect(scheduler.replaceWorker(0)).toBe(true); + expect(workers[1]?.requests).toHaveLength(0); + + const newExecution = execution("same-thread", "new-execution"); + const newLease = claimed(scheduler, newExecution, 2); + expect(scheduler.submit({ execution: newExecution, lease: oldLease, command: { kind: "start" }, byteLength: 100 })) + .toEqual({ kind: "stale-execution" }); + const newEvent = admitted(scheduler, newExecution, newLease, { kind: "event", sequence: 1 }); + workers[0]?.reply(oldRequest, 99); + expect(scheduler.depth().pending).toBe(1); + workers[1]?.reply(request(workers[1]!, 0), 1); + expect(await newEvent.completion).toEqual({ kind: "reply", result: { revision: 1 } }); + expect(scheduler.release(oldExecution, oldLease)).toBe(false); + scheduler.shutdown(); + }); + + it("revokes a worker whose reply claims a different execution", async () => { + const { scheduler, workers, lost } = fixture(); + const identity = execution("target"); + const lease = claimed(scheduler, identity); + const admission = admitted(scheduler, identity, lease, { kind: "start" }); + const sent = request(workers[0]!, 0); + workers[0]?.onmessage?.(new MessageEvent("message", { + data: { ...sent, execution: execution("other"), result: { revision: 1 } }, + })); + expect(await admission.completion).toEqual({ kind: "worker-lost" }); + expect(lost).toEqual([[identity]]); + scheduler.shutdown(); + }); + + it("keeps a different worker usable after one worker crashes", async () => { + const { scheduler, workers, lost } = fixture(2); + const failed = execution("failed"); + const healthy = execution("healthy"); + const failedLease = claimed(scheduler, failed); + const healthyLease = claimed(scheduler, healthy); + const failedEvent = admitted(scheduler, failed, failedLease, { kind: "event", sequence: 1 }); + const healthyEvent = admitted(scheduler, healthy, healthyLease, { kind: "event", sequence: 1 }); + workers[0]?.crash(); + workers[1]?.reply(request(workers[1]!, 0), 1); + expect(await failedEvent.completion).toEqual({ kind: "worker-lost" }); + expect(await healthyEvent.completion).toEqual({ kind: "reply", result: { revision: 1 } }); + expect(lost).toEqual([[failed]]); + const another = execution("another"); + expect(claimed(scheduler, another).workerIndex).toBe(1); + scheduler.shutdown(); + }); + + it("rejects a second active execution on the same thread and settles shutdown", async () => { + const { scheduler } = fixture(); + const identity = execution("one", "first"); + const lease = claimed(scheduler, identity); + expect(scheduler.claim(execution("one", "second"), 2)).toEqual({ kind: "thread-busy" }); + const admission = admitted(scheduler, identity, lease, { kind: "start" }); + expect(scheduler.release(identity, lease)).toBe(false); + scheduler.shutdown(); + expect(await admission.completion).toEqual({ kind: "shutdown" }); + expect(scheduler.depth()).toMatchObject({ pending: 0, pendingBytes: 0, activeExecutions: 0 }); + }); +}); diff --git a/apps/server/src/features/agents/execution/execution-mailbox-protocol.ts b/apps/server/src/features/agents/execution/execution-mailbox-protocol.ts new file mode 100644 index 000000000..11f6de596 --- /dev/null +++ b/apps/server/src/features/agents/execution/execution-mailbox-protocol.ts @@ -0,0 +1,46 @@ +/** The exact turn attempt that owns every command and reply in a mailbox. */ +export interface ExecutionIdentity { + readonly threadId: string; + readonly turnId: string; + readonly executionId: string; +} + +/** A durable owner epoch plus the local worker incarnation that may use it. */ +export interface ExecutionLease { + readonly ownerEpoch: number; + readonly workerIndex: number; + readonly workerGeneration: number; + readonly leaseId: string; +} + +/** One ordered, data-only command sent to the worker that owns an execution. */ +export interface ExecutionWorkerRequest { + readonly requestId: number; + readonly execution: ExecutionIdentity; + readonly lease: ExecutionLease; + readonly ordinal: number; + readonly command: Command; +} + +/** A reply must echo the complete identity and lease before it can be accepted. */ +export interface ExecutionWorkerReply { + readonly requestId: number; + readonly execution: ExecutionIdentity; + readonly lease: ExecutionLease; + readonly ordinal: number; + readonly result: Result; +} + +/** A worker reply is transport completion; the result defines durable success or failure. */ +export type ExecutionMailboxCompletion = + | { readonly kind: "reply"; readonly result: Result } + | { readonly kind: "worker-lost" } + | { readonly kind: "shutdown" }; + +/** The worker port used by the fixed execution mailbox scheduler. */ +export interface ExecutionWorkerPort { + onmessage: ((event: MessageEvent>) => void) | null; + onerror: ((event: ErrorEvent) => void) | null; + postMessage(request: ExecutionWorkerRequest): void; + terminate(): void; +} diff --git a/apps/server/src/features/agents/execution/execution-mailbox-scheduler.ts b/apps/server/src/features/agents/execution/execution-mailbox-scheduler.ts new file mode 100644 index 000000000..c48eaebc5 --- /dev/null +++ b/apps/server/src/features/agents/execution/execution-mailbox-scheduler.ts @@ -0,0 +1,404 @@ +import * as NodeCrypto from "node:crypto"; + +import type { + ExecutionIdentity, + ExecutionLease, + ExecutionMailboxCompletion, + ExecutionWorkerPort, + ExecutionWorkerReply, + ExecutionWorkerRequest, +} from "./execution-mailbox-protocol.js"; + +/** The caller supplies typed start, event, checkpoint, outcome, and finalization work. */ +export type ExecutionMailboxCommand = + | Work + | { readonly kind: "stop"; readonly requestId: string }; + +interface Pending { + readonly request: ExecutionWorkerRequest; + readonly byteLength: number; + readonly control: boolean; + readonly resolve: (completion: ExecutionMailboxCompletion) => void; +} + +interface Assignment { + readonly execution: ExecutionIdentity; + readonly lease: ExecutionLease; + readonly slot: Slot; + nextOrdinal: number; + pendingCount: number; + pendingBytes: number; + normalCount: number; + normalBytes: number; + stopPending: boolean; +} + +interface Slot { + readonly index: number; + generation: number; + worker: ExecutionWorkerPort | undefined; + inFlight: Pending | undefined; + readonly queues: Map[]>; + readonly readyThreads: string[]; + activeCount: number; +} + +/** Capacity includes posted work until its reply arrives. Stop has its own reserve. */ +export interface ExecutionMailboxLimits { + readonly maxPending: number; + readonly maxPendingBytes: number; + readonly reservedStop: number; + readonly reservedStopBytes: number; + readonly maxPerExecutionPending: number; + readonly maxPerExecutionBytes: number; + readonly reservedPerExecutionStop: number; + readonly reservedPerExecutionStopBytes: number; +} + +/** The host supplies durable owner epochs and a worker factory for each fixed slot. */ +export interface ExecutionMailboxOptions { + readonly workerCount: number; + readonly limits: ExecutionMailboxLimits; + readonly createWorker: (workerIndex: number) => ExecutionWorkerPort; + readonly onWorkerLost: (executions: readonly ExecutionIdentity[]) => void; +} + +export type ExecutionClaim = + | { readonly kind: "claimed"; readonly lease: ExecutionLease } + | { readonly kind: "thread-busy" | "worker-unavailable" | "shutdown" }; + +export type ExecutionAdmission = + | { readonly kind: "admitted"; readonly ordinal: number; readonly completion: Promise> } + | { readonly kind: "stale-execution" | "overloaded" | "invalid-size" | "stop-pending" | "shutdown" }; + +/** Bounded state visible to diagnostics without exposing command payloads. */ +export interface ExecutionMailboxDepth { + readonly pending: number; + readonly pendingBytes: number; + readonly activeExecutions: number; + readonly workers: readonly { readonly index: number; readonly queued: number; readonly inFlight: boolean }[]; +} + +/** + * Holds an execution on one fixed worker for its lifetime. This scheduler only + * admits and transports commands; a worker reply carries the writer's durable + * receipt and must be interpreted by the caller before publishing success. + */ +export class ExecutionMailboxScheduler { + private readonly slots: Slot, Result>[]; + private readonly byThread = new Map, Result>>(); + private readonly limits: ExecutionMailboxLimits; + private readonly createWorker: ExecutionMailboxOptions, Result>["createWorker"]; + private readonly onWorkerLost: ExecutionMailboxOptions, Result>["onWorkerLost"]; + private pendingCount = 0; + private pendingBytes = 0; + private normalCount = 0; + private normalBytes = 0; + private nextRequestId = 1; + private nextSlotIndex = 0; + private stopped = false; + + constructor(options: ExecutionMailboxOptions, Result>) { + validateOptions(options); + this.limits = options.limits; + this.createWorker = options.createWorker; + this.onWorkerLost = options.onWorkerLost; + this.slots = Array.from({ length: options.workerCount }, (_, index) => ({ + index, + generation: 0, + worker: undefined, + inFlight: undefined, + queues: new Map(), + readyThreads: [], + activeCount: 0, + })); + try { + for (const slot of this.slots) this.startWorker(slot); + } catch (error) { + for (const slot of this.slots) slot.worker?.terminate(); + throw error; + } + } + + /** Bind one turn attempt and its durable owner epoch before submitting start. */ + claim(execution: ExecutionIdentity, ownerEpoch: number): ExecutionClaim { + if (this.stopped) return { kind: "shutdown" }; + if (!validIdentity(execution) || !positiveInteger(ownerEpoch)) throw new Error("Invalid execution identity or owner epoch"); + if (this.byThread.has(execution.threadId)) return { kind: "thread-busy" }; + const slot = this.chooseSlot(); + if (!slot) return { kind: "worker-unavailable" }; + const lease: ExecutionLease = { + ownerEpoch, + workerIndex: slot.index, + workerGeneration: slot.generation, + leaseId: NodeCrypto.randomUUID(), + }; + this.byThread.set(execution.threadId, { + execution: { ...execution }, + lease, + slot, + nextOrdinal: 1, + pendingCount: 0, + pendingBytes: 0, + normalCount: 0, + normalBytes: 0, + stopPending: false, + }); + slot.activeCount += 1; + return { kind: "claimed", lease }; + } + + /** Admit in FIFO order. byteLength must cover the entire cloneable request. */ + submit(input: { + readonly execution: ExecutionIdentity; + readonly lease: ExecutionLease; + readonly command: ExecutionMailboxCommand; + readonly byteLength: number; + }): ExecutionAdmission { + if (this.stopped) return { kind: "shutdown" }; + const assignment = this.currentAssignment(input.execution, input.lease); + if (!assignment) return { kind: "stale-execution" }; + if (!positiveInteger(input.byteLength)) return { kind: "invalid-size" }; + const control = input.command.kind === "stop"; + if (control && assignment.stopPending) return { kind: "stop-pending" }; + if (!this.hasCapacity(assignment, input.byteLength, control)) return { kind: "overloaded" }; + const ordinal = assignment.nextOrdinal++; + const request: ExecutionWorkerRequest> = { + requestId: this.nextRequestId++, + execution: assignment.execution, + lease: assignment.lease, + ordinal, + command: input.command, + }; + let resolveCompletion: (completion: ExecutionMailboxCompletion) => void = () => {}; + const completion = new Promise>((resolve) => { + resolveCompletion = resolve; + }); + const pending: Pending, Result> = { + request, + byteLength: input.byteLength, + control, + resolve: resolveCompletion, + }; + this.retain(assignment, pending); + this.enqueue(assignment.slot, pending); + this.dispatch(assignment.slot); + return { kind: "admitted", ordinal, completion }; + } + + /** Release only after the worker has acknowledged every command for the execution. */ + release(execution: ExecutionIdentity, lease: ExecutionLease): boolean { + const assignment = this.currentAssignment(execution, lease); + if (!assignment || assignment.pendingCount !== 0) return false; + this.byThread.delete(execution.threadId); + assignment.slot.activeCount -= 1; + return true; + } + + /** Replace a failed slot only after its revoked executions have been reconciled. */ + replaceWorker(index: number): boolean { + const slot = this.slots[index]; + if (this.stopped || !slot || slot.worker || slot.activeCount !== 0) return false; + this.startWorker(slot); + return true; + } + + /** Stop admission and settle retained work without replaying external effects. */ + shutdown(): void { + if (this.stopped) return; + this.stopped = true; + for (const slot of this.slots) { + slot.worker?.terminate(); + slot.worker = undefined; + this.settleSlot(slot, "shutdown"); + } + this.byThread.clear(); + } + + /** Return only counts, so telemetry cannot log provider or user content. */ + depth(): ExecutionMailboxDepth { + return { + pending: this.pendingCount, + pendingBytes: this.pendingBytes, + activeExecutions: this.byThread.size, + workers: this.slots.map((slot) => ({ + index: slot.index, + queued: [...slot.queues.values()].reduce((sum, queue) => sum + queue.length, 0), + inFlight: slot.inFlight !== undefined, + })), + }; + } + + private chooseSlot(): Slot, Result> | undefined { + const available = this.slots.filter((slot) => slot.worker !== undefined); + if (available.length === 0) return undefined; + const least = Math.min(...available.map((slot) => slot.activeCount)); + for (let offset = 0; offset < this.slots.length; offset += 1) { + const index = (this.nextSlotIndex + offset) % this.slots.length; + const slot = this.slots[index]; + if (slot?.worker && slot.activeCount === least) { + this.nextSlotIndex = (index + 1) % this.slots.length; + return slot; + } + } + return undefined; + } + + private currentAssignment(execution: ExecutionIdentity, lease: ExecutionLease): Assignment, Result> | undefined { + const assignment = this.byThread.get(execution.threadId); + if (!assignment || !sameIdentity(assignment.execution, execution) || !sameLease(assignment.lease, lease)) return undefined; + return assignment; + } + + private hasCapacity(assignment: Assignment, Result>, bytes: number, control: boolean): boolean { + const limits = this.limits; + if (this.pendingCount + 1 > limits.maxPending || this.pendingBytes + bytes > limits.maxPendingBytes) return false; + if (assignment.pendingCount + 1 > limits.maxPerExecutionPending + || assignment.pendingBytes + bytes > limits.maxPerExecutionBytes) return false; + if (control) return true; + return this.normalCount + 1 <= limits.maxPending - limits.reservedStop + && this.normalBytes + bytes <= limits.maxPendingBytes - limits.reservedStopBytes + && assignment.normalCount + 1 <= limits.maxPerExecutionPending - limits.reservedPerExecutionStop + && assignment.normalBytes + bytes <= limits.maxPerExecutionBytes - limits.reservedPerExecutionStopBytes; + } + + private retain(assignment: Assignment, Result>, pending: Pending, Result>): void { + this.pendingCount += 1; + this.pendingBytes += pending.byteLength; + assignment.pendingCount += 1; + assignment.pendingBytes += pending.byteLength; + if (pending.control) { + assignment.stopPending = true; + return; + } + this.normalCount += 1; + this.normalBytes += pending.byteLength; + assignment.normalCount += 1; + assignment.normalBytes += pending.byteLength; + } + + private settle(pending: Pending, Result>, completion: ExecutionMailboxCompletion): void { + const assignment = this.byThread.get(pending.request.execution.threadId); + this.pendingCount -= 1; + this.pendingBytes -= pending.byteLength; + if (assignment && sameLease(assignment.lease, pending.request.lease)) { + assignment.pendingCount -= 1; + assignment.pendingBytes -= pending.byteLength; + if (pending.control) assignment.stopPending = false; + else { + assignment.normalCount -= 1; + assignment.normalBytes -= pending.byteLength; + } + } + if (!pending.control) { + this.normalCount -= 1; + this.normalBytes -= pending.byteLength; + } + pending.resolve(completion); + } + + private enqueue(slot: Slot, Result>, pending: Pending, Result>): void { + const threadId = pending.request.execution.threadId; + const queue = slot.queues.get(threadId); + if (queue) queue.push(pending); + else { + slot.queues.set(threadId, [pending]); + slot.readyThreads.push(threadId); + } + } + + private dispatch(slot: Slot, Result>): void { + if (this.stopped || !slot.worker || slot.inFlight) return; + const threadId = slot.readyThreads.shift(); + if (!threadId) return; + const queue = slot.queues.get(threadId); + const next = queue?.shift(); + if (!queue || !next) throw new Error("Mailbox ready list lost its queue"); + if (queue.length > 0) slot.readyThreads.push(threadId); + else slot.queues.delete(threadId); + slot.inFlight = next; + try { + slot.worker.postMessage(next.request); + } catch { + this.workerLost(slot, slot.generation); + } + } + + private startWorker(slot: Slot, Result>): void { + if (this.stopped) return; + const generation = ++slot.generation; + const worker = this.createWorker(slot.index); + worker.onmessage = (event) => this.receive(slot, generation, event.data); + worker.onerror = () => this.workerLost(slot, generation); + slot.worker = worker; + } + + private receive(slot: Slot, Result>, generation: number, reply: ExecutionWorkerReply): void { + if (this.stopped || slot.generation !== generation) return; + const pending = slot.inFlight; + if (!pending || !matchesReply(pending.request, reply)) { + this.workerLost(slot, generation); + return; + } + slot.inFlight = undefined; + this.settle(pending, { kind: "reply", result: reply.result }); + this.dispatch(slot); + } + + private workerLost(slot: Slot, Result>, generation: number): void { + if (this.stopped || slot.generation !== generation) return; + slot.worker?.terminate(); + slot.worker = undefined; + slot.generation += 1; + const revoked = [...this.byThread.values()] + .filter((assignment) => assignment.slot === slot) + .map((assignment) => assignment.execution); + this.settleSlot(slot, "worker-lost"); + for (const execution of revoked) this.byThread.delete(execution.threadId); + slot.activeCount = 0; + this.onWorkerLost(revoked); + } + + private settleSlot(slot: Slot, Result>, kind: "worker-lost" | "shutdown"): void { + const pending = [slot.inFlight, ...slot.queues.values()].flat().filter((item) => item !== undefined); + slot.inFlight = undefined; + slot.queues.clear(); + slot.readyThreads.length = 0; + for (const item of pending) this.settle(item, { kind }); + } +} + +function sameIdentity(left: ExecutionIdentity, right: ExecutionIdentity): boolean { + return left.threadId === right.threadId && left.turnId === right.turnId && left.executionId === right.executionId; +} + +function sameLease(left: ExecutionLease, right: ExecutionLease): boolean { + return left.ownerEpoch === right.ownerEpoch && left.workerIndex === right.workerIndex + && left.workerGeneration === right.workerGeneration && left.leaseId === right.leaseId; +} + +function matchesReply(request: ExecutionWorkerRequest, reply: ExecutionWorkerReply): boolean { + return request.requestId === reply.requestId && request.ordinal === reply.ordinal + && sameIdentity(request.execution, reply.execution) && sameLease(request.lease, reply.lease); +} + +function validIdentity(execution: ExecutionIdentity): boolean { + return execution.threadId.length > 0 && execution.turnId.length > 0 && execution.executionId.length > 0; +} + +function positiveInteger(value: number): boolean { + return Number.isSafeInteger(value) && value > 0; +} + +function validateOptions(options: ExecutionMailboxOptions): void { + const limits = options.limits; + const values = [options.workerCount, limits.maxPending, limits.maxPendingBytes, limits.maxPerExecutionPending, + limits.maxPerExecutionBytes, limits.reservedStop, limits.reservedStopBytes, + limits.reservedPerExecutionStop, limits.reservedPerExecutionStopBytes]; + if (values.some((value) => !positiveInteger(value))) throw new Error("Mailbox limits must be positive safe integers"); + if (limits.reservedStop >= limits.maxPending || limits.reservedStopBytes >= limits.maxPendingBytes + || limits.reservedPerExecutionStop >= limits.maxPerExecutionPending + || limits.reservedPerExecutionStopBytes >= limits.maxPerExecutionBytes) { + throw new Error("Mailbox Stop reserves must leave normal capacity"); + } +} From 9e6b064d29f22f58e8f1af30eebfb84d991dc195 Mon Sep 17 00:00:00 2001 From: "(CJ) Chukwudi Nwobodo" <142016413+chuks-qua@users.noreply.github.com> Date: Thu, 24 Sep 2026 19:30:32 +0100 Subject: [PATCH 003/136] perf(server): attribute event apply trace by type --- .../__tests__/server-work-trace.test.ts | 37 +++++++++-- .../agents/diagnostics/server-work-trace.ts | 61 +++++++++++++++++-- .../turns/provider-turn-event-application.ts | 6 +- .../composition/provider-event-ingress.ts | 5 +- 4 files changed, 97 insertions(+), 12 deletions(-) diff --git a/apps/server/src/features/agents/diagnostics/__tests__/server-work-trace.test.ts b/apps/server/src/features/agents/diagnostics/__tests__/server-work-trace.test.ts index 741c6ec6a..a24018a35 100644 --- a/apps/server/src/features/agents/diagnostics/__tests__/server-work-trace.test.ts +++ b/apps/server/src/features/agents/diagnostics/__tests__/server-work-trace.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest"; -import { ServerWorkTrace, type ServerWorkTraceReport } from "../server-work-trace.js"; +import { eventApplyType, ServerWorkTrace, type ServerWorkTraceReport } from "../server-work-trace.js"; describe("ServerWorkTrace", () => { it("attributes aggregate work in a stalled tick and bounds content-free output", () => { @@ -7,13 +7,15 @@ describe("ServerWorkTrace", () => { const trace = new ServerWorkTrace((report) => reports.push(report)); trace.tick(1_000); for (let index = 0; index < 40; index += 1) { - trace.record("event-apply", "thread-a", "execution-a", 12); + trace.record("event-apply", "thread-a", "execution-a", 12, "textDelta"); } trace.record("canonical-write", "thread-b", "execution-b", 30); - trace.record("event-apply", "secret prompt with spaces", "secret output with spaces", 1); + trace.record("event-apply", "secret prompt with spaces", "secret output with spaces", 1, "other"); for (let index = 0; index < 80; index += 1) { trace.record("worker-wait", `thread-${index}`, "execution-c", 1); } + trace.record("event-apply", "thread-overflow", "execution-d", 9, "toolResult"); + trace.record("narrative-checkpoint", "thread-overflow", "execution-d", 7); trace.tick(1_240); expect(reports).toHaveLength(1); @@ -22,7 +24,11 @@ describe("ServerWorkTrace", () => { if (report?.kind !== "server-work-stall") return; expect(report.delayMs).toBe(220); expect(report.samples).toHaveLength(64); - expect(report.overflowCount).toBe(19); + expect(report.overflowCount).toBe(21); + expect(report.eventApplyByType.textDelta).toEqual({ count: 40, totalMs: 480, maxMs: 12 }); + expect(report.eventApplyByType.toolResult).toEqual({ count: 1, totalMs: 9, maxMs: 9 }); + expect(report.eventApplyByType.other).toEqual({ count: 1, totalMs: 1, maxMs: 1 }); + expect(report.narrativeCheckpoint).toEqual({ count: 1, totalMs: 7, maxMs: 7 }); expect(report.samples[0]).toEqual({ phase: "event-apply", threadId: "thread-a", executionId: "execution-a", count: 40, totalMs: 480, maxMs: 12, @@ -37,4 +43,27 @@ describe("ServerWorkTrace", () => { trace.tick(1_260); expect(reports).toHaveLength(1); }); + + it("classifies only the fixed event type across threads", () => { + const reports: ServerWorkTraceReport[] = []; + const trace = new ServerWorkTrace((report) => reports.push(report)); + trace.tick(1_000); + const types = ["textDelta", "toolUse", "toolResult", "assistantMessageBoundary", "unrecognized"]; + types.forEach((type, index) => { + trace.record("event-apply", `thread-${index}`, "execution-a", index + 1, eventApplyType(type)); + trace.record("narrative-checkpoint", `thread-${index}`, "execution-a", index + 2); + }); + trace.tick(1_200); + const report = reports[0]; + expect(report?.kind).toBe("server-work-stall"); + if (report?.kind !== "server-work-stall") return; + expect(report.eventApplyByType).toEqual({ + textDelta: { count: 1, totalMs: 1, maxMs: 1 }, + toolUse: { count: 1, totalMs: 2, maxMs: 2 }, + toolResult: { count: 1, totalMs: 3, maxMs: 3 }, + assistantMessageBoundary: { count: 1, totalMs: 4, maxMs: 4 }, + other: { count: 1, totalMs: 5, maxMs: 5 }, + }); + expect(report.narrativeCheckpoint).toEqual({ count: 5, totalMs: 20, maxMs: 6 }); + }); }); diff --git a/apps/server/src/features/agents/diagnostics/server-work-trace.ts b/apps/server/src/features/agents/diagnostics/server-work-trace.ts index 7ea48f471..6a353e8ee 100644 --- a/apps/server/src/features/agents/diagnostics/server-work-trace.ts +++ b/apps/server/src/features/agents/diagnostics/server-work-trace.ts @@ -4,9 +4,41 @@ import { logger } from "@mcode/shared"; /** Fixed names keep trace output free of event bodies and provider text. */ export type ServerWorkPhase = | "provider-callback" | "worker-admission" | "worker-wait" | "mailbox-wait" - | "canonical-write" | "event-apply" | "finalization" | "publication" + | "canonical-write" | "event-apply" | "narrative-checkpoint" | "finalization" | "publication" | "terminal-create" | "thread-create" | "agent-send"; +/** Fixed event categories; unknown and future event types stay in other. */ +export type EventApplyType = "textDelta" | "toolUse" | "toolResult" | "assistantMessageBoundary" | "other"; + +/** Classify only the event discriminant, never an event payload. */ +export function eventApplyType(type: string): EventApplyType { + switch (type) { + case "textDelta": + case "toolUse": + case "toolResult": + case "assistantMessageBoundary": + return type; + default: + return "other"; + } +} + +interface EventApplyTotals { + count: number; + totalMs: number; + maxMs: number; +} + +function emptyEventApplyTotals(): Record { + return { + textDelta: { count: 0, totalMs: 0, maxMs: 0 }, + toolUse: { count: 0, totalMs: 0, maxMs: 0 }, + toolResult: { count: 0, totalMs: 0, maxMs: 0 }, + assistantMessageBoundary: { count: 0, totalMs: 0, maxMs: 0 }, + other: { count: 0, totalMs: 0, maxMs: 0 }, + }; +} + interface WorkSample { phase: ServerWorkPhase; threadId: string | null; @@ -23,6 +55,8 @@ export type ServerWorkTraceReport = { delayMs: number; windowMs: number; samples: WorkSample[]; + eventApplyByType: Record; + narrativeCheckpoint: EventApplyTotals; overflowCount: number; } | { kind: "server-work-slow-operation"; @@ -42,6 +76,8 @@ function safeId(value: string | undefined): string | null { /** Aggregates fixed-phase timings in memory and emits only delayed loop windows. */ export class ServerWorkTrace { private readonly samples = new Map(); + private eventApplyByType = emptyEventApplyTotals(); + private narrativeCheckpoint: EventApplyTotals = { count: 0, totalMs: 0, maxMs: 0 }; private overflowCount = 0; private lastTick = NodePerfHooks.performance.now(); private timer: ReturnType | undefined; @@ -61,21 +97,34 @@ export class ServerWorkTrace { if (this.timer) clearInterval(this.timer); this.timer = undefined; this.samples.clear(); + this.eventApplyByType = emptyEventApplyTotals(); + this.narrativeCheckpoint = { count: 0, totalMs: 0, maxMs: 0 }; this.overflowCount = 0; } /** Measure synchronous work without retaining its input or result. */ - measure(phase: ServerWorkPhase, threadId: string | undefined, executionId: string | undefined, work: () => T): T { + measure(phase: ServerWorkPhase, threadId: string | undefined, executionId: string | undefined, work: () => T, applyType?: EventApplyType): T { const started = NodePerfHooks.performance.now(); try { return work(); } finally { - this.record(phase, threadId, executionId, NodePerfHooks.performance.now() - started); + this.record(phase, threadId, executionId, NodePerfHooks.performance.now() - started, applyType); } } /** Record an asynchronous wait or a measured operation after completion. */ - record(phase: ServerWorkPhase, threadId: string | undefined, executionId: string | undefined, durationMs: number): void { + record(phase: ServerWorkPhase, threadId: string | undefined, executionId: string | undefined, durationMs: number, applyType?: EventApplyType): void { + if (phase === "event-apply" && applyType) { + const total = this.eventApplyByType[applyType]; + total.count += 1; + total.totalMs += durationMs; + total.maxMs = Math.max(total.maxMs, durationMs); + } + if (phase === "narrative-checkpoint") { + this.narrativeCheckpoint.count += 1; + this.narrativeCheckpoint.totalMs += durationMs; + this.narrativeCheckpoint.maxMs = Math.max(this.narrativeCheckpoint.maxMs, durationMs); + } const safeThreadId = safeId(threadId); const safeExecutionId = safeId(executionId); if (this.isSlowOperation(phase, durationMs)) { @@ -113,9 +162,11 @@ export class ServerWorkTrace { this.lastTick = now; const delayMs = Math.max(0, windowMs - INTERVAL_MS); if (delayMs >= REPORT_DELAY_MS) { - this.emit({ kind: "server-work-stall", at: Date.now(), delayMs, windowMs, samples: [...this.samples.values()], overflowCount: this.overflowCount }); + this.emit({ kind: "server-work-stall", at: Date.now(), delayMs, windowMs, samples: [...this.samples.values()], eventApplyByType: this.eventApplyByType, narrativeCheckpoint: this.narrativeCheckpoint, overflowCount: this.overflowCount }); } this.samples.clear(); + this.eventApplyByType = emptyEventApplyTotals(); + this.narrativeCheckpoint = { count: 0, totalMs: 0, maxMs: 0 }; this.overflowCount = 0; } } diff --git a/apps/server/src/features/agents/turns/provider-turn-event-application.ts b/apps/server/src/features/agents/turns/provider-turn-event-application.ts index 31c2be3a8..844120ca8 100644 --- a/apps/server/src/features/agents/turns/provider-turn-event-application.ts +++ b/apps/server/src/features/agents/turns/provider-turn-event-application.ts @@ -7,6 +7,7 @@ import { type ProviderId, } from "@mcode/contracts"; import { logger } from "@mcode/shared"; +import { serverWorkTrace } from "../diagnostics/server-work-trace.js"; import { broadcast } from "../../../application/transport/push.js"; import { BrowserNarrativeEventSanitizer } from "../../browser-automation/index.js"; @@ -588,7 +589,10 @@ export class ProviderTurnEventApplication implements TurnEventApplication { private checkpointNarrative(event: AgentEvent, publish: boolean): boolean { if (!publish || this.isUnsavedNarrationBoundary(event)) return true; try { - this.parentNarrativeRecovery.checkpoint(event); + if (serverWorkTrace) { + serverWorkTrace.measure("narrative-checkpoint", event.threadId, event.turnExecutionId, + () => this.parentNarrativeRecovery.checkpoint(event)); + } else this.parentNarrativeRecovery.checkpoint(event); return true; } catch { this.runtime.stopForEventApplicationFailure(event, "Parent narrative recovery checkpoint failed"); diff --git a/apps/server/src/features/providers/composition/provider-event-ingress.ts b/apps/server/src/features/providers/composition/provider-event-ingress.ts index da7a02247..0f5ea0d0e 100644 --- a/apps/server/src/features/providers/composition/provider-event-ingress.ts +++ b/apps/server/src/features/providers/composition/provider-event-ingress.ts @@ -31,7 +31,7 @@ import { type ProviderEventWorkerTask, } from "./provider-event-worker-protocol.js"; import { normalizeProviderError } from "./provider-error-normalize.js"; -import { serverWorkTrace } from "../../agents/diagnostics/server-work-trace.js"; +import { eventApplyType, serverWorkTrace } from "../../agents/diagnostics/server-work-trace.js"; const MAX_PENDING_NON_TERMINAL_EVENTS = 8_192; const MAX_PENDING_NON_TERMINAL_EVENTS_PER_THREAD = 2_048; @@ -459,7 +459,8 @@ export class ProviderEventIngress { serverWorkTrace.record("mailbox-wait", queued.event.event.threadId, queued.event.event.turnExecutionId, NodePerfHooks.performance.now() - queued.traceQueuedAt); serverWorkTrace.measure("event-apply", queued.event.event.threadId, - queued.event.event.turnExecutionId, () => this.consumer?.handleProviderEvent(queued.event)); + queued.event.event.turnExecutionId, () => this.consumer?.handleProviderEvent(queued.event), + eventApplyType(queued.event.event.type)); } else this.consumer.handleProviderEvent(queued.event); } if (this.pendingEventCount > 0) this.scheduleYieldedDrain(); From 2322bd4785e642ece9033138d5a1ca1e269578e9 Mon Sep 17 00:00:00 2001 From: "(CJ) Chukwudi Nwobodo" <142016413+chuks-qua@users.noreply.github.com> Date: Thu, 24 Sep 2026 19:33:11 +0100 Subject: [PATCH 004/136] fix(server): keep execution mailbox closed after Stop --- .../execution-mailbox-scheduler.test.ts | 70 ++++++++++++--- .../execution/execution-mailbox-protocol.ts | 2 + .../execution/execution-mailbox-scheduler.ts | 89 +++++++++++-------- 3 files changed, 113 insertions(+), 48 deletions(-) diff --git a/apps/server/src/features/agents/execution/__tests__/execution-mailbox-scheduler.test.ts b/apps/server/src/features/agents/execution/__tests__/execution-mailbox-scheduler.test.ts index f952d9190..fc37c2fca 100644 --- a/apps/server/src/features/agents/execution/__tests__/execution-mailbox-scheduler.test.ts +++ b/apps/server/src/features/agents/execution/__tests__/execution-mailbox-scheduler.test.ts @@ -11,19 +11,24 @@ import type { ExecutionWorkerRequest, } from "../execution-mailbox-protocol.js"; -type Work = { readonly kind: "start" } | { readonly kind: "event"; readonly sequence: number }; +type Work = + | { readonly kind: "start" } + | { readonly kind: "event"; readonly sequence: number } + | { readonly kind: "checkpoint"; readonly revision: number } + | { readonly kind: "effect-result"; readonly effectId: string } + | { readonly kind: "finalize" }; type Command = Work | { readonly kind: "stop"; readonly requestId: string }; type Result = { readonly revision: number }; const LIMITS: ExecutionMailboxLimits = { maxPending: 8, maxPendingBytes: 800, - reservedStop: 2, - reservedStopBytes: 200, - maxPerExecutionPending: 4, - maxPerExecutionBytes: 400, - reservedPerExecutionStop: 1, - reservedPerExecutionStopBytes: 100, + reservedControl: 2, + reservedControlBytes: 200, + maxPerExecutionPending: 5, + maxPerExecutionBytes: 500, + reservedPerExecutionControl: 2, + reservedPerExecutionControlBytes: 200, }; class FakeWorker implements ExecutionWorkerPort { @@ -131,7 +136,7 @@ describe("ExecutionMailboxScheduler", () => { scheduler.shutdown(); }); - it("reserves count and byte credits for Stop, including within one execution", async () => { + it("reserves count and byte credits for control commands within one execution", async () => { const { scheduler, workers } = fixture(); const identity = execution("busy"); const lease = claimed(scheduler, identity); @@ -143,14 +148,53 @@ describe("ExecutionMailboxScheduler", () => { const stop = admitted(scheduler, identity, lease, { kind: "stop", requestId: "stop-1" }); expect(scheduler.depth()).toMatchObject({ pending: 4, pendingBytes: 400 }); expect(scheduler.submit({ execution: identity, lease, command: { kind: "stop", requestId: "stop-2" }, byteLength: 100 })) - .toEqual({ kind: "stop-pending" }); + .toEqual({ kind: "stop-already-requested" }); + const checkpoint = admitted(scheduler, identity, lease, { kind: "checkpoint", revision: 1 }); + expect(scheduler.depth()).toMatchObject({ pending: 5, pendingBytes: 500 }); - for (let index = 0; index < 4; index += 1) workers[0]?.reply(request(workers[0]!, index), index + 1); + for (let index = 0; index < 5; index += 1) workers[0]?.reply(request(workers[0]!, index), index + 1); expect((await stop.completion).kind).toBe("reply"); + expect((await checkpoint.completion).kind).toBe("reply"); expect((await first.completion).kind).toBe("reply"); expect((await second.completion).kind).toBe("reply"); expect((await third.completion).kind).toBe("reply"); - expect(workers[0]?.requests.map((item) => item.command.kind)).toEqual(["event", "event", "event", "stop"]); + expect(workers[0]?.requests.map((item) => item.command.kind)).toEqual(["event", "event", "event", "stop", "checkpoint"]); + scheduler.shutdown(); + }); + + it("closes ordinary admission at Stop's watermark while lifecycle controls still settle", async () => { + const { scheduler, workers } = fixture(); + const identity = execution("stopping"); + const lease = claimed(scheduler, identity); + const start = admitted(scheduler, identity, lease, { kind: "start" }); + const event = admitted(scheduler, identity, lease, { kind: "event", sequence: 1 }); + const stop = admitted(scheduler, identity, lease, { kind: "stop", requestId: "stop-now" }); + expect(scheduler.submit({ execution: identity, lease, command: { kind: "event", sequence: 2 }, byteLength: 100 })) + .toEqual({ kind: "stopping" }); + const checkpoint = admitted(scheduler, identity, lease, { kind: "checkpoint", revision: 2 }); + const effect = admitted(scheduler, identity, lease, { kind: "effect-result", effectId: "file-1" }); + expect(request(workers[0]!, 0).ordinal).toBe(1); + workers[0]?.reply(request(workers[0]!, 0), 1); + workers[0]?.reply(request(workers[0]!, 1), 2); + expect(request(workers[0]!, 2)).toMatchObject({ + ordinal: 3, + stopWatermark: 2, + command: { kind: "stop", requestId: "stop-now" }, + }); + workers[0]?.reply(request(workers[0]!, 2), 3); + expect(await stop.completion).toEqual({ kind: "reply", result: { revision: 3 } }); + expect(scheduler.submit({ execution: identity, lease, command: { kind: "event", sequence: 3 }, byteLength: 100 })) + .toEqual({ kind: "stopping" }); + expect(scheduler.submit({ execution: identity, lease, command: { kind: "stop", requestId: "again" }, byteLength: 100 })) + .toEqual({ kind: "stop-already-requested" }); + const finalize = admitted(scheduler, identity, lease, { kind: "finalize" }); + for (let index = 3; index < 6; index += 1) workers[0]?.reply(request(workers[0]!, index), index + 1); + expect(await start.completion).toEqual({ kind: "reply", result: { revision: 1 } }); + expect(await event.completion).toEqual({ kind: "reply", result: { revision: 2 } }); + expect((await checkpoint.completion).kind).toBe("reply"); + expect((await effect.completion).kind).toBe("reply"); + expect((await finalize.completion).kind).toBe("reply"); + expect(scheduler.release(identity, lease)).toBe(true); scheduler.shutdown(); }); @@ -159,8 +203,8 @@ describe("ExecutionMailboxScheduler", () => { ...LIMITS, maxPending: 10, maxPendingBytes: 500, - reservedStop: 1, - reservedStopBytes: 100, + reservedControl: 1, + reservedControlBytes: 100, maxPerExecutionBytes: 500, }; const { scheduler } = fixture(1, limits); diff --git a/apps/server/src/features/agents/execution/execution-mailbox-protocol.ts b/apps/server/src/features/agents/execution/execution-mailbox-protocol.ts index 11f6de596..27ae1e185 100644 --- a/apps/server/src/features/agents/execution/execution-mailbox-protocol.ts +++ b/apps/server/src/features/agents/execution/execution-mailbox-protocol.ts @@ -19,6 +19,8 @@ export interface ExecutionWorkerRequest { readonly execution: ExecutionIdentity; readonly lease: ExecutionLease; readonly ordinal: number; + /** Last admitted ordinal before Stop. Present only on a Stop request. */ + readonly stopWatermark?: number; readonly command: Command; } diff --git a/apps/server/src/features/agents/execution/execution-mailbox-scheduler.ts b/apps/server/src/features/agents/execution/execution-mailbox-scheduler.ts index c48eaebc5..e3e062953 100644 --- a/apps/server/src/features/agents/execution/execution-mailbox-scheduler.ts +++ b/apps/server/src/features/agents/execution/execution-mailbox-scheduler.ts @@ -14,6 +14,11 @@ export type ExecutionMailboxCommand = | Work | { readonly kind: "stop"; readonly requestId: string }; +/** Lifecycle results that must be allowed to settle after Stop is admitted. */ +export const EXECUTION_CONTROL_KINDS = ["checkpoint", "effect-result", "provider-outcome", "finalize"] as const; +export type ExecutionControlKind = typeof EXECUTION_CONTROL_KINDS[number]; +const CONTROL_KINDS: ReadonlySet = new Set(EXECUTION_CONTROL_KINDS); + interface Pending { readonly request: ExecutionWorkerRequest; readonly byteLength: number; @@ -30,7 +35,7 @@ interface Assignment { pendingBytes: number; normalCount: number; normalBytes: number; - stopPending: boolean; + stopping: boolean; } interface Slot { @@ -43,23 +48,23 @@ interface Slot { activeCount: number; } -/** Capacity includes posted work until its reply arrives. Stop has its own reserve. */ +/** Capacity includes posted work until its reply arrives. Control has a reserve. */ export interface ExecutionMailboxLimits { readonly maxPending: number; readonly maxPendingBytes: number; - readonly reservedStop: number; - readonly reservedStopBytes: number; + readonly reservedControl: number; + readonly reservedControlBytes: number; readonly maxPerExecutionPending: number; readonly maxPerExecutionBytes: number; - readonly reservedPerExecutionStop: number; - readonly reservedPerExecutionStopBytes: number; + readonly reservedPerExecutionControl: number; + readonly reservedPerExecutionControlBytes: number; } /** The host supplies durable owner epochs and a worker factory for each fixed slot. */ -export interface ExecutionMailboxOptions { +export interface ExecutionMailboxOptions { readonly workerCount: number; readonly limits: ExecutionMailboxLimits; - readonly createWorker: (workerIndex: number) => ExecutionWorkerPort; + readonly createWorker: (workerIndex: number) => ExecutionWorkerPort, Result>; readonly onWorkerLost: (executions: readonly ExecutionIdentity[]) => void; } @@ -69,7 +74,11 @@ export type ExecutionClaim = export type ExecutionAdmission = | { readonly kind: "admitted"; readonly ordinal: number; readonly completion: Promise> } - | { readonly kind: "stale-execution" | "overloaded" | "invalid-size" | "stop-pending" | "shutdown" }; + | { readonly kind: "stale-execution" | "overloaded" | "invalid-size" | "stop-already-requested" | "stopping" | "shutdown" }; + +type AdmissionDecision = + | { readonly kind: "accept"; readonly control: boolean; readonly stop: boolean } + | { readonly kind: "invalid-size" | "overloaded" | "stop-already-requested" | "stopping" }; /** Bounded state visible to diagnostics without exposing command payloads. */ export interface ExecutionMailboxDepth { @@ -88,8 +97,8 @@ export class ExecutionMailboxScheduler, Result>[]; private readonly byThread = new Map, Result>>(); private readonly limits: ExecutionMailboxLimits; - private readonly createWorker: ExecutionMailboxOptions, Result>["createWorker"]; - private readonly onWorkerLost: ExecutionMailboxOptions, Result>["onWorkerLost"]; + private readonly createWorker: ExecutionMailboxOptions["createWorker"]; + private readonly onWorkerLost: ExecutionMailboxOptions["onWorkerLost"]; private pendingCount = 0; private pendingBytes = 0; private normalCount = 0; @@ -98,7 +107,7 @@ export class ExecutionMailboxScheduler, Result>) { + constructor(options: ExecutionMailboxOptions) { validateOptions(options); this.limits = options.limits; this.createWorker = options.createWorker; @@ -142,7 +151,7 @@ export class ExecutionMailboxScheduler> = { requestId: this.nextRequestId++, execution: assignment.execution, lease: assignment.lease, ordinal, + ...(decision.stop ? { stopWatermark: ordinal - 1 } : {}), command: input.command, }; let resolveCompletion: (completion: ExecutionMailboxCompletion) => void = () => {}; @@ -177,10 +185,11 @@ export class ExecutionMailboxScheduler, Result> = { request, byteLength: input.byteLength, - control, + control: decision.control, resolve: resolveCompletion, }; this.retain(assignment, pending); + if (decision.stop) assignment.stopping = true; this.enqueue(assignment.slot, pending); this.dispatch(assignment.slot); return { kind: "admitted", ordinal, completion }; @@ -250,16 +259,30 @@ export class ExecutionMailboxScheduler, Result>, + command: ExecutionMailboxCommand, + bytes: number, + ): AdmissionDecision { + if (!positiveInteger(bytes)) return { kind: "invalid-size" }; + const stop = command.kind === "stop"; + const control = stop || CONTROL_KINDS.has(command.kind); + if (stop && assignment.stopping) return { kind: "stop-already-requested" }; + if (assignment.stopping && !control) return { kind: "stopping" }; + if (!this.hasCapacity(assignment, bytes, control)) return { kind: "overloaded" }; + return { kind: "accept", control, stop }; + } + private hasCapacity(assignment: Assignment, Result>, bytes: number, control: boolean): boolean { const limits = this.limits; if (this.pendingCount + 1 > limits.maxPending || this.pendingBytes + bytes > limits.maxPendingBytes) return false; if (assignment.pendingCount + 1 > limits.maxPerExecutionPending || assignment.pendingBytes + bytes > limits.maxPerExecutionBytes) return false; if (control) return true; - return this.normalCount + 1 <= limits.maxPending - limits.reservedStop - && this.normalBytes + bytes <= limits.maxPendingBytes - limits.reservedStopBytes - && assignment.normalCount + 1 <= limits.maxPerExecutionPending - limits.reservedPerExecutionStop - && assignment.normalBytes + bytes <= limits.maxPerExecutionBytes - limits.reservedPerExecutionStopBytes; + return this.normalCount + 1 <= limits.maxPending - limits.reservedControl + && this.normalBytes + bytes <= limits.maxPendingBytes - limits.reservedControlBytes + && assignment.normalCount + 1 <= limits.maxPerExecutionPending - limits.reservedPerExecutionControl + && assignment.normalBytes + bytes <= limits.maxPerExecutionBytes - limits.reservedPerExecutionControlBytes; } private retain(assignment: Assignment, Result>, pending: Pending, Result>): void { @@ -267,10 +290,7 @@ export class ExecutionMailboxScheduler 0; } -function validateOptions(options: ExecutionMailboxOptions): void { +function validateOptions(options: ExecutionMailboxOptions): void { const limits = options.limits; const values = [options.workerCount, limits.maxPending, limits.maxPendingBytes, limits.maxPerExecutionPending, - limits.maxPerExecutionBytes, limits.reservedStop, limits.reservedStopBytes, - limits.reservedPerExecutionStop, limits.reservedPerExecutionStopBytes]; + limits.maxPerExecutionBytes, limits.reservedControl, limits.reservedControlBytes, + limits.reservedPerExecutionControl, limits.reservedPerExecutionControlBytes]; if (values.some((value) => !positiveInteger(value))) throw new Error("Mailbox limits must be positive safe integers"); - if (limits.reservedStop >= limits.maxPending || limits.reservedStopBytes >= limits.maxPendingBytes - || limits.reservedPerExecutionStop >= limits.maxPerExecutionPending - || limits.reservedPerExecutionStopBytes >= limits.maxPerExecutionBytes) { - throw new Error("Mailbox Stop reserves must leave normal capacity"); + if (limits.reservedControl >= limits.maxPending || limits.reservedControlBytes >= limits.maxPendingBytes + || limits.reservedPerExecutionControl >= limits.maxPerExecutionPending + || limits.reservedPerExecutionControlBytes >= limits.maxPerExecutionBytes) { + throw new Error("Mailbox control reserves must leave normal capacity"); } } From ae7af2f8a103691d2c62fc15da5ecdf2a9af6bf7 Mon Sep 17 00:00:00 2001 From: "(CJ) Chukwudi Nwobodo" <142016413+chuks-qua@users.noreply.github.com> Date: Thu, 24 Sep 2026 19:32:25 +0100 Subject: [PATCH 005/136] feat(server): add acknowledged canonical SQLite writer worker --- .../build-server-runtime-bundles.test.mjs | 15 ++ .../canonical-agent-writer-client.test.ts | 163 ++++++++++++++++++ .../canonical-agent-writer-client.ts | 138 +++++++++++++++ .../canonical-agent-writer-protocol.ts | 35 ++++ .../canonical-agent-writer.worker.ts | 90 ++++++++++ scripts/build-server-dev-bundle.mjs | 6 + 6 files changed, 447 insertions(+) create mode 100644 apps/server/src/features/agents/canonical/__tests__/canonical-agent-writer-client.test.ts create mode 100644 apps/server/src/features/agents/canonical/canonical-agent-writer-client.ts create mode 100644 apps/server/src/features/agents/canonical/canonical-agent-writer-protocol.ts create mode 100644 apps/server/src/features/agents/canonical/canonical-agent-writer.worker.ts diff --git a/apps/desktop/scripts/__tests__/build-server-runtime-bundles.test.mjs b/apps/desktop/scripts/__tests__/build-server-runtime-bundles.test.mjs index 1e030af0b..432fb08f9 100644 --- a/apps/desktop/scripts/__tests__/build-server-runtime-bundles.test.mjs +++ b/apps/desktop/scripts/__tests__/build-server-runtime-bundles.test.mjs @@ -24,6 +24,7 @@ describe("buildServerRuntimeBundles", () => { const serverOutFile = NodePath.join(outputRoot, "server.cjs"); const ptyHostOutFile = NodePath.join(outputRoot, "pty-host.cjs"); const providerEventWorkerOutFile = NodePath.join(outputRoot, "provider-event.worker.cjs"); + const canonicalWriterWorkerOutFile = NodePath.join(outputRoot, "canonical-agent-writer.worker.cjs"); try { compileServerWithSwc(serverRoot); @@ -36,6 +37,7 @@ describe("buildServerRuntimeBundles", () => { expect(() => new NodeVM.Script(bundledEntry, { filename: "server.cjs" })).not.toThrow(); await NodeFSPromises.access(providerEventWorkerOutFile); + await NodeFSPromises.access(canonicalWriterWorkerOutFile); expect(await NodeFSPromises.readFile(serverOutFile, "utf8")).toContain("provider-event.worker.cjs"); } finally { await NodeFSPromises.rm(outputRoot, { recursive: true, force: true }); @@ -63,17 +65,28 @@ describe("buildServerRuntimeBundles", () => { "composition", "provider-event.worker.js", ); + const canonicalWriterWorkerEntry = NodePath.join( + serverRoot, + "dist-tsc", + "features", + "agents", + "canonical", + "canonical-agent-writer.worker.js", + ); const serverOutFile = NodePath.join(fixtureRoot, "dist", "server.cjs"); const ptyHostOutFile = NodePath.join(fixtureRoot, "dist", "pty-host.cjs"); const providerEventWorkerOutFile = NodePath.join(fixtureRoot, "dist", "provider-event.worker.cjs"); + const canonicalWriterWorkerOutFile = NodePath.join(fixtureRoot, "dist", "canonical-agent-writer.worker.cjs"); await Promise.all([ NodeFSPromises.mkdir(NodePath.dirname(ptyHostEntry), { recursive: true }), NodeFSPromises.mkdir(NodePath.dirname(providerEventWorkerEntry), { recursive: true }), + NodeFSPromises.mkdir(NodePath.dirname(canonicalWriterWorkerEntry), { recursive: true }), ]); await NodeFSPromises.writeFile(serverEntry, 'console.log("server-entry");\n'); await NodeFSPromises.writeFile(ptyHostEntry, 'console.log("pty-host-entry");\n'); await NodeFSPromises.writeFile(providerEventWorkerEntry, 'console.log("provider-event-worker");\n'); + await NodeFSPromises.writeFile(canonicalWriterWorkerEntry, 'console.log("canonical-writer-worker");\n'); await buildServerRuntimeBundles({ serverRoot, @@ -85,8 +98,10 @@ describe("buildServerRuntimeBundles", () => { await NodeFSPromises.access(serverOutFile); await NodeFSPromises.access(ptyHostOutFile); await NodeFSPromises.access(providerEventWorkerOutFile); + await NodeFSPromises.access(canonicalWriterWorkerOutFile); expect(await NodeFSPromises.readFile(serverOutFile, "utf8")).toContain("server-entry"); expect(await NodeFSPromises.readFile(ptyHostOutFile, "utf8")).toContain("pty-host-entry"); expect(await NodeFSPromises.readFile(providerEventWorkerOutFile, "utf8")).toContain("provider-event-worker"); + expect(await NodeFSPromises.readFile(canonicalWriterWorkerOutFile, "utf8")).toContain("canonical-writer-worker"); }, 30_000); }); diff --git a/apps/server/src/features/agents/canonical/__tests__/canonical-agent-writer-client.test.ts b/apps/server/src/features/agents/canonical/__tests__/canonical-agent-writer-client.test.ts new file mode 100644 index 000000000..21300173d --- /dev/null +++ b/apps/server/src/features/agents/canonical/__tests__/canonical-agent-writer-client.test.ts @@ -0,0 +1,163 @@ +import "reflect-metadata"; +import type { Database } from "bun:sqlite"; +import * as NodeFSPromises from "node:fs/promises"; +import * as NodeOS from "node:os"; +import * as NodePath from "node:path"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { openDatabase } from "../../../../runtime/persistence/sqlite/database.js"; +import type { CanonicalAgentEventDraft } from "../canonical-agent-boundary.js"; +import { CanonicalAgentWriterClient } from "../canonical-agent-writer-client.js"; + +const THREAD_ID = "writer-thread"; +const TURN_ID = "writer-turn"; +const EXECUTION_ID = "00000000-0000-4000-8000-000000000176"; +const NOW = "2026-09-24T12:00:00.000Z"; + +function events(): CanonicalAgentEventDraft[] { + const sourceIdentities = [{ providerId: "codex" as const, scope: "thread" as const, value: "native-writer-thread", provenance: "native" as const }]; + return [ + { + eventId: `${EXECUTION_ID}:thread`, + routing: { threadId: THREAD_ID, executionId: EXECUTION_ID }, + sourceProviderId: "codex", + sourceIdentities, + payload: { + type: "thread.recorded", + thread: { + id: THREAD_ID, + workspaceId: "writer-workspace", + rootThreadId: THREAD_ID, + providerId: "codex", + providerIdentities: sourceIdentities, + activityState: "Active", + conversationRevision: 0, + rosterRevision: 0, + createdAt: NOW, + updatedAt: NOW, + }, + }, + }, + { + eventId: `${EXECUTION_ID}:turn`, + routing: { threadId: THREAD_ID, turnId: TURN_ID, executionId: EXECUTION_ID }, + sourceProviderId: "codex", + sourceIdentities, + payload: { + type: "turn.created", + turn: { + id: TURN_ID, + threadId: THREAD_ID, + status: "Pending", + trigger: { kind: "user" }, + permissionMode: "supervised", + approvalReviewMode: "manual", + approvalReviewReason: "manual-requested", + providerIdentities: sourceIdentities, + startedAt: null, + endedAt: null, + createdAt: NOW, + updatedAt: NOW, + }, + }, + }, + { + eventId: `${EXECUTION_ID}:started`, + routing: { threadId: THREAD_ID, turnId: TURN_ID, executionId: EXECUTION_ID }, + sourceProviderId: "codex", + sourceIdentities, + payload: { type: "turn.started", startedAt: NOW }, + }, + ]; +} + +describe("canonical SQLite writer", () => { + let tempDir: string; + let dbPath: string; + let db: Database; + let writer: CanonicalAgentWriterClient | undefined; + + beforeEach(async () => { + tempDir = await NodeFSPromises.mkdtemp(NodePath.join(NodeOS.tmpdir(), "mcode-canonical-writer-")); + dbPath = NodePath.join(tempDir, "app.sqlite"); + db = openDatabase({ dbPath }); + db.prepare("INSERT INTO workspaces (id, name, path, created_at, updated_at) VALUES (?, ?, ?, ?, ?)") + .run("writer-workspace", "Writer test", tempDir, NOW, NOW); + db.prepare("INSERT INTO threads (id, workspace_id, title, branch, provider, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?)") + .run(THREAD_ID, "writer-workspace", "Writer thread", "main", "codex", NOW, NOW); + }); + + afterEach(async () => { + await writer?.close(); + db.close(true); + await NodeFSPromises.rm(tempDir, { recursive: true, force: true }); + }); + + it("acknowledges only committed events and deduplicates a replay", async () => { + writer = new CanonicalAgentWriterClient(dbPath); + const batch = { threadId: THREAD_ID, turnId: TURN_ID, executionId: EXECUTION_ID, phase: "running", events: events() }; + const first = await writer.commit("writer-operation-1", batch); + expect(first).toMatchObject({ outcome: "committed", acceptedThrough: 3, durableThrough: 3 }); + expect(first.events.map((event) => event.eventId)).toEqual(batch.events.map((event) => event.eventId)); + expect(first).not.toHaveProperty("canonicalDelivery"); + expect(db.prepare("SELECT COUNT(*) AS count FROM canonical_agent_events").get()).toEqual({ count: 3 }); + + const replay = await writer.commit("writer-operation-1", batch); + expect(replay).toMatchObject({ outcome: "duplicate", acceptedThrough: 3, durableThrough: 3, events: [] }); + expect(db.prepare("SELECT COUNT(*) AS count FROM canonical_agent_events").get()).toEqual({ count: 3 }); + }); + + it("rejects a database failure without reporting a durable receipt", async () => { + writer = new CanonicalAgentWriterClient(dbPath); + await writer.whenReady(); + db.run("DROP TABLE canonical_agent_events"); + await expect(writer.commit("writer-operation-2", { + threadId: THREAD_ID, + turnId: TURN_ID, + executionId: EXECUTION_ID, + phase: "running", + events: events(), + })).rejects.toThrow("Canonical writer write-failed"); + expect(db.prepare("SELECT COUNT(*) AS count FROM canonical_agent_ingest_checkpoints").get()).toEqual({ count: 0 }); + }); + + it("rejects an unmigrated path without creating a second database", async () => { + const missingPath = NodePath.join(tempDir, "missing.sqlite"); + writer = new CanonicalAgentWriterClient(missingPath); + await expect(writer.whenReady()).rejects.toThrow("Canonical writer open-failed"); + await expect(NodeFSPromises.stat(missingPath)).rejects.toMatchObject({ code: "ENOENT" }); + }); + + it("bounds write admission while SQLite is busy", async () => { + writer = new CanonicalAgentWriterClient(dbPath); + const activeWriter = writer; + await activeWriter.whenReady(); + const batch = { threadId: THREAD_ID, turnId: TURN_ID, executionId: EXECUTION_ID, phase: "running", events: events() }; + db.run("BEGIN IMMEDIATE"); + let locked = true; + try { + const accepted = Array.from({ length: 64 }, (_, index) => activeWriter.commit(`queued-${index}`, batch)); + await expect(activeWriter.commit("overflow", batch)).rejects.toThrow("Canonical writer admission is full"); + db.run("ROLLBACK"); + locked = false; + const results = await Promise.all(accepted); + expect(results[0]?.outcome).toBe("committed"); + expect(results.slice(1).every((result) => result.outcome === "duplicate")).toBe(true); + } finally { + if (locked) db.run("ROLLBACK"); + } + }); + + it("rejects future writes if the worker exits", async () => { + let worker: Worker | undefined; + writer = new CanonicalAgentWriterClient(dbPath, () => { + worker = new Worker(new URL("../canonical-agent-writer.worker.ts", import.meta.url), { type: "module" }); + return worker; + }); + const batch = { threadId: THREAD_ID, turnId: TURN_ID, executionId: EXECUTION_ID, phase: "running", events: events() }; + await writer.commit("writer-operation-3", batch); + const closed = new Promise((resolve) => worker?.addEventListener("close", resolve, { once: true })); + worker?.terminate(); + await closed; + await expect(writer.commit("writer-operation-4", batch)).rejects.toThrow("Canonical writer worker closed"); + }); +}); diff --git a/apps/server/src/features/agents/canonical/canonical-agent-writer-client.ts b/apps/server/src/features/agents/canonical/canonical-agent-writer-client.ts new file mode 100644 index 000000000..dbcc6e2e3 --- /dev/null +++ b/apps/server/src/features/agents/canonical/canonical-agent-writer-client.ts @@ -0,0 +1,138 @@ +import * as NodePath from "node:path"; +import * as NodeCrypto from "node:crypto"; +import type { + CanonicalProviderWriteInput, + CanonicalProviderWriteReceipt, + CanonicalWriterRequest, + CanonicalWriterResponse, +} from "./canonical-agent-writer-protocol.js"; + +const MAX_PENDING_WRITES = 64; +const WRITER_EXECUTION_ID = "writer:init"; + +interface PendingRequest { + request: CanonicalWriterRequest; + resolve(response: CanonicalWriterResponse): void; + reject(error: Error): void; +} + +/** Sends cloneable canonical batches to one dedicated SQLite worker with bounded admission. */ +export class CanonicalAgentWriterClient { + private readonly worker: Worker; + private readonly pending = new Map(); + private readonly ready: Promise; + private readonly closed: Promise; + private markClosed: (() => void) | undefined; + private failure: Error | undefined; + + constructor(dbPath: string, createWorker: () => Worker = defaultCreateWorker) { + if (!NodePath.isAbsolute(dbPath)) throw new Error("Canonical writer database path must be absolute"); + this.worker = createWorker(); + this.closed = new Promise((resolve) => { this.markClosed = resolve; }); + this.worker.onmessage = (message: MessageEvent) => this.receive(message.data); + this.worker.onerror = () => this.fail(new Error("Canonical writer worker failed")); + this.worker.addEventListener("close", () => { + this.markClosed?.(); + this.fail(new Error("Canonical writer worker closed")); + }); + this.ready = this.send({ + kind: "open", + requestId: NodeCrypto.randomUUID(), + operationId: "writer:open", + executionId: WRITER_EXECUTION_ID, + dbPath, + }).then((response) => { + if (response.kind !== "opened") throw new Error("Canonical writer did not open its database"); + }).catch((error: Error) => { + this.fail(error); + throw error; + }); + } + + /** Waits until the worker has opened the already-migrated database. */ + whenReady(): Promise { + return this.ready; + } + + /** Resolves with committed envelopes for main-loop publication; it never publishes them itself. */ + async commit(operationId: string, input: CanonicalProviderWriteInput): Promise { + if (!operationId || !input.executionId) throw new Error("Canonical writer operation and execution IDs are required"); + await this.ready; + const response = await this.send({ + kind: "commit", + requestId: NodeCrypto.randomUUID(), + operationId, + executionId: input.executionId, + input, + }); + if (response.kind !== "committed") throw new Error("Canonical writer returned an unexpected response"); + return response.receipt; + } + + /** Closes the database after accepted writes settle, then releases the worker. */ + async close(): Promise { + try { + await this.ready; + if (!this.failure) { + await this.send({ + kind: "close", + requestId: NodeCrypto.randomUUID(), + operationId: "writer:close", + executionId: WRITER_EXECUTION_ID, + }); + } + } catch (error) { + if (!this.failure) throw error; + } finally { + this.fail(new Error("Canonical writer closed")); + await this.closed; + } + } + + private send(request: CanonicalWriterRequest): Promise { + if (this.failure) return Promise.reject(this.failure); + if (request.kind === "commit" && this.pending.size >= MAX_PENDING_WRITES) { + return Promise.reject(new Error("Canonical writer admission is full")); + } + return new Promise((resolve, reject) => { + this.pending.set(request.requestId, { request, resolve, reject }); + try { + this.worker.postMessage(request); + } catch { + this.pending.delete(request.requestId); + reject(new Error("Canonical writer request could not be sent")); + } + }); + } + + private receive(response: CanonicalWriterResponse): void { + const pending = this.pending.get(response.requestId); + if (!pending) return; + if (pending.request.operationId !== response.operationId + || pending.request.executionId !== response.executionId) { + this.fail(new Error("Canonical writer response identity mismatch")); + return; + } + this.pending.delete(response.requestId); + if (response.kind === "failed") { + pending.reject(new Error(`Canonical writer ${response.reason}`)); + return; + } + pending.resolve(response); + } + + private fail(error: Error): void { + if (this.failure) return; + this.failure = error; + this.worker.terminate(); + for (const pending of this.pending.values()) pending.reject(error); + this.pending.clear(); + } +} + +function defaultCreateWorker(): Worker { + const workerFile = import.meta.url.endsWith(".cjs") + ? "./canonical-agent-writer.worker.cjs" + : "./canonical-agent-writer.worker.ts"; + return new Worker(new URL(workerFile, import.meta.url), { type: "module" }); +} diff --git a/apps/server/src/features/agents/canonical/canonical-agent-writer-protocol.ts b/apps/server/src/features/agents/canonical/canonical-agent-writer-protocol.ts new file mode 100644 index 000000000..c92f61984 --- /dev/null +++ b/apps/server/src/features/agents/canonical/canonical-agent-writer-protocol.ts @@ -0,0 +1,35 @@ +import type { CanonicalAgentEventEnvelope } from "@mcode/contracts"; +import type { CanonicalAgentCommitInput, CanonicalAgentCommitResult, CanonicalAgentEventDraft } from "./canonical-agent-boundary.js"; + +/** Cloneable provider batch accepted by the SQLite writer. Compatibility callbacks stay with their owner. */ +export type CanonicalProviderWriteInput = Pick< + CanonicalAgentCommitInput, + "threadId" | "turnId" | "executionId" | "phase" | "nativeCursor" +> & { events: readonly CanonicalAgentEventDraft[] }; + +/** Acknowledgement returned only after the canonical SQLite transaction has committed. */ +export interface CanonicalProviderWriteReceipt { + outcome: CanonicalAgentCommitResult["outcome"]; + conversationRevision: number; + rosterRevision: number; + acceptedThrough: number; + durableThrough: number; + events: readonly CanonicalAgentEventEnvelope[]; +} + +interface Correlation { + requestId: string; + operationId: string; + executionId: string; +} + +export type CanonicalWriterRequest = + | (Correlation & { kind: "open"; dbPath: string }) + | (Correlation & { kind: "commit"; input: CanonicalProviderWriteInput }) + | (Correlation & { kind: "close" }); + +export type CanonicalWriterResponse = + | (Correlation & { kind: "opened" }) + | (Correlation & { kind: "committed"; receipt: CanonicalProviderWriteReceipt }) + | (Correlation & { kind: "closed" }) + | (Correlation & { kind: "failed"; reason: "open-failed" | "write-failed" }); diff --git a/apps/server/src/features/agents/canonical/canonical-agent-writer.worker.ts b/apps/server/src/features/agents/canonical/canonical-agent-writer.worker.ts new file mode 100644 index 000000000..793dfa055 --- /dev/null +++ b/apps/server/src/features/agents/canonical/canonical-agent-writer.worker.ts @@ -0,0 +1,90 @@ +import "reflect-metadata"; +import { Database } from "bun:sqlite"; +import * as NodeFS from "node:fs"; +import * as NodePath from "node:path"; +import { applySQLiteConnectionPolicy } from "../../../runtime/persistence/sqlite/sqlite-connection-policy.js"; +import { CanonicalAgentBoundary } from "./canonical-agent-boundary.js"; +import type { + CanonicalProviderWriteInput, + CanonicalWriterRequest, + CanonicalWriterResponse, +} from "./canonical-agent-writer-protocol.js"; + +let db: Database | undefined; +let boundary: CanonicalAgentBoundary | undefined; + +function openDatabase(dbPath: string): void { + if (boundary || !NodePath.isAbsolute(dbPath) || !NodeFS.existsSync(dbPath)) { + throw new Error("Canonical writer requires an existing absolute database path"); + } + const connection = new Database(dbPath, { strict: true, readwrite: true }); + try { + applySQLiteConnectionPolicy(connection, true); + boundary = new CanonicalAgentBoundary(connection, () => {}); + db = connection; + } catch (error) { + connection.close(true); + throw error; + } +} + +function assertRouting(input: CanonicalProviderWriteInput, executionId: string): void { + if (input.executionId !== executionId || !input.threadId || !input.turnId || !input.phase + || !Array.isArray(input.events) || input.events.length === 0) { + throw new Error("Invalid canonical writer request"); + } + if (!input.events.every((event) => eventMatchesRouting(event, input, executionId))) { + throw new Error("Canonical writer event routing mismatch"); + } +} + +function eventMatchesRouting( + event: CanonicalProviderWriteInput["events"][number], + input: CanonicalProviderWriteInput, + executionId: string, +): boolean { + return event.routing.threadId === input.threadId + && event.routing.executionId === executionId + && (event.routing.turnId === undefined || event.routing.turnId === input.turnId); +} + +function handle(request: CanonicalWriterRequest): CanonicalWriterResponse { + const correlation = { + requestId: request.requestId, + operationId: request.operationId, + executionId: request.executionId, + }; + if (request.kind === "open") { + try { + openDatabase(request.dbPath); + return { ...correlation, kind: "opened" }; + } catch { + return { ...correlation, kind: "failed", reason: "open-failed" }; + } + } + if (request.kind === "close") { + boundary = undefined; + db?.close(true); + db = undefined; + return { ...correlation, kind: "closed" }; + } + try { + if (!boundary) throw new Error("Canonical writer has not opened its database"); + assertRouting(request.input, request.executionId); + const result = boundary.commit(request.input); + const { outcome, conversationRevision, rosterRevision, acceptedThrough, durableThrough, events } = result; + return { + ...correlation, + kind: "committed", + receipt: { outcome, conversationRevision, rosterRevision, acceptedThrough, durableThrough, events }, + }; + } catch { + return { ...correlation, kind: "failed", reason: "write-failed" }; + } +} + +globalThis.onmessage = (message: MessageEvent): void => { + globalThis.postMessage(handle(message.data)); +}; + +process.on("exit", () => db?.close(true)); diff --git a/scripts/build-server-dev-bundle.mjs b/scripts/build-server-dev-bundle.mjs index 70038022a..8af56652e 100644 --- a/scripts/build-server-dev-bundle.mjs +++ b/scripts/build-server-dev-bundle.mjs @@ -580,6 +580,7 @@ export async function buildServerRuntimeBundles({ serverOutFile, ptyHostOutFile, providerEventWorkerOutFile = NodePath.resolve(NodePath.dirname(serverOutFile), "provider-event.worker.cjs"), + canonicalWriterWorkerOutFile = NodePath.resolve(NodePath.dirname(serverOutFile), "canonical-agent-writer.worker.cjs"), production = false, }) { const shared = { @@ -620,6 +621,11 @@ export async function buildServerRuntimeBundles({ entryPoints: [providerEventWorkerEntry], outfile: providerEventWorkerOutFile, })); + bundles.push(build({ + ...shared, + entryPoints: [NodePath.resolve(serverRoot, "dist-tsc/features/agents/canonical/canonical-agent-writer.worker.js")], + outfile: canonicalWriterWorkerOutFile, + })); await Promise.all(bundles); } From 65d74b3f19233c5dce2b6c48c50c33bec33f0cd7 Mon Sep 17 00:00:00 2001 From: "(CJ) Chukwudi Nwobodo" <142016413+chuks-qua@users.noreply.github.com> Date: Thu, 24 Sep 2026 19:35:37 +0100 Subject: [PATCH 006/136] perf(server): split narrative checkpoint trace timings --- .../__tests__/server-work-trace.test.ts | 18 ++++++++++++- .../agents/diagnostics/server-work-trace.ts | 26 +++++++++++++++++-- .../parent-narrative-recovery-coordinator.ts | 17 +++++++++--- 3 files changed, 55 insertions(+), 6 deletions(-) diff --git a/apps/server/src/features/agents/diagnostics/__tests__/server-work-trace.test.ts b/apps/server/src/features/agents/diagnostics/__tests__/server-work-trace.test.ts index a24018a35..77467d1d4 100644 --- a/apps/server/src/features/agents/diagnostics/__tests__/server-work-trace.test.ts +++ b/apps/server/src/features/agents/diagnostics/__tests__/server-work-trace.test.ts @@ -16,6 +16,9 @@ describe("ServerWorkTrace", () => { } trace.record("event-apply", "thread-overflow", "execution-d", 9, "toolResult"); trace.record("narrative-checkpoint", "thread-overflow", "execution-d", 7); + trace.record("narrative-prepare", "thread-overflow", "execution-d", 4); + trace.record("narrative-persist", "thread-overflow", "execution-d", 2); + trace.record("narrative-confirm", "thread-overflow", "execution-d", 1); trace.tick(1_240); expect(reports).toHaveLength(1); @@ -24,11 +27,16 @@ describe("ServerWorkTrace", () => { if (report?.kind !== "server-work-stall") return; expect(report.delayMs).toBe(220); expect(report.samples).toHaveLength(64); - expect(report.overflowCount).toBe(21); + expect(report.overflowCount).toBe(24); expect(report.eventApplyByType.textDelta).toEqual({ count: 40, totalMs: 480, maxMs: 12 }); expect(report.eventApplyByType.toolResult).toEqual({ count: 1, totalMs: 9, maxMs: 9 }); expect(report.eventApplyByType.other).toEqual({ count: 1, totalMs: 1, maxMs: 1 }); expect(report.narrativeCheckpoint).toEqual({ count: 1, totalMs: 7, maxMs: 7 }); + expect(report.narrativeCheckpointByStep).toEqual({ + "narrative-prepare": { count: 1, totalMs: 4, maxMs: 4 }, + "narrative-persist": { count: 1, totalMs: 2, maxMs: 2 }, + "narrative-confirm": { count: 1, totalMs: 1, maxMs: 1 }, + }); expect(report.samples[0]).toEqual({ phase: "event-apply", threadId: "thread-a", executionId: "execution-a", count: 40, totalMs: 480, maxMs: 12, @@ -52,6 +60,9 @@ describe("ServerWorkTrace", () => { types.forEach((type, index) => { trace.record("event-apply", `thread-${index}`, "execution-a", index + 1, eventApplyType(type)); trace.record("narrative-checkpoint", `thread-${index}`, "execution-a", index + 2); + trace.record("narrative-prepare", `thread-${index}`, "execution-a", index + 1); + trace.record("narrative-persist", `thread-${index}`, "execution-a", 1); + trace.record("narrative-confirm", `thread-${index}`, "execution-a", 1); }); trace.tick(1_200); const report = reports[0]; @@ -65,5 +76,10 @@ describe("ServerWorkTrace", () => { other: { count: 1, totalMs: 5, maxMs: 5 }, }); expect(report.narrativeCheckpoint).toEqual({ count: 5, totalMs: 20, maxMs: 6 }); + expect(report.narrativeCheckpointByStep).toEqual({ + "narrative-prepare": { count: 5, totalMs: 15, maxMs: 5 }, + "narrative-persist": { count: 5, totalMs: 5, maxMs: 1 }, + "narrative-confirm": { count: 5, totalMs: 5, maxMs: 1 }, + }); }); }); diff --git a/apps/server/src/features/agents/diagnostics/server-work-trace.ts b/apps/server/src/features/agents/diagnostics/server-work-trace.ts index 6a353e8ee..b2ffd912a 100644 --- a/apps/server/src/features/agents/diagnostics/server-work-trace.ts +++ b/apps/server/src/features/agents/diagnostics/server-work-trace.ts @@ -4,7 +4,9 @@ import { logger } from "@mcode/shared"; /** Fixed names keep trace output free of event bodies and provider text. */ export type ServerWorkPhase = | "provider-callback" | "worker-admission" | "worker-wait" | "mailbox-wait" - | "canonical-write" | "event-apply" | "narrative-checkpoint" | "finalization" | "publication" + | "canonical-write" | "event-apply" | "narrative-checkpoint" + | "narrative-prepare" | "narrative-persist" | "narrative-confirm" + | "finalization" | "publication" | "terminal-create" | "thread-create" | "agent-send"; /** Fixed event categories; unknown and future event types stay in other. */ @@ -29,6 +31,16 @@ interface EventApplyTotals { maxMs: number; } +type NarrativeCheckpointStep = "narrative-prepare" | "narrative-persist" | "narrative-confirm"; + +function emptyNarrativeStepTotals(): Record { + return { + "narrative-prepare": { count: 0, totalMs: 0, maxMs: 0 }, + "narrative-persist": { count: 0, totalMs: 0, maxMs: 0 }, + "narrative-confirm": { count: 0, totalMs: 0, maxMs: 0 }, + }; +} + function emptyEventApplyTotals(): Record { return { textDelta: { count: 0, totalMs: 0, maxMs: 0 }, @@ -57,6 +69,7 @@ export type ServerWorkTraceReport = { samples: WorkSample[]; eventApplyByType: Record; narrativeCheckpoint: EventApplyTotals; + narrativeCheckpointByStep: Record; overflowCount: number; } | { kind: "server-work-slow-operation"; @@ -78,6 +91,7 @@ export class ServerWorkTrace { private readonly samples = new Map(); private eventApplyByType = emptyEventApplyTotals(); private narrativeCheckpoint: EventApplyTotals = { count: 0, totalMs: 0, maxMs: 0 }; + private narrativeCheckpointByStep = emptyNarrativeStepTotals(); private overflowCount = 0; private lastTick = NodePerfHooks.performance.now(); private timer: ReturnType | undefined; @@ -99,6 +113,7 @@ export class ServerWorkTrace { this.samples.clear(); this.eventApplyByType = emptyEventApplyTotals(); this.narrativeCheckpoint = { count: 0, totalMs: 0, maxMs: 0 }; + this.narrativeCheckpointByStep = emptyNarrativeStepTotals(); this.overflowCount = 0; } @@ -125,6 +140,12 @@ export class ServerWorkTrace { this.narrativeCheckpoint.totalMs += durationMs; this.narrativeCheckpoint.maxMs = Math.max(this.narrativeCheckpoint.maxMs, durationMs); } + if (phase === "narrative-prepare" || phase === "narrative-persist" || phase === "narrative-confirm") { + const total = this.narrativeCheckpointByStep[phase]; + total.count += 1; + total.totalMs += durationMs; + total.maxMs = Math.max(total.maxMs, durationMs); + } const safeThreadId = safeId(threadId); const safeExecutionId = safeId(executionId); if (this.isSlowOperation(phase, durationMs)) { @@ -162,11 +183,12 @@ export class ServerWorkTrace { this.lastTick = now; const delayMs = Math.max(0, windowMs - INTERVAL_MS); if (delayMs >= REPORT_DELAY_MS) { - this.emit({ kind: "server-work-stall", at: Date.now(), delayMs, windowMs, samples: [...this.samples.values()], eventApplyByType: this.eventApplyByType, narrativeCheckpoint: this.narrativeCheckpoint, overflowCount: this.overflowCount }); + this.emit({ kind: "server-work-stall", at: Date.now(), delayMs, windowMs, samples: [...this.samples.values()], eventApplyByType: this.eventApplyByType, narrativeCheckpoint: this.narrativeCheckpoint, narrativeCheckpointByStep: this.narrativeCheckpointByStep, overflowCount: this.overflowCount }); } this.samples.clear(); this.eventApplyByType = emptyEventApplyTotals(); this.narrativeCheckpoint = { count: 0, totalMs: 0, maxMs: 0 }; + this.narrativeCheckpointByStep = emptyNarrativeStepTotals(); this.overflowCount = 0; } } diff --git a/apps/server/src/features/agents/turns/parent-narrative-recovery-coordinator.ts b/apps/server/src/features/agents/turns/parent-narrative-recovery-coordinator.ts index a8df4d111..2b0c2b6bf 100644 --- a/apps/server/src/features/agents/turns/parent-narrative-recovery-coordinator.ts +++ b/apps/server/src/features/agents/turns/parent-narrative-recovery-coordinator.ts @@ -5,6 +5,7 @@ import { } from "@mcode/contracts"; import type { ParentTurnDurability } from "./parent-turn-durability.js"; import type { NarrativeStore } from "../conversation/narrative/narrative-store.js"; +import { serverWorkTrace } from "../diagnostics/server-work-trace.js"; interface ParentNarrativeRecoveryCheckpoint { persist(): void; @@ -22,10 +23,20 @@ export class ParentNarrativeRecoveryCoordinator { /** Commit only the semantic records changed by this accepted provider event. */ checkpoint(event: AgentEvent): void { - const checkpoint = this.prepareCheckpoint(event); + if (!serverWorkTrace) { + const checkpoint = this.prepareCheckpoint(event); + if (!checkpoint) return; + checkpoint.persist(); + checkpoint.confirm(); + return; + } + const checkpoint = serverWorkTrace.measure("narrative-prepare", event.threadId, + event.turnExecutionId, () => this.prepareCheckpoint(event)); if (!checkpoint) return; - checkpoint.persist(); - checkpoint.confirm(); + serverWorkTrace.measure("narrative-persist", event.threadId, + event.turnExecutionId, () => checkpoint.persist()); + serverWorkTrace.measure("narrative-confirm", event.threadId, + event.turnExecutionId, () => checkpoint.confirm()); } /** Prepare a recovery commit whose dedupe state advances only after its transaction commits. */ From 3c467e9e98e2b7be52ac9a5a592e089740a2124b Mon Sep 17 00:00:00 2001 From: "(CJ) Chukwudi Nwobodo" <142016413+chuks-qua@users.noreply.github.com> Date: Thu, 24 Sep 2026 19:36:48 +0100 Subject: [PATCH 007/136] feat(server): write parent narrative recovery in SQLite worker --- .../canonical-agent-writer-client.test.ts | 79 +++++++++++++++++++ .../canonical-agent-writer-client.ts | 24 +++++- .../canonical-agent-writer-protocol.ts | 14 +++- .../canonical-agent-writer.worker.ts | 11 +++ 4 files changed, 126 insertions(+), 2 deletions(-) diff --git a/apps/server/src/features/agents/canonical/__tests__/canonical-agent-writer-client.test.ts b/apps/server/src/features/agents/canonical/__tests__/canonical-agent-writer-client.test.ts index 21300173d..ecfbced75 100644 --- a/apps/server/src/features/agents/canonical/__tests__/canonical-agent-writer-client.test.ts +++ b/apps/server/src/features/agents/canonical/__tests__/canonical-agent-writer-client.test.ts @@ -4,6 +4,7 @@ import * as NodeFSPromises from "node:fs/promises"; import * as NodeOS from "node:os"; import * as NodePath from "node:path"; import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import type { ParentNarrativeRecoveryItem } from "@mcode/contracts"; import { openDatabase } from "../../../../runtime/persistence/sqlite/database.js"; import type { CanonicalAgentEventDraft } from "../canonical-agent-boundary.js"; import { CanonicalAgentWriterClient } from "../canonical-agent-writer-client.js"; @@ -70,6 +71,37 @@ function events(): CanonicalAgentEventDraft[] { ]; } +function recoveryToolCall(): ParentNarrativeRecoveryItem { + return { + kind: "toolCall", + record: { + id: "writer-recovery-tool", + message_id: "", + parent_tool_call_id: null, + tool_name: "Read", + display_name: null, + provider_agent_key: null, + subagent_identity_key: null, + subagent_provider_name: null, + subagent_prompt: null, + subagent_type: null, + subagent_agent_id: null, + subagent_duration_ms: null, + model: null, + reasoning_effort: null, + input_summary: "a file", + output_summary: "read", + output_total_bytes: null, + output_artifact_path: null, + exit_code: null, + status: "completed", + started_at: NOW, + completed_at: NOW, + sort_order: 0, + }, + }; +} + describe("canonical SQLite writer", () => { let tempDir: string; let dbPath: string; @@ -120,6 +152,53 @@ describe("canonical SQLite writer", () => { expect(db.prepare("SELECT COUNT(*) AS count FROM canonical_agent_ingest_checkpoints").get()).toEqual({ count: 0 }); }); + it("acknowledges structured recovery after persistence and converges on replay", async () => { + writer = new CanonicalAgentWriterClient(dbPath); + await writer.commit("narrative-start", { + threadId: THREAD_ID, turnId: TURN_ID, executionId: EXECUTION_ID, phase: "running", events: events(), + }); + const recovery = { executionId: EXECUTION_ID, items: [recoveryToolCall()] }; + expect(await writer.recordParentNarrativeRecovery("narrative-record", recovery)).toEqual({ recorded: true }); + const row = db.prepare("SELECT payload_json FROM canonical_agent_items WHERE id = ?") + .get("toolCall:writer-recovery-tool") as { payload_json: string }; + expect(JSON.parse(row.payload_json)).toMatchObject({ + projection: "narrativeRecovery", + narrative: recoveryToolCall(), + }); + expect(await writer.recordParentNarrativeRecovery("narrative-record", recovery)).toEqual({ recorded: true }); + expect(db.prepare("SELECT COUNT(*) AS count FROM canonical_agent_items WHERE id = ?") + .get("toolCall:writer-recovery-tool")).toEqual({ count: 1 }); + + expect(await writer.recordParentNarrativeRecovery("narrative-discard", { + executionId: EXECUTION_ID, + items: [], + discardedItemIds: ["toolCall:writer-recovery-tool"], + })).toEqual({ recorded: true }); + expect(db.prepare("SELECT COUNT(*) AS count FROM canonical_agent_items WHERE id = ?") + .get("toolCall:writer-recovery-tool")).toEqual({ count: 0 }); + }); + + it("rejects a failed recovery write without a durability acknowledgement", async () => { + writer = new CanonicalAgentWriterClient(dbPath); + await writer.commit("narrative-start", { + threadId: THREAD_ID, turnId: TURN_ID, executionId: EXECUTION_ID, phase: "running", events: events(), + }); + db.run("DROP TABLE canonical_agent_items"); + await expect(writer.recordParentNarrativeRecovery("narrative-failed", { + executionId: EXECUTION_ID, + items: [recoveryToolCall()], + })).rejects.toThrow("Canonical writer write-failed"); + }); + + it("reports a missing execution without claiming recovery was recorded", async () => { + writer = new CanonicalAgentWriterClient(dbPath); + expect(await writer.recordParentNarrativeRecovery("missing-recovery", { + executionId: "missing-execution", + items: [recoveryToolCall()], + })).toEqual({ recorded: false }); + expect(db.prepare("SELECT COUNT(*) AS count FROM canonical_agent_items").get()).toEqual({ count: 0 }); + }); + it("rejects an unmigrated path without creating a second database", async () => { const missingPath = NodePath.join(tempDir, "missing.sqlite"); writer = new CanonicalAgentWriterClient(missingPath); diff --git a/apps/server/src/features/agents/canonical/canonical-agent-writer-client.ts b/apps/server/src/features/agents/canonical/canonical-agent-writer-client.ts index dbcc6e2e3..75e2ab0e9 100644 --- a/apps/server/src/features/agents/canonical/canonical-agent-writer-client.ts +++ b/apps/server/src/features/agents/canonical/canonical-agent-writer-client.ts @@ -1,11 +1,13 @@ import * as NodePath from "node:path"; import * as NodeCrypto from "node:crypto"; import type { + CanonicalParentNarrativeRecoveryReceipt, CanonicalProviderWriteInput, CanonicalProviderWriteReceipt, CanonicalWriterRequest, CanonicalWriterResponse, } from "./canonical-agent-writer-protocol.js"; +import type { ParentNarrativeRecoveryCommitInput } from "./canonical-agent-boundary.js"; const MAX_PENDING_WRITES = 64; const WRITER_EXECUTION_ID = "writer:init"; @@ -69,6 +71,26 @@ export class CanonicalAgentWriterClient { return response.receipt; } + /** Resolves after recovery writes finish. A lost reply requires a durable-state check before retrying discards. */ + async recordParentNarrativeRecovery( + operationId: string, + input: ParentNarrativeRecoveryCommitInput, + ): Promise { + if (!operationId || !input.executionId) throw new Error("Canonical writer operation and execution IDs are required"); + await this.ready; + const response = await this.send({ + kind: "record-parent-narrative-recovery", + requestId: NodeCrypto.randomUUID(), + operationId, + executionId: input.executionId, + input, + }); + if (response.kind !== "parent-narrative-recovery-recorded") { + throw new Error("Canonical writer returned an unexpected response"); + } + return response.receipt; + } + /** Closes the database after accepted writes settle, then releases the worker. */ async close(): Promise { try { @@ -91,7 +113,7 @@ export class CanonicalAgentWriterClient { private send(request: CanonicalWriterRequest): Promise { if (this.failure) return Promise.reject(this.failure); - if (request.kind === "commit" && this.pending.size >= MAX_PENDING_WRITES) { + if (request.kind !== "open" && request.kind !== "close" && this.pending.size >= MAX_PENDING_WRITES) { return Promise.reject(new Error("Canonical writer admission is full")); } return new Promise((resolve, reject) => { diff --git a/apps/server/src/features/agents/canonical/canonical-agent-writer-protocol.ts b/apps/server/src/features/agents/canonical/canonical-agent-writer-protocol.ts index c92f61984..9208579ee 100644 --- a/apps/server/src/features/agents/canonical/canonical-agent-writer-protocol.ts +++ b/apps/server/src/features/agents/canonical/canonical-agent-writer-protocol.ts @@ -1,5 +1,10 @@ import type { CanonicalAgentEventEnvelope } from "@mcode/contracts"; -import type { CanonicalAgentCommitInput, CanonicalAgentCommitResult, CanonicalAgentEventDraft } from "./canonical-agent-boundary.js"; +import type { + CanonicalAgentCommitInput, + CanonicalAgentCommitResult, + CanonicalAgentEventDraft, + ParentNarrativeRecoveryCommitInput, +} from "./canonical-agent-boundary.js"; /** Cloneable provider batch accepted by the SQLite writer. Compatibility callbacks stay with their owner. */ export type CanonicalProviderWriteInput = Pick< @@ -17,6 +22,11 @@ export interface CanonicalProviderWriteReceipt { events: readonly CanonicalAgentEventEnvelope[]; } +/** Acknowledges completed recovery writes; false means the execution was not found. */ +export interface CanonicalParentNarrativeRecoveryReceipt { + recorded: boolean; +} + interface Correlation { requestId: string; operationId: string; @@ -26,10 +36,12 @@ interface Correlation { export type CanonicalWriterRequest = | (Correlation & { kind: "open"; dbPath: string }) | (Correlation & { kind: "commit"; input: CanonicalProviderWriteInput }) + | (Correlation & { kind: "record-parent-narrative-recovery"; input: ParentNarrativeRecoveryCommitInput }) | (Correlation & { kind: "close" }); export type CanonicalWriterResponse = | (Correlation & { kind: "opened" }) | (Correlation & { kind: "committed"; receipt: CanonicalProviderWriteReceipt }) + | (Correlation & { kind: "parent-narrative-recovery-recorded"; receipt: CanonicalParentNarrativeRecoveryReceipt }) | (Correlation & { kind: "closed" }) | (Correlation & { kind: "failed"; reason: "open-failed" | "write-failed" }); diff --git a/apps/server/src/features/agents/canonical/canonical-agent-writer.worker.ts b/apps/server/src/features/agents/canonical/canonical-agent-writer.worker.ts index 793dfa055..98bd8922a 100644 --- a/apps/server/src/features/agents/canonical/canonical-agent-writer.worker.ts +++ b/apps/server/src/features/agents/canonical/canonical-agent-writer.worker.ts @@ -70,6 +70,17 @@ function handle(request: CanonicalWriterRequest): CanonicalWriterResponse { } try { if (!boundary) throw new Error("Canonical writer has not opened its database"); + if (request.kind === "record-parent-narrative-recovery") { + if (request.input.executionId !== request.executionId) { + throw new Error("Canonical writer recovery execution mismatch"); + } + const recorded = boundary.recordParentNarrativeRecovery(request.input); + return { + ...correlation, + kind: "parent-narrative-recovery-recorded", + receipt: { recorded }, + }; + } assertRouting(request.input, request.executionId); const result = boundary.commit(request.input); const { outcome, conversationRevision, rosterRevision, acceptedThrough, durableThrough, events } = result; From 33ad5f1aa178c1fbf6e4f66872eb5f7b336085c0 Mon Sep 17 00:00:00 2001 From: "(CJ) Chukwudi Nwobodo" <142016413+chuks-qua@users.noreply.github.com> Date: Thu, 24 Sep 2026 19:41:53 +0100 Subject: [PATCH 008/136] feat(server): atomically classify parent narrative recovery --- .../canonical-agent-writer-client.test.ts | 36 ++++++++++++++++++ .../canonical-agent-writer-client.ts | 21 ++++++++++ .../canonical-agent-writer-protocol.ts | 8 ++++ .../canonical-agent-writer.worker.ts | 38 +++++++++++++++++++ 4 files changed, 103 insertions(+) diff --git a/apps/server/src/features/agents/canonical/__tests__/canonical-agent-writer-client.test.ts b/apps/server/src/features/agents/canonical/__tests__/canonical-agent-writer-client.test.ts index ecfbced75..8fb23edce 100644 --- a/apps/server/src/features/agents/canonical/__tests__/canonical-agent-writer-client.test.ts +++ b/apps/server/src/features/agents/canonical/__tests__/canonical-agent-writer-client.test.ts @@ -8,6 +8,7 @@ import type { ParentNarrativeRecoveryItem } from "@mcode/contracts"; import { openDatabase } from "../../../../runtime/persistence/sqlite/database.js"; import type { CanonicalAgentEventDraft } from "../canonical-agent-boundary.js"; import { CanonicalAgentWriterClient } from "../canonical-agent-writer-client.js"; +import { ParentAssistantTextCheckpointService } from "../../turns/parent-assistant-text-checkpoint-service.js"; const THREAD_ID = "writer-thread"; const TURN_ID = "writer-turn"; @@ -199,6 +200,41 @@ describe("canonical SQLite writer", () => { expect(db.prepare("SELECT COUNT(*) AS count FROM canonical_agent_items").get()).toEqual({ count: 0 }); }); + it("commits recovery and assistant-text reset together", async () => { + writer = new CanonicalAgentWriterClient(dbPath); + await writer.commit("classification-start", { + threadId: THREAD_ID, turnId: TURN_ID, executionId: EXECUTION_ID, phase: "running", events: events(), + }); + const checkpoints = new ParentAssistantTextCheckpointService(db); + checkpoints.appendChunk([{ + executionId: EXECUTION_ID, threadId: THREAD_ID, turnId: TURN_ID, sequence: 1, text: "provisional text", + }]); + const input = { executionId: EXECUTION_ID, items: [recoveryToolCall()] }; + expect(await writer.classifyParentNarrativeRecovery("classification-1", input)) + .toEqual({ recorded: true, reset: true }); + expect(db.prepare("SELECT COUNT(*) AS count FROM parent_assistant_text_checkpoints WHERE execution_id = ?") + .get(EXECUTION_ID)).toEqual({ count: 0 }); + expect(db.prepare("SELECT COUNT(*) AS count FROM canonical_agent_items WHERE id = ?") + .get("toolCall:writer-recovery-tool")).toEqual({ count: 1 }); + await expect(writer.classifyParentNarrativeRecovery("classification-1", input)) + .rejects.toThrow("Canonical writer write-failed"); + expect(db.prepare("SELECT COUNT(*) AS count FROM canonical_agent_items WHERE id = ?") + .get("toolCall:writer-recovery-tool")).toEqual({ count: 1 }); + }); + + it("rolls back recovery when the assistant-text checkpoint is missing", async () => { + writer = new CanonicalAgentWriterClient(dbPath); + await writer.commit("classification-start", { + threadId: THREAD_ID, turnId: TURN_ID, executionId: EXECUTION_ID, phase: "running", events: events(), + }); + await expect(writer.classifyParentNarrativeRecovery("classification-missing", { + executionId: EXECUTION_ID, + items: [recoveryToolCall()], + })).rejects.toThrow("Canonical writer write-failed"); + expect(db.prepare("SELECT COUNT(*) AS count FROM canonical_agent_items WHERE id = ?") + .get("toolCall:writer-recovery-tool")).toEqual({ count: 0 }); + }); + it("rejects an unmigrated path without creating a second database", async () => { const missingPath = NodePath.join(tempDir, "missing.sqlite"); writer = new CanonicalAgentWriterClient(missingPath); diff --git a/apps/server/src/features/agents/canonical/canonical-agent-writer-client.ts b/apps/server/src/features/agents/canonical/canonical-agent-writer-client.ts index 75e2ab0e9..323a0f1f4 100644 --- a/apps/server/src/features/agents/canonical/canonical-agent-writer-client.ts +++ b/apps/server/src/features/agents/canonical/canonical-agent-writer-client.ts @@ -1,6 +1,7 @@ import * as NodePath from "node:path"; import * as NodeCrypto from "node:crypto"; import type { + CanonicalParentNarrativeClassificationReceipt, CanonicalParentNarrativeRecoveryReceipt, CanonicalProviderWriteInput, CanonicalProviderWriteReceipt, @@ -91,6 +92,26 @@ export class CanonicalAgentWriterClient { return response.receipt; } + /** Atomically persists recovery and resets provisional assistant text; the caller retires its journal after receipt. */ + async classifyParentNarrativeRecovery( + operationId: string, + input: ParentNarrativeRecoveryCommitInput, + ): Promise { + if (!operationId || !input.executionId) throw new Error("Canonical writer operation and execution IDs are required"); + await this.ready; + const response = await this.send({ + kind: "classify-parent-narrative-recovery", + requestId: NodeCrypto.randomUUID(), + operationId, + executionId: input.executionId, + input, + }); + if (response.kind !== "parent-narrative-recovery-classified") { + throw new Error("Canonical writer returned an unexpected response"); + } + return response.receipt; + } + /** Closes the database after accepted writes settle, then releases the worker. */ async close(): Promise { try { diff --git a/apps/server/src/features/agents/canonical/canonical-agent-writer-protocol.ts b/apps/server/src/features/agents/canonical/canonical-agent-writer-protocol.ts index 9208579ee..ccf608cb6 100644 --- a/apps/server/src/features/agents/canonical/canonical-agent-writer-protocol.ts +++ b/apps/server/src/features/agents/canonical/canonical-agent-writer-protocol.ts @@ -27,6 +27,12 @@ export interface CanonicalParentNarrativeRecoveryReceipt { recorded: boolean; } +/** Both writes are acknowledged only after their shared transaction commits. */ +export interface CanonicalParentNarrativeClassificationReceipt { + recorded: true; + reset: true; +} + interface Correlation { requestId: string; operationId: string; @@ -37,11 +43,13 @@ export type CanonicalWriterRequest = | (Correlation & { kind: "open"; dbPath: string }) | (Correlation & { kind: "commit"; input: CanonicalProviderWriteInput }) | (Correlation & { kind: "record-parent-narrative-recovery"; input: ParentNarrativeRecoveryCommitInput }) + | (Correlation & { kind: "classify-parent-narrative-recovery"; input: ParentNarrativeRecoveryCommitInput }) | (Correlation & { kind: "close" }); export type CanonicalWriterResponse = | (Correlation & { kind: "opened" }) | (Correlation & { kind: "committed"; receipt: CanonicalProviderWriteReceipt }) | (Correlation & { kind: "parent-narrative-recovery-recorded"; receipt: CanonicalParentNarrativeRecoveryReceipt }) + | (Correlation & { kind: "parent-narrative-recovery-classified"; receipt: CanonicalParentNarrativeClassificationReceipt }) | (Correlation & { kind: "closed" }) | (Correlation & { kind: "failed"; reason: "open-failed" | "write-failed" }); diff --git a/apps/server/src/features/agents/canonical/canonical-agent-writer.worker.ts b/apps/server/src/features/agents/canonical/canonical-agent-writer.worker.ts index 98bd8922a..c256fbc9e 100644 --- a/apps/server/src/features/agents/canonical/canonical-agent-writer.worker.ts +++ b/apps/server/src/features/agents/canonical/canonical-agent-writer.worker.ts @@ -4,14 +4,18 @@ import * as NodeFS from "node:fs"; import * as NodePath from "node:path"; import { applySQLiteConnectionPolicy } from "../../../runtime/persistence/sqlite/sqlite-connection-policy.js"; import { CanonicalAgentBoundary } from "./canonical-agent-boundary.js"; +import { ParentAssistantTextCheckpointService } from "../turns/parent-assistant-text-checkpoint-service.js"; import type { + CanonicalParentNarrativeClassificationReceipt, CanonicalProviderWriteInput, CanonicalWriterRequest, CanonicalWriterResponse, } from "./canonical-agent-writer-protocol.js"; +import type { ParentNarrativeRecoveryCommitInput } from "./canonical-agent-boundary.js"; let db: Database | undefined; let boundary: CanonicalAgentBoundary | undefined; +let assistantTextCheckpoints: ParentAssistantTextCheckpointService | undefined; function openDatabase(dbPath: string): void { if (boundary || !NodePath.isAbsolute(dbPath) || !NodeFS.existsSync(dbPath)) { @@ -21,13 +25,32 @@ function openDatabase(dbPath: string): void { try { applySQLiteConnectionPolicy(connection, true); boundary = new CanonicalAgentBoundary(connection, () => {}); + assistantTextCheckpoints = new ParentAssistantTextCheckpointService(connection); db = connection; } catch (error) { + boundary = undefined; + assistantTextCheckpoints = undefined; connection.close(true); throw error; } } +function classifyParentNarrativeRecovery( + input: ParentNarrativeRecoveryCommitInput, +): CanonicalParentNarrativeClassificationReceipt { + const connection = db; + const canonical = boundary; + const checkpoints = assistantTextCheckpoints; + if (!connection || !canonical || !checkpoints) throw new Error("Canonical writer has not opened its database"); + return connection.transaction(() => { + const recorded = canonical.recordParentNarrativeRecovery(input); + if (!recorded) throw new Error("Canonical parent turn was not found"); + const reset = checkpoints.resetInTransaction(input.executionId); + if (!reset) throw new Error("Provisional assistant text checkpoint was not reset"); + return { recorded, reset }; + })(); +} + function assertRouting(input: CanonicalProviderWriteInput, executionId: string): void { if (input.executionId !== executionId || !input.threadId || !input.turnId || !input.phase || !Array.isArray(input.events) || input.events.length === 0) { @@ -64,12 +87,27 @@ function handle(request: CanonicalWriterRequest): CanonicalWriterResponse { } if (request.kind === "close") { boundary = undefined; + assistantTextCheckpoints = undefined; db?.close(true); db = undefined; return { ...correlation, kind: "closed" }; } + return handleWrite(request, correlation); +} + +function handleWrite( + request: Extract, + correlation: Pick, +): CanonicalWriterResponse { try { if (!boundary) throw new Error("Canonical writer has not opened its database"); + if (request.kind === "classify-parent-narrative-recovery") { + if (request.input.executionId !== request.executionId) { + throw new Error("Canonical writer classification execution mismatch"); + } + const receipt = classifyParentNarrativeRecovery(request.input); + return { ...correlation, kind: "parent-narrative-recovery-classified", receipt }; + } if (request.kind === "record-parent-narrative-recovery") { if (request.input.executionId !== request.executionId) { throw new Error("Canonical writer recovery execution mismatch"); From ff0b0ec348e157a32fcfbb6e63adb84116321f83 Mon Sep 17 00:00:00 2001 From: "(CJ) Chukwudi Nwobodo" <142016413+chuks-qua@users.noreply.github.com> Date: Thu, 24 Sep 2026 19:42:48 +0100 Subject: [PATCH 009/136] feat(server): fence async turn event application before finalization --- .../__tests__/turn-event-pipeline.test.ts | 124 ++++++++++++++++++ .../agents/turns/turn-event-pipeline.ts | 106 +++++++++++++-- 2 files changed, 218 insertions(+), 12 deletions(-) diff --git a/apps/server/src/features/agents/turns/__tests__/turn-event-pipeline.test.ts b/apps/server/src/features/agents/turns/__tests__/turn-event-pipeline.test.ts index b49e0718f..5544dccdf 100644 --- a/apps/server/src/features/agents/turns/__tests__/turn-event-pipeline.test.ts +++ b/apps/server/src/features/agents/turns/__tests__/turn-event-pipeline.test.ts @@ -129,6 +129,130 @@ describe("TurnEventPipeline", () => { expect(finalize).toHaveBeenCalledOnce(); }); + it("waits for an asynchronous application without holding another thread", async () => { + let acknowledge!: (accepted: boolean) => void; + const durable = new Promise((resolve) => { acknowledge = resolve; }); + const applied: string[] = []; + const { pipeline, finalize } = createPipeline((_input, event) => { + const delta = (event as Extract).delta; + applied.push(delta); + return delta === "first" ? durable : true; + }); + + pipeline.handleProviderEvent(textDelta("first")); + pipeline.handleProviderEvent(textDelta("second")); + pipeline.handleProviderEvent(textDelta("other", "thread-2")); + const finalization = pipeline.finalizeTurn({ + threadId: "thread-1", + executionId: EXECUTION_ID, + outcome: "completed", + source: "provider", + }); + + expect(applied).toEqual(["first", "other"]); + expect(finalize).not.toHaveBeenCalled(); + acknowledge(true); + await expect(finalization).resolves.toBe(true); + expect(applied).toEqual(["first", "other", "second"]); + }); + + it("waits for an in-flight acknowledgement before finalizing Stop", async () => { + let acknowledge!: (accepted: boolean) => void; + const durable = new Promise((resolve) => { acknowledge = resolve; }); + const apply = vi.fn(() => durable); + const { pipeline, finalize } = createPipeline(apply); + + pipeline.handleProviderEvent(textDelta("in flight")); + pipeline.handleProviderEvent(textDelta("discarded")); + const finalization = pipeline.finalizeTurn({ + threadId: "thread-1", + executionId: EXECUTION_ID, + outcome: "cancelled", + source: "user-stop", + }); + + expect(finalize).not.toHaveBeenCalled(); + acknowledge(true); + await expect(finalization).resolves.toBe(true); + expect(apply).toHaveBeenCalledOnce(); + expect(finalize).toHaveBeenCalledOnce(); + }); + + it("keeps an asynchronous blocked event at the queue head until resume", async () => { + let acknowledge!: (accepted: boolean) => void; + const durable = new Promise((resolve) => { acknowledge = resolve; }); + const applied: string[] = []; + let ready = false; + const { pipeline } = createPipeline((_input, event) => { + const delta = (event as Extract).delta; + applied.push(delta); + return delta === "first" && !ready ? durable : true; + }); + + pipeline.handleProviderEvent(textDelta("first")); + pipeline.handleProviderEvent(textDelta("second")); + acknowledge(false); + await Promise.resolve(); + expect(applied).toEqual(["first"]); + + ready = true; + pipeline.resume("thread-1"); + expect(applied).toEqual(["first", "first", "second"]); + }); + + it("does not let a discarded acknowledgement consume the next execution's event", async () => { + const firstExecution = "00000000-0000-4000-8000-000000000010"; + const nextExecution = "00000000-0000-4000-8000-000000000011"; + let acknowledge!: (accepted: boolean) => void; + const durable = new Promise((resolve) => { acknowledge = resolve; }); + const applied: string[] = []; + const { pipeline } = createPipeline((_input, event) => { + applied.push(`${event.turnExecutionId}:${event.type}`); + return event.type === AgentEventType.TextDelta && event.turnExecutionId === firstExecution + ? durable + : true; + }); + + pipeline.handleProviderEvent(turnStarted(firstExecution)); + pipeline.handleProviderEvent({ + ...textDelta("old"), + event: { ...textDelta("old").event, turnExecutionId: firstExecution }, + }); + pipeline.discard("thread-1", firstExecution); + pipeline.handleProviderEvent(turnStarted(nextExecution)); + expect(applied).toEqual([`${firstExecution}:turnStarted`, `${firstExecution}:textDelta`]); + + acknowledge(true); + await vi.waitFor(() => { + expect(applied).toEqual([ + `${firstExecution}:turnStarted`, + `${firstExecution}:textDelta`, + `${nextExecution}:turnStarted`, + ]); + }); + }); + + it("does not finalize after an asynchronous application fails", async () => { + let fail!: (error: Error) => void; + const durable = new Promise((_resolve, reject) => { fail = reject; }); + const apply = vi.fn(() => durable); + const { pipeline, finalize } = createPipeline(apply); + + pipeline.handleProviderEvent(textDelta("write failure")); + const finalization = pipeline.finalizeTurn({ + threadId: "thread-1", + executionId: EXECUTION_ID, + outcome: "completed", + source: "provider", + }); + + fail(new Error("checkpoint failed")); + await expect(finalization).rejects.toThrow("checkpoint failed"); + pipeline.resume("thread-1"); + expect(apply).toHaveBeenCalledOnce(); + expect(finalize).not.toHaveBeenCalled(); + }); + it("holds finalization until its thread-affine ingress worker is idle", async () => { let releaseIngress!: () => void; const ingressIdle = new Promise((resolve) => { releaseIngress = resolve; }); diff --git a/apps/server/src/features/agents/turns/turn-event-pipeline.ts b/apps/server/src/features/agents/turns/turn-event-pipeline.ts index 07c874fa2..8e8c3cd35 100644 --- a/apps/server/src/features/agents/turns/turn-event-pipeline.ts +++ b/apps/server/src/features/agents/turns/turn-event-pipeline.ts @@ -40,7 +40,7 @@ export interface TurnLifecycleControl { /** Applies one ordered event to the existing focused turn collaborators. */ export interface TurnEventApplication { /** Apply a validated event after the pipeline admits it in per-turn order. */ - apply(input: ProviderEventIngressEvent, event: AgentEvent, publish: boolean): boolean; + apply(input: ProviderEventIngressEvent, event: AgentEvent, publish: boolean): boolean | Promise; /** Record a provider file mutation before its corresponding public event arrives. */ observeFileMutation(event: ProviderFileMutationStart): void; /** Abort one turn only when its completion cannot be compacted into the bounded queue. */ @@ -71,6 +71,8 @@ interface QueuedTurnEvent { export class TurnEventPipeline implements ProviderEventIngressConsumer { private readonly queues = new Map(); private readonly drainingThreads = new Set(); + private readonly inFlightApplications = new Map>(); + private readonly applicationErrors = new Map(); private readonly queuedBytesByThread = new Map(); private readonly finalizations = new Map>(); private readonly barriersByThread = new Map>(); @@ -79,7 +81,10 @@ export class TurnEventPipeline implements ProviderEventIngressConsumer { private readonly barrierGenerationByThread = new Map(); private readonly earlyFileEffects = new WeakSet(); private readonly fileBarrierDeferredEvents = new WeakSet(); - private readonly queueEmptyWaitersByThread = new Map void>>(); + private readonly queueEmptyWaitersByThread = new Map void; + reject: (error: unknown) => void; + }>>(); constructor( private readonly lifecycle: TurnLifecycleControl, @@ -115,18 +120,27 @@ export class TurnEventPipeline implements ProviderEventIngressConsumer { /** Materialize a terminal turn once, after every earlier event in its turn queue. */ finalizeTurn(command: FinalizeTurnCommand): Promise | null { + const inFlight = this.inFlightApplications.get(command.threadId); if (cancelsDeferredWork(command.source)) this.discard(command.threadId, command.executionId); const existing = this.finalizations.get(command.threadId); if (existing) return existing; - const pending = this.ingressFence + // A cancelled queue may still have one durable write in progress. + const afterIngress = () => this.ingressFence ? this.ingressFence.waitForThread(command.threadId).then(() => this.finalizeAfterIngress(command)) : this.finalizeAfterIngress(command); + const pending = inFlight ? inFlight.then(afterIngress) : afterIngress(); this.finalizations.set(command.threadId, pending); - void pending.finally(() => this.finalizations.delete(command.threadId)); + void pending.then( + () => this.finalizations.delete(command.threadId), + () => this.finalizations.delete(command.threadId), + ); return pending; } private finalizeAfterIngress(command: FinalizeTurnCommand): Promise { + if (this.applicationErrors.has(command.threadId)) { + return Promise.reject(this.applicationErrors.get(command.threadId)); + } this.drain(command.threadId); return this.isReadyForFinalization(command.threadId) ? this.startFinalization(command) @@ -146,7 +160,8 @@ export class TurnEventPipeline implements ProviderEventIngressConsumer { this.deferredExecutionByThread.delete(threadId); this.queues.delete(threadId); this.queuedBytesByThread.delete(threadId); - this.completeQueue(threadId); + this.applicationErrors.delete(threadId); + if (!this.inFlightApplications.has(threadId)) this.completeQueue(threadId); } /** Return whether the pipeline already attributed this deferred event's file effect. */ @@ -188,8 +203,12 @@ export class TurnEventPipeline implements ProviderEventIngressConsumer { } private drain(threadId: string): void { - if (this.drainingThreads.has(threadId)) return; + if (this.drainingThreads.has(threadId) || this.applicationErrors.has(threadId)) return; this.drainingThreads.add(threadId); + this.drainOwnedQueue(threadId); + } + + private drainOwnedQueue(threadId: string): void { try { const queue = this.queues.get(threadId); while (queue && queue.length > 0) { @@ -199,29 +218,92 @@ export class TurnEventPipeline implements ProviderEventIngressConsumer { this.decrementQueuedBytes(threadId, next.byteLength); continue; } - if (!this.application.apply(next.input, next.event, next.publish)) return; + const applied = this.application.apply(next.input, next.event, next.publish); + if (typeof applied !== "boolean") { + const inFlight = Promise.resolve(applied); + this.inFlightApplications.set(threadId, inFlight); + void inFlight.then( + (accepted) => this.completeAsyncApplication(threadId, queue, next, inFlight, accepted), + (error: unknown) => this.failAsyncApplication(threadId, queue, inFlight, error), + ); + return; + } + if (!applied) return; queue.shift(); this.decrementQueuedBytes(threadId, next.byteLength); } - if (queue?.length === 0) this.completeQueue(threadId); + if (!queue || queue.length === 0) this.completeQueue(threadId); } finally { + if (!this.inFlightApplications.has(threadId)) this.drainingThreads.delete(threadId); + } + } + + private completeAsyncApplication( + threadId: string, + queue: QueuedTurnEvent[], + next: QueuedTurnEvent, + inFlight: Promise, + accepted: boolean, + ): void { + if (this.inFlightApplications.get(threadId) !== inFlight) return; + this.inFlightApplications.delete(threadId); + const currentQueue = this.queues.get(threadId); + if (accepted && currentQueue === queue && queue[0] === next) { + queue.shift(); + this.decrementQueuedBytes(threadId, next.byteLength); + } + if (!accepted && currentQueue === queue) { this.drainingThreads.delete(threadId); + return; + } + try { + this.drainOwnedQueue(threadId); + } catch (error) { + this.failApplication(threadId, error); } } + private failAsyncApplication( + threadId: string, + queue: QueuedTurnEvent[], + inFlight: Promise, + error: unknown, + ): void { + if (this.inFlightApplications.get(threadId) !== inFlight) return; + this.inFlightApplications.delete(threadId); + if (this.queues.get(threadId) === queue) { + this.failApplication(threadId, error); + return; + } + try { + this.drainOwnedQueue(threadId); + } catch (nextError) { + this.failApplication(threadId, nextError); + } + } + + private failApplication(threadId: string, error: unknown): void { + this.drainingThreads.delete(threadId); + this.applicationErrors.set(threadId, error); + const waiters = this.queueEmptyWaitersByThread.get(threadId) ?? []; + this.queueEmptyWaitersByThread.delete(threadId); + for (const waiter of waiters) waiter.reject(error); + } + private waitForQueue(threadId: string): Promise { this.drain(threadId); + if (this.applicationErrors.has(threadId)) return Promise.reject(this.applicationErrors.get(threadId)); if (!this.queues.has(threadId) && !this.drainingThreads.has(threadId)) return Promise.resolve(); - return new Promise((resolve) => { + return new Promise((resolve, reject) => { const waiters = this.queueEmptyWaitersByThread.get(threadId) ?? []; - waiters.push(resolve); + waiters.push({ resolve, reject }); this.queueEmptyWaitersByThread.set(threadId, waiters); }); } private isReadyForFinalization(threadId: string): boolean { const queuedEvents = this.queues.get(threadId)?.length ?? 0; - return queuedEvents === 0 || (queuedEvents === 1 && this.drainingThreads.has(threadId)); + return queuedEvents === 0 && !this.inFlightApplications.has(threadId); } private startFinalization(command: FinalizeTurnCommand): Promise { @@ -247,7 +329,7 @@ export class TurnEventPipeline implements ProviderEventIngressConsumer { this.queues.delete(threadId); const waiters = this.queueEmptyWaitersByThread.get(threadId) ?? []; this.queueEmptyWaitersByThread.delete(threadId); - for (const resolve of waiters) resolve(); + for (const waiter of waiters) waiter.resolve(); } private deferBehindFileFinalization(next: QueuedTurnEvent): boolean { From cd79780a362ceaa044c796d5bdf6896c52689aa1 Mon Sep 17 00:00:00 2001 From: "(CJ) Chukwudi Nwobodo" <142016413+chuks-qua@users.noreply.github.com> Date: Thu, 24 Sep 2026 19:48:30 +0100 Subject: [PATCH 010/136] feat(server): route canonical provider commits through SQLite writer --- .../application/bootstrap/server-bootstrap.ts | 4 ++++ .../provider-composition-container.test.ts | 2 ++ .../__tests__/provider-host-ports.test.ts | 11 +++++++--- .../composition/provider-host-ports.ts | 21 ++++++++++++++++--- .../composition/register-providers.ts | 10 +++++++-- 5 files changed, 40 insertions(+), 8 deletions(-) diff --git a/apps/server/src/application/bootstrap/server-bootstrap.ts b/apps/server/src/application/bootstrap/server-bootstrap.ts index 5975addb5..9b5a1681a 100644 --- a/apps/server/src/application/bootstrap/server-bootstrap.ts +++ b/apps/server/src/application/bootstrap/server-bootstrap.ts @@ -70,6 +70,7 @@ import { startAgentOrchestration, } from "../../features/agents"; import { AgentEventPublicationRegistry } from "../../features/agents/orchestration/agent-event-publication-registry.js"; +import { CanonicalAgentWriterClient } from "../../features/agents/canonical/canonical-agent-writer-client.js"; import { AgentEventPublicationRuntimePort, AgentReliabilityPort, @@ -342,6 +343,7 @@ const messageRepo = container.resolve(MessageRepo); const threadRepo = container.resolve(ThreadRepo); const providerRegistry = container.resolve(ProviderRegistry); const providerEventIngress = container.resolve(ProviderEventIngress); +const canonicalWriter = container.resolve(CanonicalAgentWriterClient); const cursorProvider = container.resolve("CursorProvider"); const providerAvailability = container.resolve(ProviderAvailabilityService); const toolCallRecordRepo = container.resolve(ToolCallRecordRepo); @@ -845,6 +847,7 @@ async function bootstrapServer(): Promise { }); } await canonicalSink.materializeConversationDisplay(); + await canonicalWriter.whenReady(); recordStartupCheckpoint("conversation display materialization completed"); interruptThreadStartupsAtStartup(); @@ -951,6 +954,7 @@ async function shutdown(): Promise { await captureCleanupFailure(() => providerRegistry.shutdown()); shutdownCoordinator.setPhase("shutdown provider event workers"); providerEventIngress.shutdown(); + await captureCleanupFailure(() => canonicalWriter.close()); browserAutomationBroker.shutdown(); browserAutomationSessionLease.shutdown(); diff --git a/apps/server/src/features/providers/composition/__tests__/provider-composition-container.test.ts b/apps/server/src/features/providers/composition/__tests__/provider-composition-container.test.ts index 05f96508c..405c47486 100644 --- a/apps/server/src/features/providers/composition/__tests__/provider-composition-container.test.ts +++ b/apps/server/src/features/providers/composition/__tests__/provider-composition-container.test.ts @@ -13,6 +13,7 @@ import type { ProviderHostPorts } from "@mcode/providers"; import { setupContainer } from "../../../../application/composition/container.js"; import { CanonicalAgentBoundary } from "../../../agents/canonical/canonical-agent-boundary.js"; +import { CanonicalAgentWriterClient } from "../../../agents/canonical/canonical-agent-writer-client.js"; import { MessageRepo } from "../../../agents/conversation/persistence/message-repo.js"; import { ProviderRegistry } from "../provider-registry.js"; import { SettingsService } from "../../../settings/settings-service.js"; @@ -91,6 +92,7 @@ describe("provider composition container", () => { vi.restoreAllMocks(); await container.resolve(ProviderRegistry).shutdown(); container.resolve(ProviderEventIngress).shutdown(); + await container.resolve(CanonicalAgentWriterClient).close(); container.resolve(SettingsService).dispose(); database?.close(true); database = undefined; diff --git a/apps/server/src/features/providers/composition/__tests__/provider-host-ports.test.ts b/apps/server/src/features/providers/composition/__tests__/provider-host-ports.test.ts index 4e94cef32..753f425a7 100644 --- a/apps/server/src/features/providers/composition/__tests__/provider-host-ports.test.ts +++ b/apps/server/src/features/providers/composition/__tests__/provider-host-ports.test.ts @@ -54,6 +54,7 @@ describe("createProviderHostPorts", () => { threadControl: {}, grants: {}, events: {}, + publishCanonicalEvents: vi.fn(), ingress: {}, } as never); @@ -70,7 +71,7 @@ describe("createProviderHostPorts", () => { it("hands a committed canonical batch directly to ingress after durable acceptance", async () => { const events = [committedRuntimeEnvelope()]; const deliveryOrder: string[] = []; - const commit = vi.fn(() => { + const commit = vi.fn(async () => { deliveryOrder.push("commit"); return { outcome: "committed" as const, @@ -81,6 +82,7 @@ describe("createProviderHostPorts", () => { events, }; }); + const publishCanonicalEvents = vi.fn(() => deliveryOrder.push("publication")); const acceptCommitted = vi.fn(() => deliveryOrder.push("ingress")); const ports = createProviderHostPorts({ runtime: { platform: "linux", architecture: "x64", nodeAbi: "127" }, @@ -90,6 +92,7 @@ describe("createProviderHostPorts", () => { threadControl: {}, grants: {}, events: { commit }, + publishCanonicalEvents, ingress: { acceptCommitted }, } as never); const batch = { threadId: "thread-1", turnId: "turn-1", executionId: EXECUTION_ID, phase: "streaming", events: [] }; @@ -105,9 +108,10 @@ describe("createProviderHostPorts", () => { }, delivery: { ingress: "queued" }, }); - expect(commit).toHaveBeenCalledWith({ ...batch, nativeCursor: undefined }); + expect(commit).toHaveBeenCalledWith(expect.any(String), { ...batch, nativeCursor: undefined }); + expect(publishCanonicalEvents).toHaveBeenCalledWith(events); expect(acceptCommitted).toHaveBeenCalledWith(events); - expect(deliveryOrder).toEqual(["commit", "ingress"]); + expect(deliveryOrder).toEqual(["commit", "publication", "ingress"]); }); it("does not hand duplicate or failed commits to ingress", async () => { @@ -132,6 +136,7 @@ describe("createProviderHostPorts", () => { threadControl: {}, grants: {}, events: { commit }, + publishCanonicalEvents: vi.fn(), ingress: { acceptCommitted }, } as never); const batch = { threadId: "thread-1", turnId: "turn-1", executionId: EXECUTION_ID, phase: "streaming", events: [] }; diff --git a/apps/server/src/features/providers/composition/provider-host-ports.ts b/apps/server/src/features/providers/composition/provider-host-ports.ts index 890f578d9..179f6717e 100644 --- a/apps/server/src/features/providers/composition/provider-host-ports.ts +++ b/apps/server/src/features/providers/composition/provider-host-ports.ts @@ -1,9 +1,12 @@ import type { ProviderHostPorts } from "@mcode/providers"; +import * as NodeCrypto from "node:crypto"; import type { HostRuntime } from "@mcode/shared/node/host-runtime"; +import { logger } from "@mcode/shared"; import type { JobObject } from "../../../runtime/process/containment/job-object.js"; import type { EnvService } from "../../../runtime/environment/env-service.js"; import type { ScopedPreGrantService } from "../../agents/permissions/scoped-pre-grant.js"; -import type { CanonicalAgentBoundary } from "../../agents/index.js"; +import type { CanonicalAgentEventPublisher } from "../../agents/canonical/canonical-agent-boundary.js"; +import type { CanonicalAgentWriterClient } from "../../agents/canonical/canonical-agent-writer-client.js"; import type { BrowserAutomationSessionLease } from "../../browser-automation/index.js"; import type { InternalThreadControlMcpRuntime } from "../../thread-control/index.js"; import { killProcessTree } from "../../../runtime/process/containment/process-kill.js"; @@ -17,7 +20,8 @@ export interface ProviderHostPortDependencies { browser: BrowserAutomationSessionLease; threadControl: InternalThreadControlMcpRuntime; grants: ScopedPreGrantService; - events: CanonicalAgentBoundary; + events: Pick; + publishCanonicalEvents: CanonicalAgentEventPublisher; ingress: ProviderEventIngress; } @@ -83,7 +87,10 @@ export function createProviderHostPorts( }, events: { submit: async (batch) => { - const result = dependencies.events.commit({ + const operationId = NodeCrypto.createHash("sha256") + .update(JSON.stringify([batch.executionId, batch.events.map((event) => event.eventId)])) + .digest("hex"); + const result = await dependencies.events.commit(operationId, { threadId: batch.threadId, turnId: batch.turnId, executionId: batch.executionId, @@ -92,6 +99,14 @@ export function createProviderHostPorts( events: batch.events, }); if (result.outcome === "committed" && result.events.length > 0) { + try { + dependencies.publishCanonicalEvents(result.events); + } catch { + logger.warn("Canonical provider publication deferred after durable commit", { + threadId: batch.threadId, + executionId: batch.executionId, + }); + } dependencies.ingress.acceptCommitted(result.events); } return { diff --git a/apps/server/src/features/providers/composition/register-providers.ts b/apps/server/src/features/providers/composition/register-providers.ts index 6f7011140..7ad724a34 100644 --- a/apps/server/src/features/providers/composition/register-providers.ts +++ b/apps/server/src/features/providers/composition/register-providers.ts @@ -1,4 +1,5 @@ import { instanceCachingFactory, Lifecycle, type DependencyContainer } from "tsyringe"; +import type { Database } from "bun:sqlite"; import { hostRuntime } from "@mcode/shared/node/host-runtime"; import { ClaudeProvider } from "../adapters/claude/claude-provider.js"; @@ -8,7 +9,8 @@ import { ProviderRegistry } from "./provider-registry.js"; import { createProviderHostPorts } from "./provider-host-ports.js"; import { BrowserAutomationSessionLease } from "../../browser-automation/index.js"; import { InternalThreadControlMcpRuntime } from "../../thread-control/index.js"; -import { CanonicalAgentBoundary } from "../../agents/index.js"; +import { publishCanonicalAgentEvents } from "../../agents/canonical/canonical-agent-boundary.js"; +import { CanonicalAgentWriterClient } from "../../agents/canonical/canonical-agent-writer-client.js"; import { ScopedPreGrantService } from "../../agents/permissions/scoped-pre-grant.js"; import { EnvService } from "../../../runtime/environment/env-service.js"; import type { JobObject } from "../../../runtime/process/containment/job-object.js"; @@ -28,6 +30,9 @@ import { /** Register provider adapters, the provider registry, and provider host ports. */ export function registerProviderAdapters(container: DependencyContainer): void { + container.register(CanonicalAgentWriterClient, { + useFactory: instanceCachingFactory((c) => new CanonicalAgentWriterClient(c.resolve("Database").filename)), + }); container.register(PROVIDER_EVENT_WORKER_POOL, { useFactory: instanceCachingFactory(() => new ThreadEventWorkerPool()), }); @@ -88,7 +93,8 @@ export function registerProviderAdapters(container: DependencyContainer): void { browser: c.resolve(BrowserAutomationSessionLease), threadControl: c.resolve(InternalThreadControlMcpRuntime), grants: c.resolve(ScopedPreGrantService), - events: c.resolve(CanonicalAgentBoundary), + events: c.resolve(CanonicalAgentWriterClient), + publishCanonicalEvents: publishCanonicalAgentEvents, ingress: c.resolve(ProviderEventIngress), }), }); From b25293b2edcfac7e0909ecbdf8714f29b1c1ee54 Mon Sep 17 00:00:00 2001 From: "(CJ) Chukwudi Nwobodo" <142016413+chuks-qua@users.noreply.github.com> Date: Thu, 24 Sep 2026 19:49:58 +0100 Subject: [PATCH 011/136] fix(server): include provider phase in write identity --- .../__tests__/provider-host-ports.test.ts | 30 +++++++++++++++++++ .../composition/provider-host-ports.ts | 7 ++++- 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/apps/server/src/features/providers/composition/__tests__/provider-host-ports.test.ts b/apps/server/src/features/providers/composition/__tests__/provider-host-ports.test.ts index 753f425a7..28832b3d0 100644 --- a/apps/server/src/features/providers/composition/__tests__/provider-host-ports.test.ts +++ b/apps/server/src/features/providers/composition/__tests__/provider-host-ports.test.ts @@ -148,4 +148,34 @@ describe("createProviderHostPorts", () => { await expect(ports.events.submit(batch)).rejects.toThrow("commit failed"); expect(acceptCommitted).not.toHaveBeenCalled(); }); + + it("uses one operation identity for a retry and distinct identities for other phases", async () => { + const commit = vi.fn(async () => ({ + outcome: "duplicate" as const, + conversationRevision: 1, + rosterRevision: 0, + acceptedThrough: 1, + durableThrough: 1, + events: [], + })); + const ports = createProviderHostPorts({ + runtime: { platform: "linux", architecture: "x64", nodeAbi: "127" }, + envService: { getEnv: () => ({}) }, + jobObject: { isWindowsJob: false }, + browser: {}, + threadControl: {}, + grants: {}, + events: { commit }, + publishCanonicalEvents: vi.fn(), + ingress: { acceptCommitted: vi.fn() }, + } as never); + const batch = { threadId: "thread-1", turnId: "turn-1", executionId: EXECUTION_ID, phase: "streaming", events: [] }; + + await ports.events.submit(batch); + await ports.events.submit(batch); + await ports.events.submit({ ...batch, phase: "completed" }); + const operationIds = commit.mock.calls.map(([operationId]) => operationId); + expect(operationIds[0]).toBe(operationIds[1]); + expect(operationIds[2]).not.toBe(operationIds[0]); + }); }); diff --git a/apps/server/src/features/providers/composition/provider-host-ports.ts b/apps/server/src/features/providers/composition/provider-host-ports.ts index 179f6717e..fb02dcdbc 100644 --- a/apps/server/src/features/providers/composition/provider-host-ports.ts +++ b/apps/server/src/features/providers/composition/provider-host-ports.ts @@ -88,7 +88,12 @@ export function createProviderHostPorts( events: { submit: async (batch) => { const operationId = NodeCrypto.createHash("sha256") - .update(JSON.stringify([batch.executionId, batch.events.map((event) => event.eventId)])) + .update(JSON.stringify([ + batch.executionId, + batch.phase, + batch.nativeCursor ?? null, + batch.events.map((event) => event.eventId), + ])) .digest("hex"); const result = await dependencies.events.commit(operationId, { threadId: batch.threadId, From 9e5782bf7cbf5f2638d5366cfb9ec2a2fd6c4653 Mon Sep 17 00:00:00 2001 From: "(CJ) Chukwudi Nwobodo" <142016413+chuks-qua@users.noreply.github.com> Date: Thu, 24 Sep 2026 19:52:50 +0100 Subject: [PATCH 012/136] fix(server): preserve checkpoint failure across Stop --- .../turns/__tests__/turn-event-pipeline.test.ts | 14 ++++++++++++++ .../features/agents/turns/turn-event-pipeline.ts | 5 ++++- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/apps/server/src/features/agents/turns/__tests__/turn-event-pipeline.test.ts b/apps/server/src/features/agents/turns/__tests__/turn-event-pipeline.test.ts index 5544dccdf..c9b96df95 100644 --- a/apps/server/src/features/agents/turns/__tests__/turn-event-pipeline.test.ts +++ b/apps/server/src/features/agents/turns/__tests__/turn-event-pipeline.test.ts @@ -253,6 +253,20 @@ describe("TurnEventPipeline", () => { expect(finalize).not.toHaveBeenCalled(); }); + it("does not turn a failed checkpoint into a successful later Stop", async () => { + const { pipeline, finalize } = createPipeline(() => Promise.reject(new Error("checkpoint failed"))); + + pipeline.handleProviderEvent(textDelta("write failure")); + await Promise.resolve(); + await expect(pipeline.finalizeTurn({ + threadId: "thread-1", + executionId: EXECUTION_ID, + outcome: "cancelled", + source: "user-stop", + })).rejects.toThrow("checkpoint failed"); + expect(finalize).not.toHaveBeenCalled(); + }); + it("holds finalization until its thread-affine ingress worker is idle", async () => { let releaseIngress!: () => void; const ingressIdle = new Promise((resolve) => { releaseIngress = resolve; }); diff --git a/apps/server/src/features/agents/turns/turn-event-pipeline.ts b/apps/server/src/features/agents/turns/turn-event-pipeline.ts index 8e8c3cd35..6d1704b5d 100644 --- a/apps/server/src/features/agents/turns/turn-event-pipeline.ts +++ b/apps/server/src/features/agents/turns/turn-event-pipeline.ts @@ -121,6 +121,7 @@ export class TurnEventPipeline implements ProviderEventIngressConsumer { /** Materialize a terminal turn once, after every earlier event in its turn queue. */ finalizeTurn(command: FinalizeTurnCommand): Promise | null { const inFlight = this.inFlightApplications.get(command.threadId); + const priorFailure = this.applicationErrors.get(command.threadId); if (cancelsDeferredWork(command.source)) this.discard(command.threadId, command.executionId); const existing = this.finalizations.get(command.threadId); if (existing) return existing; @@ -128,7 +129,9 @@ export class TurnEventPipeline implements ProviderEventIngressConsumer { const afterIngress = () => this.ingressFence ? this.ingressFence.waitForThread(command.threadId).then(() => this.finalizeAfterIngress(command)) : this.finalizeAfterIngress(command); - const pending = inFlight ? inFlight.then(afterIngress) : afterIngress(); + const pending = priorFailure !== undefined + ? Promise.reject(priorFailure) + : inFlight ? inFlight.then(afterIngress) : afterIngress(); this.finalizations.set(command.threadId, pending); void pending.then( () => this.finalizations.delete(command.threadId), From b7bdb46ae3c336e1abf996e1b9f1fbdfb3695dd0 Mon Sep 17 00:00:00 2001 From: "(CJ) Chukwudi Nwobodo" <142016413+chuks-qua@users.noreply.github.com> Date: Thu, 24 Sep 2026 19:53:09 +0100 Subject: [PATCH 013/136] feat(server): add data-only parent turn writer operations --- .../canonical-parent-turn-write.test.ts | 146 +++++++++++++++++ .../canonical/canonical-parent-turn-write.ts | 154 ++++++++++++++++++ 2 files changed, 300 insertions(+) create mode 100644 apps/server/src/features/agents/canonical/__tests__/canonical-parent-turn-write.test.ts create mode 100644 apps/server/src/features/agents/canonical/canonical-parent-turn-write.ts diff --git a/apps/server/src/features/agents/canonical/__tests__/canonical-parent-turn-write.test.ts b/apps/server/src/features/agents/canonical/__tests__/canonical-parent-turn-write.test.ts new file mode 100644 index 000000000..9fa4c22e9 --- /dev/null +++ b/apps/server/src/features/agents/canonical/__tests__/canonical-parent-turn-write.test.ts @@ -0,0 +1,146 @@ +import "reflect-metadata"; +import * as NodeFS from "node:fs"; +import * as NodeOS from "node:os"; +import * as NodePath from "node:path"; +import type { Database } from "bun:sqlite"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; + +import { openDatabase } from "../../../../runtime/persistence/sqlite/database.js"; +import { MessageRepo } from "../../conversation/persistence/message-repo.js"; +import { + CanonicalParentTurnWrite, + type DataOnlyParentTurnFinishInput, + type DataOnlyParentTurnStartInput, +} from "../canonical-parent-turn-write.js"; + +const THREAD_ID = "thread-1"; +const TURN_ID = "turn-1"; +const EXECUTION_ID = "00000000-0000-4000-8000-000000000001"; +const NOW = "2026-09-24T10:00:00.000Z"; + +function seedThread(db: Database): void { + db.prepare("INSERT INTO workspaces (id, name, path, created_at, updated_at) VALUES (?, ?, ?, ?, ?)") + .run("workspace-1", "Workspace", "C:/fixture", NOW, NOW); + db.prepare("INSERT INTO threads (id, workspace_id, title, branch, provider, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?)") + .run(THREAD_ID, "workspace-1", "Thread", "main", "codex", NOW, NOW); +} + +function startInput(): DataOnlyParentTurnStartInput { + return { + thread: { id: THREAD_ID, workspaceId: "workspace-1", providerId: "codex", createdAt: NOW }, + turnId: TURN_ID, + executionId: EXECUTION_ID, + permissionMode: "supervised", + providerIdentities: [], + userMessage: { kind: "create", messageId: "user-1", content: "Question", sequence: 1 }, + }; +} + +function finishInput(message: ReturnType): DataOnlyParentTurnFinishInput { + return { + threadId: THREAD_ID, + turnId: TURN_ID, + executionId: EXECUTION_ID, + providerId: "codex", + providerIdentities: [], + outcome: "completed", + projection: { message, narrative: [] }, + }; +} + +describe("CanonicalParentTurnWrite", () => { + let directory: string; + let path: string; + let db: Database; + let published: string[][]; + let writer: CanonicalParentTurnWrite; + + beforeEach(() => { + directory = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "mcode-parent-write-")); + path = NodePath.join(directory, "mcode.db"); + db = openDatabase({ dbPath: path }); + seedThread(db); + published = []; + writer = new CanonicalParentTurnWrite(db, (events) => { + published.push(events.map((event) => event.eventId)); + }); + }); + + afterEach(() => { + db.close(true); + NodeFS.rmSync(directory, { recursive: true, force: true }); + }); + + it("commits cloneable start, event checkpoint and terminal projection across a reload", async () => { + const start = startInput(); + expect(() => structuredClone(start)).not.toThrow(); + expect(writer.start(start).outcome).toBe("committed"); + expect(writer.start(start).outcome).toBe("duplicate"); + + const event = { + eventId: `${EXECUTION_ID}:item-1`, + routing: { threadId: THREAD_ID, turnId: TURN_ID, executionId: EXECUTION_ID, itemId: "item-1" }, + sourceProviderId: "codex", + sourceIdentities: [], + payload: { + type: "item.recorded" as const, + item: { + id: "item-1", + threadId: THREAD_ID, + turnId: TURN_ID, + kind: "message" as const, + providerIdentities: [], + payload: { projection: "message", content: "Event" }, + createdAt: NOW, + updatedAt: NOW, + }, + }, + }; + expect(writer.append({ threadId: THREAD_ID, turnId: TURN_ID, executionId: EXECUTION_ID, phase: "running", events: [event] }).outcome) + .toBe("committed"); + expect(writer.append({ threadId: THREAD_ID, turnId: TURN_ID, executionId: EXECUTION_ID, phase: "running", events: [event] }).outcome) + .toBe("duplicate"); + + const staged = new MessageRepo(db).create(THREAD_ID, "assistant", "Answer", 2, undefined, undefined, undefined, "model", true); + const finish = finishInput(staged); + expect(() => structuredClone(finish)).not.toThrow(); + expect((await writer.finish(finish)).outcome).toBe("committed"); + expect((await writer.finish(finish)).outcome).toBe("terminal-outcome-confirmed"); + expect(published.flat()).toContain(`${EXECUTION_ID}:turn.completed`); + + db.close(true); + db = openDatabase({ dbPath: path }); + const checkpoint = db.prepare("SELECT phase, terminal_outcome, last_accepted_sequence, last_durable_sequence FROM canonical_agent_ingest_checkpoints WHERE execution_id = ?") + .get(EXECUTION_ID) as { phase: string; terminal_outcome: string; last_accepted_sequence: number; last_durable_sequence: number }; + expect(checkpoint).toMatchObject({ phase: "completed", terminal_outcome: "completed" }); + expect(checkpoint.last_accepted_sequence).toBe(checkpoint.last_durable_sequence); + expect(checkpoint.last_accepted_sequence).toBeGreaterThan(4); + expect(new MessageRepo(db).findByIdInThread(THREAD_ID, staged.id)).toMatchObject({ is_internal: false, outcome: "completed" }); + }); + + it("rolls back a failed user-message projection with canonical start", () => { + db.prepare("UPDATE threads SET user_completed_at = ? WHERE id = ?").run(NOW, THREAD_ID); + db.run("CREATE TRIGGER fail_user_message BEFORE INSERT ON messages BEGIN SELECT RAISE(ABORT, 'message unavailable'); END"); + expect(() => writer.start({ ...startInput(), reopenThread: true })).toThrow("message unavailable"); + expect(db.prepare("SELECT user_completed_at FROM threads WHERE id = ?").get(THREAD_ID)).toEqual({ user_completed_at: NOW }); + expect(db.prepare("SELECT COUNT(*) AS count FROM canonical_agent_ingest_checkpoints").get()).toEqual({ count: 0 }); + expect(published).toEqual([]); + }); + + it("does not confirm a terminal checkpoint when assistant publication fails", async () => { + writer.start(startInput()); + const staged = new MessageRepo(db).create(THREAD_ID, "assistant", "Answer", 2, undefined, undefined, undefined, "model", true); + db.run("CREATE TRIGGER fail_assistant_outcome BEFORE UPDATE OF outcome ON messages BEGIN SELECT RAISE(ABORT, 'outcome unavailable'); END"); + const finish = finishInput(staged); + await expect(writer.finish(finish)).rejects.toThrow("outcome unavailable"); + expect(db.prepare("SELECT terminal_outcome FROM canonical_agent_ingest_checkpoints WHERE execution_id = ?").get(EXECUTION_ID)) + .toEqual({ terminal_outcome: null }); + expect(new MessageRepo(db).listIncludingInternal(THREAD_ID).find((message) => message.id === staged.id)) + .toMatchObject({ is_internal: true, outcome: null }); + expect(published.flat()).not.toContain(`${EXECUTION_ID}:turn.completed`); + + db.run("DROP TRIGGER fail_assistant_outcome"); + expect((await writer.finish(finish)).outcome).toBe("committed"); + expect(new MessageRepo(db).findByIdInThread(THREAD_ID, staged.id)).toMatchObject({ is_internal: false, outcome: "completed" }); + }); +}); diff --git a/apps/server/src/features/agents/canonical/canonical-parent-turn-write.ts b/apps/server/src/features/agents/canonical/canonical-parent-turn-write.ts new file mode 100644 index 000000000..7ae75a3c0 --- /dev/null +++ b/apps/server/src/features/agents/canonical/canonical-parent-turn-write.ts @@ -0,0 +1,154 @@ +import type { Database } from "bun:sqlite"; + +import { MessageRepo } from "../conversation/persistence/message-repo.js"; +import { PlanQuestionAnswersRepo } from "../planning/persistence/plan-question-answers-repo.js"; +import type { + ParentTurnFinishInput, + ParentTurnProjection, + ParentTurnStartInput, +} from "../turns/parent-turn-durability.js"; +import { ThreadRepo } from "../../thread-control/persistence/thread-repo.js"; +import { + CanonicalAgentBoundary, + type CanonicalAgentBatchedCommitResult, + type CanonicalAgentCommitInput, + type CanonicalAgentCommitResult, + type CanonicalAgentEventDraft, + type CanonicalAgentEventPublisher, +} from "./canonical-agent-boundary.js"; + +type CreateMessageArgument = Parameters; + +/** User-message data the writer can project inside the canonical start transaction. */ +export type ParentUserMessageWrite = + | { kind: "existing"; messageId: string } + | { + kind: "create"; + messageId?: string; + content: string; + sequence: number; + attachments?: CreateMessageArgument[4]; + replyToMessageId?: CreateMessageArgument[5]; + quotedText?: CreateMessageArgument[6]; + mentions?: CreateMessageArgument[9]; + previewAnnotations?: CreateMessageArgument[10]; + origin?: CreateMessageArgument[11]; + selectedTextComments?: CreateMessageArgument[13]; + }; + +/** Values for an atomic start; no caller closure enters the writer. */ +export interface DataOnlyParentTurnStartInput extends Omit { + userMessage: ParentUserMessageWrite; + reopenThread?: boolean; + answeredPlanQuestionMessageId?: string; +} + +/** Values for one canonical event and checkpoint transaction. */ +export interface DataOnlyParentEventInput extends Omit { + events: readonly CanonicalAgentEventDraft[]; +} + +/** A staged assistant and narrative to confirm in the terminal transaction. */ +export interface DataOnlyParentTurnFinishInput extends Omit { + projection: ParentTurnProjection; +} + +/** Writer-local semantic operations with cloneable inputs and durable receipts. */ +export class CanonicalParentTurnWrite { + private readonly canonical: CanonicalAgentBoundary; + private readonly messages: MessageRepo; + private readonly threads: ThreadRepo; + private readonly planAnswers: PlanQuestionAnswersRepo; + private readonly stagedAssistant: ReturnType; + + constructor(db: Database, publish: CanonicalAgentEventPublisher) { + this.canonical = new CanonicalAgentBoundary(db, publish); + this.messages = new MessageRepo(db); + this.threads = new ThreadRepo(db); + this.planAnswers = new PlanQuestionAnswersRepo(db); + this.stagedAssistant = db.prepare("SELECT role, content FROM messages WHERE id = ? AND thread_id = ?"); + } + + /** Commit the user message, optional thread reopen and plan answer with canonical start. */ + start(input: DataOnlyParentTurnStartInput): CanonicalAgentCommitResult { + return this.canonical.startParentTurn({ + ...input, + projectUserMessage: () => { + if (input.reopenThread && !this.threads.reopen(input.thread.id)) { + throw new Error(`Thread not found: ${input.thread.id}`); + } + const message = this.projectUserMessage(input); + if (input.answeredPlanQuestionMessageId) { + this.planAnswers.markAnswered(input.answeredPlanQuestionMessageId, input.thread.id); + } + return message; + }, + }); + } + + /** Commit a plain canonical event batch and its checkpoint before publication. */ + append(input: DataOnlyParentEventInput): CanonicalAgentCommitResult { + return this.canonical.commit(input); + } + + /** Confirm the terminal checkpoint and publish the staged assistant in one transaction. */ + finish(input: DataOnlyParentTurnFinishInput): Promise { + this.assertStagedAssistant(input); + const projection: ParentTurnProjection = { + message: input.projection.message + ? { + ...input.projection.message, + is_internal: false, + outcome: input.outcome, + outcomeExecutionId: input.executionId, + } + : null, + narrative: input.projection.narrative, + }; + return this.canonical.finishParentTurnBatched({ + ...input, + projectTurn: () => projection, + finalizeCompatibility: () => { + if (!projection.message) return; + this.messages.setAssistantOutcome(projection.message.id, input.outcome, input.executionId); + this.messages.publishAssistant(projection.message.id); + }, + }); + } + + private projectUserMessage(input: DataOnlyParentTurnStartInput) { + const { userMessage } = input; + if (userMessage.kind === "existing") { + const existing = this.messages.findByIdInThread(input.thread.id, userMessage.messageId); + if (!existing || existing.role !== "user") { + throw new Error(`Queued user message not found: ${userMessage.messageId}`); + } + return existing; + } + return this.messages.create( + input.thread.id, + "user", + userMessage.content, + userMessage.sequence, + userMessage.attachments, + userMessage.replyToMessageId, + userMessage.quotedText, + undefined, + undefined, + userMessage.mentions, + userMessage.previewAnnotations, + userMessage.origin, + userMessage.messageId, + userMessage.selectedTextComments, + ); + } + + private assertStagedAssistant(input: DataOnlyParentTurnFinishInput): void { + const projected = input.projection.message; + if (!projected) return; + const staged = this.stagedAssistant.get(projected.id, input.threadId) as { role: string; content: string } | null; + if (!staged || staged.role !== "assistant" || staged.content !== projected.content) { + throw new Error(`Staged assistant projection not found: ${projected.id}`); + } + } +} From c78d5289ad444aeecd0adcbcaef83546819aa7c7 Mon Sep 17 00:00:00 2001 From: "(CJ) Chukwudi Nwobodo" <142016413+chuks-qua@users.noreply.github.com> Date: Thu, 24 Sep 2026 19:50:53 +0100 Subject: [PATCH 014/136] feat(server): model worker-owned execution commands --- .../execution-worker-handler.test.ts | 207 ++++++++++++++ .../execution/execution-mailbox-scheduler.ts | 2 +- .../execution/execution-worker-handler.ts | 257 ++++++++++++++++++ 3 files changed, 465 insertions(+), 1 deletion(-) create mode 100644 apps/server/src/features/agents/execution/__tests__/execution-worker-handler.test.ts create mode 100644 apps/server/src/features/agents/execution/execution-worker-handler.ts diff --git a/apps/server/src/features/agents/execution/__tests__/execution-worker-handler.test.ts b/apps/server/src/features/agents/execution/__tests__/execution-worker-handler.test.ts new file mode 100644 index 000000000..0f25fd725 --- /dev/null +++ b/apps/server/src/features/agents/execution/__tests__/execution-worker-handler.test.ts @@ -0,0 +1,207 @@ +import type { ProviderEventDraft } from "@mcode/providers"; +import { describe, expect, it } from "vitest"; + +import { + ExecutionMailboxScheduler, + type ExecutionMailboxCommand, + type ExecutionMailboxLimits, +} from "../execution-mailbox-scheduler.js"; +import type { + ExecutionIdentity, + ExecutionLease, + ExecutionWorkerPort, + ExecutionWorkerReply, + ExecutionWorkerRequest, +} from "../execution-mailbox-protocol.js"; +import { + ExecutionWorkerHandler, + type ExecutionSemanticOperation, + type ExecutionSemanticWriter, + type ExecutionWorkCommand, + type ExecutionWorkerResult, + type ExecutionWriteReceipt, +} from "../execution-worker-handler.js"; + +type Command = ExecutionMailboxCommand; + +const EXECUTION: ExecutionIdentity = { + threadId: "fixture-thread", + turnId: "fixture-turn", + executionId: "00000000-0000-4000-8000-000000000001", +}; + +const LIMITS: ExecutionMailboxLimits = { + maxPending: 12, + maxPendingBytes: 12_000, + reservedControl: 6, + reservedControlBytes: 6_000, + maxPerExecutionPending: 8, + maxPerExecutionBytes: 8_000, + reservedPerExecutionControl: 5, + reservedPerExecutionControlBytes: 5_000, +}; + +class RecordingWriter implements ExecutionSemanticWriter { + readonly operations: ExecutionSemanticOperation[] = []; + beforeCommit: ((operation: ExecutionSemanticOperation) => Promise) | undefined; + conflictKind: ExecutionSemanticOperation["mutation"]["kind"] | undefined; + + async transact(operation: ExecutionSemanticOperation): Promise { + this.operations.push(operation); + await this.beforeCommit?.(operation); + if (operation.mutation.kind === this.conflictKind) { + return { kind: "conflict", operationId: operation.operationId }; + } + return { kind: "committed", operationId: operation.operationId, durableRevision: this.operations.length }; + } +} + +class InlineWorker implements ExecutionWorkerPort { + onmessage: ((event: MessageEvent>) => void) | null = null; + onerror: ((event: ErrorEvent) => void) | null = null; + readonly requests: ExecutionWorkerRequest[] = []; + terminated = false; + + constructor(private readonly handler: ExecutionWorkerHandler) {} + + postMessage(request: ExecutionWorkerRequest): void { + this.requests.push(request); + void this.handler.handle(request) + .then((reply) => this.onmessage?.(new MessageEvent("message", { data: reply }))) + .catch(() => this.onerror?.(new ErrorEvent("error"))); + } + + terminate(): void { + this.terminated = true; + } +} + +function fixture(writer = new RecordingWriter()) { + const workers: InlineWorker[] = []; + const lost: ExecutionIdentity[][] = []; + const scheduler = new ExecutionMailboxScheduler({ + workerCount: 1, + limits: LIMITS, + createWorker: () => { + const worker = new InlineWorker(new ExecutionWorkerHandler(writer)); + workers.push(worker); + return worker; + }, + onWorkerLost: (executions) => lost.push([...executions]), + }); + const claim = scheduler.claim(EXECUTION, 1); + if (claim.kind !== "claimed") throw new Error(`Claim failed: ${claim.kind}`); + return { scheduler, worker: workers[0]!, writer, lost, lease: claim.lease }; +} + +function submit( + scheduler: ExecutionMailboxScheduler, + lease: ExecutionLease, + command: Command, +) { + const admission = scheduler.submit({ execution: EXECUTION, lease, command, byteLength: 1_000 }); + if (admission.kind !== "admitted") throw new Error(`Admission failed: ${admission.kind}`); + return admission; +} + +function eventDraft(): ProviderEventDraft { + return { + eventId: "fixture-event-1", + routing: { ...EXECUTION, itemId: "fixture-item-1" }, + sourceProviderId: "codex", + sourceIdentities: [], + sourceSequence: 1, + payload: { type: "turn.started", startedAt: "2026-09-24T12:00:00.000Z" }, + }; +} + +async function committed(admission: ReturnType, revision: number): Promise { + await expect(admission.completion).resolves.toMatchObject({ + kind: "reply", + result: { kind: "committed", durableRevision: revision }, + }); +} + +describe("ExecutionWorkerHandler through its scheduler", () => { + it("runs one synthetic turn from start through Stop, checkpoint, and finalization", async () => { + const { scheduler, worker, writer, lease } = fixture(); + await committed(submit(scheduler, lease, { kind: "start", providerId: "codex" }), 1); + await committed(submit(scheduler, lease, { kind: "event", events: [eventDraft()] }), 2); + const stop = submit(scheduler, lease, { kind: "stop", requestId: "stop-1" }); + expect(scheduler.submit({ + execution: EXECUTION, + lease, + command: { kind: "event", events: [eventDraft()] }, + byteLength: 1_000, + })).toEqual({ kind: "stopping" }); + await committed(stop, 3); + await committed(submit(scheduler, lease, { kind: "checkpoint", phase: "stopping", nativeCursor: "cursor-1" }), 4); + await committed(submit(scheduler, lease, { kind: "effect-result", effectId: "file-1", settled: true }), 5); + await committed(submit(scheduler, lease, { kind: "provider-outcome", outcome: "cancelled" }), 6); + await committed(submit(scheduler, lease, { kind: "finalize", outcome: "cancelled" }), 7); + await expect(submit(scheduler, lease, { kind: "release" }).completion).resolves.toEqual({ + kind: "reply", + result: { kind: "released" }, + }); + expect(scheduler.release(EXECUTION, lease)).toBe(true); + + expect(writer.operations.map((operation) => operation.mutation.kind)).toEqual([ + "begin", "append-events", "stop-requested", "checkpoint", "effect-result", "provider-outcome", "finish", + ]); + expect(writer.operations[2]?.mutation).toEqual({ + kind: "stop-requested", requestId: "stop-1", lastAdmittedOrdinal: 2, + }); + expect(worker.requests.map((request) => request.ordinal)).toEqual([1, 2, 3, 4, 5, 6, 7, 8]); + expect(worker.requests.every((request) => request.execution.executionId === EXECUTION.executionId + && request.lease.leaseId === lease.leaseId)).toBe(true); + expect(scheduler.depth()).toMatchObject({ pending: 0, activeExecutions: 0 }); + scheduler.shutdown(); + }); + + it("does not acknowledge a worker command until its writer receipt arrives", async () => { + const writer = new RecordingWriter(); + let releaseWrite: (() => void) | undefined; + writer.beforeCommit = () => new Promise((resolve) => { releaseWrite = resolve; }); + const { scheduler, worker, lease } = fixture(writer); + const start = submit(scheduler, lease, { kind: "start", providerId: "codex" }); + const event = submit(scheduler, lease, { kind: "event", events: [eventDraft()] }); + await Promise.resolve(); + expect(worker.requests).toHaveLength(1); + expect(scheduler.depth()).toMatchObject({ pending: 2 }); + releaseWrite?.(); + await committed(start, 1); + expect(worker.requests).toHaveLength(2); + releaseWrite?.(); + await committed(event, 2); + scheduler.shutdown(); + }); + + it("rejects a writer conflict without reporting a durable command", async () => { + const writer = new RecordingWriter(); + writer.conflictKind = "append-events"; + const { scheduler, lease } = fixture(writer); + await committed(submit(scheduler, lease, { kind: "start", providerId: "codex" }), 1); + await expect(submit(scheduler, lease, { kind: "event", events: [eventDraft()] }).completion).resolves.toEqual({ + kind: "reply", + result: { kind: "rejected", reason: "writer-conflict" }, + }); + await expect(submit(scheduler, lease, { kind: "checkpoint", phase: "running", nativeCursor: null }).completion) + .resolves.toEqual({ kind: "reply", result: { kind: "rejected", reason: "out-of-order" } }); + scheduler.shutdown(); + }); + + it("revokes the execution when the writer fails instead of acknowledging its event", async () => { + const writer = new RecordingWriter(); + writer.beforeCommit = async (operation) => { + if (operation.mutation.kind === "append-events") throw new Error("database unavailable"); + }; + const { scheduler, worker, lost, lease } = fixture(writer); + await committed(submit(scheduler, lease, { kind: "start", providerId: "codex" }), 1); + await expect(submit(scheduler, lease, { kind: "event", events: [eventDraft()] }).completion) + .resolves.toEqual({ kind: "worker-lost" }); + expect(lost).toEqual([[EXECUTION]]); + expect(worker.terminated).toBe(true); + expect(scheduler.depth()).toMatchObject({ pending: 0, activeExecutions: 0 }); + scheduler.shutdown(); + }); +}); diff --git a/apps/server/src/features/agents/execution/execution-mailbox-scheduler.ts b/apps/server/src/features/agents/execution/execution-mailbox-scheduler.ts index e3e062953..9b6fe8685 100644 --- a/apps/server/src/features/agents/execution/execution-mailbox-scheduler.ts +++ b/apps/server/src/features/agents/execution/execution-mailbox-scheduler.ts @@ -15,7 +15,7 @@ export type ExecutionMailboxCommand = | { readonly kind: "stop"; readonly requestId: string }; /** Lifecycle results that must be allowed to settle after Stop is admitted. */ -export const EXECUTION_CONTROL_KINDS = ["checkpoint", "effect-result", "provider-outcome", "finalize"] as const; +export const EXECUTION_CONTROL_KINDS = ["checkpoint", "effect-result", "provider-outcome", "finalize", "release"] as const; export type ExecutionControlKind = typeof EXECUTION_CONTROL_KINDS[number]; const CONTROL_KINDS: ReadonlySet = new Set(EXECUTION_CONTROL_KINDS); diff --git a/apps/server/src/features/agents/execution/execution-worker-handler.ts b/apps/server/src/features/agents/execution/execution-worker-handler.ts new file mode 100644 index 000000000..b475239e0 --- /dev/null +++ b/apps/server/src/features/agents/execution/execution-worker-handler.ts @@ -0,0 +1,257 @@ +import type { TurnOutcome } from "@mcode/contracts"; +import type { ProviderEventDraft } from "@mcode/providers"; + +import type { + ExecutionIdentity, + ExecutionLease, + ExecutionWorkerReply, + ExecutionWorkerRequest, +} from "./execution-mailbox-protocol.js"; +import type { ExecutionMailboxCommand } from "./execution-mailbox-scheduler.js"; + +/** Data that an execution worker may receive without a server dependency container. */ +export type ExecutionWorkCommand = + | { readonly kind: "start"; readonly providerId: string } + | { readonly kind: "resume"; readonly providerId: string; readonly checkpointId: string } + | { readonly kind: "event"; readonly events: readonly ProviderEventDraft[] } + | { readonly kind: "checkpoint"; readonly phase: string; readonly nativeCursor: unknown | null } + | { readonly kind: "effect-result"; readonly effectId: string; readonly settled: boolean } + | { readonly kind: "provider-outcome"; readonly outcome: TurnOutcome } + | { readonly kind: "finalize"; readonly outcome: TurnOutcome } + | { readonly kind: "release" }; + +/** One complete semantic mutation. The single writer decides its SQL transaction. */ +export interface ExecutionSemanticOperation { + readonly operationId: string; + readonly execution: ExecutionIdentity; + readonly lease: ExecutionLease; + readonly ordinal: number; + readonly mutation: + | { readonly kind: "begin"; readonly providerId: string; readonly checkpointId?: string } + | { readonly kind: "append-events"; readonly events: readonly ProviderEventDraft[] } + | { readonly kind: "checkpoint"; readonly phase: string; readonly nativeCursor: unknown | null } + | { readonly kind: "stop-requested"; readonly requestId: string; readonly lastAdmittedOrdinal: number } + | { readonly kind: "effect-result"; readonly effectId: string; readonly settled: boolean } + | { readonly kind: "provider-outcome"; readonly outcome: TurnOutcome } + | { readonly kind: "finish"; readonly outcome: TurnOutcome }; +} + +/** A writer reply is valid only after the semantic operation commits durably. */ +export type ExecutionWriteReceipt = + | { readonly kind: "committed"; readonly operationId: string; readonly durableRevision: number } + | { readonly kind: "conflict"; readonly operationId: string }; + +/** + * Implemented by one acknowledged writer, not by a worker-local SQLite connection. + * It must fence the durable lease and enforce terminal prerequisites in the + * same transaction that records each semantic operation. + */ +export interface ExecutionSemanticWriter { + transact(operation: ExecutionSemanticOperation): Promise; +} + +/** A command result that never calls an uncommitted mutation successful. */ +export type ExecutionWorkerResult = + | { readonly kind: "committed"; readonly operationId: string; readonly durableRevision: number } + | { readonly kind: "released" } + | { readonly kind: "rejected"; readonly reason: "no-execution" | "stale-execution" | "out-of-order" | "invalid-transition" | "invalid-event-routing" | "invalid-stop-watermark" | "writer-conflict" }; + +type WorkerCommand = ExecutionMailboxCommand; + +interface ExecutionState { + readonly execution: ExecutionIdentity; + readonly lease: ExecutionLease; + nextOrdinal: number; + phase: "running" | "stopping" | "finalized"; + durableRevision: number; +} + +/** + * Applies one worker command at a time per slot. This module has no database + * handle, provider process, or publication callback. It only advances state + * after the single writer acknowledges the semantic operation. + */ +export class ExecutionWorkerHandler { + private readonly states = new Map(); + + constructor(private readonly writer: ExecutionSemanticWriter) {} + + /** Process one mailbox command and echo its exact execution and lease. */ + async handle(request: ExecutionWorkerRequest): Promise> { + const result = await this.apply(request); + return { + requestId: request.requestId, + execution: request.execution, + lease: request.lease, + ordinal: request.ordinal, + result, + }; + } + + private async apply(request: ExecutionWorkerRequest): Promise { + const state = this.states.get(request.execution.threadId); + if (request.command.kind === "start" || request.command.kind === "resume") { + return await this.begin(request, state); + } + if (!state) return { kind: "rejected", reason: "no-execution" }; + return await this.applyToState(request, state); + } + + private async applyToState( + request: ExecutionWorkerRequest, + state: ExecutionState, + ): Promise { + const rejection = commandRejection(request, state); + if (rejection) return { kind: "rejected", reason: rejection }; + if (request.command.kind === "release") return this.release(request, state); + if (state.phase === "finalized") return { kind: "rejected", reason: "invalid-transition" }; + const mutation = mutationFor(request, state); + if (!mutation) return { kind: "rejected", reason: invalidReason(request) }; + const receipt = await this.writer.transact(operationFor(request, mutation)); + if (!isDurableReceipt(receipt, request, state.durableRevision)) { + return { kind: "rejected", reason: "writer-conflict" }; + } + state.nextOrdinal += 1; + state.durableRevision = receipt.durableRevision; + if (request.command.kind === "stop") state.phase = "stopping"; + if (request.command.kind === "finalize") state.phase = "finalized"; + return { kind: "committed", operationId: receipt.operationId, durableRevision: receipt.durableRevision }; + } + + private async begin( + request: ExecutionWorkerRequest, + existing: ExecutionState | undefined, + ): Promise { + if (existing) return { kind: "rejected", reason: "invalid-transition" }; + if (request.ordinal !== 1) return { kind: "rejected", reason: "out-of-order" }; + const command = request.command; + if (command.kind !== "start" && command.kind !== "resume") return { kind: "rejected", reason: "invalid-transition" }; + const mutation: ExecutionSemanticOperation["mutation"] = { + kind: "begin", + providerId: command.providerId, + ...(command.kind === "resume" ? { checkpointId: command.checkpointId } : {}), + }; + const receipt = await this.writer.transact(operationFor(request, mutation)); + if (!isDurableReceipt(receipt, request, 0)) { + return { kind: "rejected", reason: "writer-conflict" }; + } + this.states.set(request.execution.threadId, { + execution: request.execution, + lease: request.lease, + nextOrdinal: 2, + phase: "running", + durableRevision: receipt.durableRevision, + }); + return { kind: "committed", operationId: receipt.operationId, durableRevision: receipt.durableRevision }; + } + + private release(request: ExecutionWorkerRequest, state: ExecutionState): ExecutionWorkerResult { + if (state.phase !== "finalized") return { kind: "rejected", reason: "invalid-transition" }; + this.states.delete(request.execution.threadId); + return { kind: "released" }; + } +} + +function mutationFor( + request: ExecutionWorkerRequest, + state: ExecutionState, +): ExecutionSemanticOperation["mutation"] | undefined { + const command = request.command; + switch (command.kind) { + case "event": + return eventMutation(command, request.execution, state); + case "stop": + return stopMutation(command, request, state); + case "checkpoint": + return { kind: "checkpoint", phase: command.phase, nativeCursor: command.nativeCursor }; + case "effect-result": + return { kind: "effect-result", effectId: command.effectId, settled: command.settled }; + case "provider-outcome": + return { kind: "provider-outcome", outcome: command.outcome }; + case "finalize": + return { kind: "finish", outcome: command.outcome }; + case "start": + case "resume": + case "release": + return undefined; + default: { + const exhaustive: never = command; + return exhaustive; + } + } +} + +function eventMutation( + command: Extract, + execution: ExecutionIdentity, + state: ExecutionState, +): ExecutionSemanticOperation["mutation"] | undefined { + if (state.phase !== "running" || !validEventRouting(command.events, execution)) return undefined; + return { kind: "append-events", events: command.events }; +} + +function stopMutation( + command: Extract, + request: ExecutionWorkerRequest, + state: ExecutionState, +): ExecutionSemanticOperation["mutation"] | undefined { + if (state.phase !== "running" || request.stopWatermark !== request.ordinal - 1) return undefined; + return { kind: "stop-requested", requestId: command.requestId, lastAdmittedOrdinal: request.stopWatermark }; +} + +function commandRejection( + request: ExecutionWorkerRequest, + state: ExecutionState, +): Extract["reason"] | null { + if (!sameIdentity(state.execution, request.execution) || !sameLease(state.lease, request.lease)) return "stale-execution"; + if (request.ordinal !== state.nextOrdinal) return "out-of-order"; + return null; +} + +function invalidReason(request: ExecutionWorkerRequest): Extract["reason"] { + if (request.command.kind === "stop" && request.stopWatermark !== request.ordinal - 1) return "invalid-stop-watermark"; + if (request.command.kind === "event" && !validEventRouting(request.command.events, request.execution)) { + return "invalid-event-routing"; + } + return "invalid-transition"; +} + +function validEventRouting(events: readonly ProviderEventDraft[], execution: ExecutionIdentity): boolean { + return events.length > 0 && events.every((event) => event.routing.threadId === execution.threadId + && event.routing.turnId === execution.turnId && event.routing.executionId === execution.executionId); +} + +function operationFor( + request: ExecutionWorkerRequest, + mutation: ExecutionSemanticOperation["mutation"], +): ExecutionSemanticOperation { + return { + operationId: operationId(request), + execution: request.execution, + lease: request.lease, + ordinal: request.ordinal, + mutation, + }; +} + +function operationId(request: ExecutionWorkerRequest): string { + return `${request.lease.leaseId}:${request.ordinal}`; +} + +function isDurableReceipt( + receipt: ExecutionWriteReceipt, + request: ExecutionWorkerRequest, + previousRevision: number, +): receipt is Extract { + return receipt.kind === "committed" && receipt.operationId === operationId(request) + && Number.isSafeInteger(receipt.durableRevision) && receipt.durableRevision >= previousRevision; +} + +function sameIdentity(left: ExecutionIdentity, right: ExecutionIdentity): boolean { + return left.threadId === right.threadId && left.turnId === right.turnId && left.executionId === right.executionId; +} + +function sameLease(left: ExecutionLease, right: ExecutionLease): boolean { + return left.ownerEpoch === right.ownerEpoch && left.workerIndex === right.workerIndex + && left.workerGeneration === right.workerGeneration && left.leaseId === right.leaseId; +} From b79e404e6b6de69190257c31acd61878797f7575 Mon Sep 17 00:00:00 2001 From: "(CJ) Chukwudi Nwobodo" <142016413+chuks-qua@users.noreply.github.com> Date: Thu, 24 Sep 2026 19:55:53 +0100 Subject: [PATCH 015/136] feat(server): replay durable canonical writer receipts --- .../drizzle/0065_ambitious_blindfold.sql | 10 + apps/server/drizzle/meta/0065_snapshot.json | 4989 +++++++++++++++++ apps/server/drizzle/meta/_journal.json | 7 + .../canonical-agent-writer-client.test.ts | 149 +- .../canonical-agent-writer-client.ts | 172 +- .../canonical-agent-writer-protocol.ts | 4 +- .../canonical-agent-writer-receipts.ts | 168 + .../canonical-agent-writer.worker.ts | 76 +- .../src/runtime/persistence/sqlite/schema.ts | 17 +- 9 files changed, 5488 insertions(+), 104 deletions(-) create mode 100644 apps/server/drizzle/0065_ambitious_blindfold.sql create mode 100644 apps/server/drizzle/meta/0065_snapshot.json create mode 100644 apps/server/src/features/agents/canonical/canonical-agent-writer-receipts.ts diff --git a/apps/server/drizzle/0065_ambitious_blindfold.sql b/apps/server/drizzle/0065_ambitious_blindfold.sql new file mode 100644 index 000000000..7b69d5cc8 --- /dev/null +++ b/apps/server/drizzle/0065_ambitious_blindfold.sql @@ -0,0 +1,10 @@ +CREATE TABLE `canonical_writer_operation_receipts` ( + `execution_id` text NOT NULL, + `operation_id` text NOT NULL, + `kind` text NOT NULL, + `input_hash` text NOT NULL, + `receipt_json` text NOT NULL, + `created_at` text DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')) NOT NULL, + PRIMARY KEY(`execution_id`, `operation_id`), + FOREIGN KEY (`execution_id`) REFERENCES `canonical_agent_turns`(`execution_id`) ON UPDATE no action ON DELETE cascade +); diff --git a/apps/server/drizzle/meta/0065_snapshot.json b/apps/server/drizzle/meta/0065_snapshot.json new file mode 100644 index 000000000..c9c8fbfdf --- /dev/null +++ b/apps/server/drizzle/meta/0065_snapshot.json @@ -0,0 +1,4989 @@ +{ + "version": "6", + "dialect": "sqlite", + "id": "d6ce0589-8bee-4f7f-b188-033550ae9cf0", + "prevId": "991b4c44-b789-4554-99a3-77caea9d753d", + "tables": { + "canonical_agent_events": { + "name": "canonical_agent_events", + "columns": { + "event_id": { + "name": "event_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "thread_id": { + "name": "thread_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "turn_id": { + "name": "turn_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "execution_id": { + "name": "execution_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "accepted_sequence": { + "name": "accepted_sequence", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "durable_revision": { + "name": "durable_revision", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "roster_revision": { + "name": "roster_revision", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "envelope_json": { + "name": "envelope_json", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "accepted_at": { + "name": "accepted_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "persisted_at": { + "name": "persisted_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_canonical_agent_events_execution_sequence": { + "name": "idx_canonical_agent_events_execution_sequence", + "columns": [ + "execution_id", + "accepted_sequence" + ], + "isUnique": true + }, + "idx_canonical_agent_events_thread_revision": { + "name": "idx_canonical_agent_events_thread_revision", + "columns": [ + "thread_id", + "durable_revision" + ], + "isUnique": false + } + }, + "foreignKeys": { + "canonical_agent_events_thread_id_canonical_agent_threads_id_fk": { + "name": "canonical_agent_events_thread_id_canonical_agent_threads_id_fk", + "tableFrom": "canonical_agent_events", + "tableTo": "canonical_agent_threads", + "columnsFrom": [ + "thread_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "canonical_agent_ingest_checkpoints": { + "name": "canonical_agent_ingest_checkpoints", + "columns": { + "execution_id": { + "name": "execution_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "thread_id": { + "name": "thread_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "turn_id": { + "name": "turn_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "last_accepted_sequence": { + "name": "last_accepted_sequence", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "last_durable_sequence": { + "name": "last_durable_sequence", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "native_cursor_json": { + "name": "native_cursor_json", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "phase": { + "name": "phase", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "terminal_outcome": { + "name": "terminal_outcome", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "recovery_incident_id": { + "name": "recovery_incident_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_canonical_agent_checkpoints_thread": { + "name": "idx_canonical_agent_checkpoints_thread", + "columns": [ + "thread_id", + "updated_at" + ], + "isUnique": false + }, + "idx_canonical_agent_checkpoints_recovery_incident": { + "name": "idx_canonical_agent_checkpoints_recovery_incident", + "columns": [ + "recovery_incident_id", + "updated_at" + ], + "isUnique": false + } + }, + "foreignKeys": { + "canonical_agent_ingest_checkpoints_thread_id_canonical_agent_threads_id_fk": { + "name": "canonical_agent_ingest_checkpoints_thread_id_canonical_agent_threads_id_fk", + "tableFrom": "canonical_agent_ingest_checkpoints", + "tableTo": "canonical_agent_threads", + "columnsFrom": [ + "thread_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "canonical_agent_ingest_checkpoints_turn_id_canonical_agent_turns_id_fk": { + "name": "canonical_agent_ingest_checkpoints_turn_id_canonical_agent_turns_id_fk", + "tableFrom": "canonical_agent_ingest_checkpoints", + "tableTo": "canonical_agent_turns", + "columnsFrom": [ + "turn_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "canonical_agent_items": { + "name": "canonical_agent_items", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "thread_id": { + "name": "thread_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "turn_id": { + "name": "turn_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "parent_item_id": { + "name": "parent_item_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "provider_identities_json": { + "name": "provider_identities_json", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'[]'" + }, + "payload_json": { + "name": "payload_json", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_canonical_agent_items_turn": { + "name": "idx_canonical_agent_items_turn", + "columns": [ + "turn_id", + "created_at" + ], + "isUnique": false + }, + "idx_canonical_agent_items_thread": { + "name": "idx_canonical_agent_items_thread", + "columns": [ + "thread_id", + "created_at" + ], + "isUnique": false + }, + "idx_canonical_agent_items_created_id": { + "name": "idx_canonical_agent_items_created_id", + "columns": [ + "created_at", + "id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "canonical_agent_items_thread_id_canonical_agent_threads_id_fk": { + "name": "canonical_agent_items_thread_id_canonical_agent_threads_id_fk", + "tableFrom": "canonical_agent_items", + "tableTo": "canonical_agent_threads", + "columnsFrom": [ + "thread_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "canonical_agent_items_turn_id_canonical_agent_turns_id_fk": { + "name": "canonical_agent_items_turn_id_canonical_agent_turns_id_fk", + "tableFrom": "canonical_agent_items", + "tableTo": "canonical_agent_turns", + "columnsFrom": [ + "turn_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "canonical_agent_threads": { + "name": "canonical_agent_threads", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "parent_thread_id": { + "name": "parent_thread_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "root_thread_id": { + "name": "root_thread_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "owning_parent_thread_id": { + "name": "owning_parent_thread_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "provider_identities_json": { + "name": "provider_identities_json", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'[]'" + }, + "activity_state": { + "name": "activity_state", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "conversation_revision": { + "name": "conversation_revision", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "roster_revision": { + "name": "roster_revision", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_canonical_agent_threads_workspace": { + "name": "idx_canonical_agent_threads_workspace", + "columns": [ + "workspace_id" + ], + "isUnique": false + }, + "idx_canonical_agent_threads_root": { + "name": "idx_canonical_agent_threads_root", + "columns": [ + "root_thread_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "canonical_agent_threads_id_threads_id_fk": { + "name": "canonical_agent_threads_id_threads_id_fk", + "tableFrom": "canonical_agent_threads", + "tableTo": "threads", + "columnsFrom": [ + "id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "canonical_agent_threads_workspace_id_workspaces_id_fk": { + "name": "canonical_agent_threads_workspace_id_workspaces_id_fk", + "tableFrom": "canonical_agent_threads", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "canonical_agent_turns": { + "name": "canonical_agent_turns", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "thread_id": { + "name": "thread_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "execution_id": { + "name": "execution_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "trigger_json": { + "name": "trigger_json", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "permission_mode": { + "name": "permission_mode", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "approval_review_mode": { + "name": "approval_review_mode", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'manual'" + }, + "approval_review_reason": { + "name": "approval_review_reason", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'manual-requested'" + }, + "provider_identities_json": { + "name": "provider_identities_json", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'[]'" + }, + "started_at": { + "name": "started_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "ended_at": { + "name": "ended_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_canonical_agent_turns_execution": { + "name": "idx_canonical_agent_turns_execution", + "columns": [ + "execution_id" + ], + "isUnique": true + }, + "idx_canonical_agent_turns_thread": { + "name": "idx_canonical_agent_turns_thread", + "columns": [ + "thread_id", + "created_at" + ], + "isUnique": false + } + }, + "foreignKeys": { + "canonical_agent_turns_thread_id_canonical_agent_threads_id_fk": { + "name": "canonical_agent_turns_thread_id_canonical_agent_threads_id_fk", + "tableFrom": "canonical_agent_turns", + "tableTo": "canonical_agent_threads", + "columnsFrom": [ + "thread_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "canonical_collaboration_actions": { + "name": "canonical_collaboration_actions", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source_thread_id": { + "name": "source_thread_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source_turn_id": { + "name": "source_turn_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source_item_id": { + "name": "source_item_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "target_thread_id": { + "name": "target_thread_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "target_turn_id": { + "name": "target_turn_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "delivery_unknown": { + "name": "delivery_unknown", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "message": { + "name": "message", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "provider_identities_json": { + "name": "provider_identities_json", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'[]'" + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_canonical_collaboration_source": { + "name": "idx_canonical_collaboration_source", + "columns": [ + "source_thread_id", + "created_at" + ], + "isUnique": false + }, + "idx_canonical_collaboration_target": { + "name": "idx_canonical_collaboration_target", + "columns": [ + "target_thread_id", + "created_at" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "canonical_conversation_display_mappings": { + "name": "canonical_conversation_display_mappings", + "columns": { + "source_item_id": { + "name": "source_item_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "target_kind": { + "name": "target_kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "target_id": { + "name": "target_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source_updated_at": { + "name": "source_updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + } + }, + "indexes": { + "idx_canonical_conversation_display_mappings_target": { + "name": "idx_canonical_conversation_display_mappings_target", + "columns": [ + "target_kind", + "target_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "canonical_conversation_display_mappings_source_item_id_canonical_agent_items_id_fk": { + "name": "canonical_conversation_display_mappings_source_item_id_canonical_agent_items_id_fk", + "tableFrom": "canonical_conversation_display_mappings", + "tableTo": "canonical_agent_items", + "columnsFrom": [ + "source_item_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "canonical_legacy_message_provenance": { + "name": "canonical_legacy_message_provenance", + "columns": { + "message_id": { + "name": "message_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "migration_version": { + "name": "migration_version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "mapping_status": { + "name": "mapping_status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "canonical_thread_id": { + "name": "canonical_thread_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "canonical_turn_id": { + "name": "canonical_turn_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "canonical_item_id": { + "name": "canonical_item_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "reason": { + "name": "reason", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + } + }, + "indexes": { + "idx_canonical_legacy_message_mapping": { + "name": "idx_canonical_legacy_message_mapping", + "columns": [ + "mapping_status", + "message_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "canonical_legacy_message_provenance_message_id_messages_id_fk": { + "name": "canonical_legacy_message_provenance_message_id_messages_id_fk", + "tableFrom": "canonical_legacy_message_provenance", + "tableTo": "messages", + "columnsFrom": [ + "message_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "canonical_legacy_migration_checkpoints": { + "name": "canonical_legacy_migration_checkpoints", + "columns": { + "version": { + "name": "version", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'pending'" + }, + "migrated_messages": { + "name": "migrated_messages", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "ambiguous_messages": { + "name": "ambiguous_messages", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + }, + "completed_at": { + "name": "completed_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "canonical_writer_operation_receipts": { + "name": "canonical_writer_operation_receipts", + "columns": { + "execution_id": { + "name": "execution_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "operation_id": { + "name": "operation_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "input_hash": { + "name": "input_hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "receipt_json": { + "name": "receipt_json", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + } + }, + "indexes": {}, + "foreignKeys": { + "canonical_writer_operation_receipts_execution_id_canonical_agent_turns_execution_id_fk": { + "name": "canonical_writer_operation_receipts_execution_id_canonical_agent_turns_execution_id_fk", + "tableFrom": "canonical_writer_operation_receipts", + "tableTo": "canonical_agent_turns", + "columnsFrom": [ + "execution_id" + ], + "columnsTo": [ + "execution_id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "canonical_writer_operation_receipts_execution_id_operation_id_pk": { + "columns": [ + "execution_id", + "operation_id" + ], + "name": "canonical_writer_operation_receipts_execution_id_operation_id_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "cleanup_jobs": { + "name": "cleanup_jobs", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "thread_id": { + "name": "thread_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "workspace_path": { + "name": "workspace_path", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "worktree_path": { + "name": "worktree_path", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "branch": { + "name": "branch", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'explicit'" + }, + "attempts": { + "name": "attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "next_retry_at": { + "name": "next_retry_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "last_error": { + "name": "last_error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "cleanup_jobs_thread_id_unique": { + "name": "cleanup_jobs_thread_id_unique", + "columns": [ + "thread_id" + ], + "isUnique": true + }, + "idx_cleanup_jobs_retry": { + "name": "idx_cleanup_jobs_retry", + "columns": [ + "next_retry_at", + "attempts", + "created_at" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "conversation_display_materialization_state": { + "name": "conversation_display_materialization_state", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "last_source_created_at": { + "name": "last_source_created_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_source_id": { + "name": "last_source_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "completed": { + "name": "completed", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "diff_summaries": { + "name": "diff_summaries", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "thread_id": { + "name": "thread_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "turn_count": { + "name": "turn_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "last_turn_id": { + "name": "last_turn_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "model": { + "name": "model", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + } + }, + "indexes": { + "idx_diff_summaries_thread": { + "name": "idx_diff_summaries_thread", + "columns": [ + "thread_id" + ], + "isUnique": true + } + }, + "foreignKeys": { + "diff_summaries_thread_id_threads_id_fk": { + "name": "diff_summaries_thread_id_threads_id_fk", + "tableFrom": "diff_summaries", + "tableTo": "threads", + "columnsFrom": [ + "thread_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "external_thread_control_deliveries": { + "name": "external_thread_control_deliveries", + "columns": { + "pairing_id": { + "name": "pairing_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "authority_epoch": { + "name": "authority_epoch", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "delivery_id": { + "name": "delivery_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "fingerprint": { + "name": "fingerprint", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'in_flight'" + }, + "result_json": { + "name": "result_json", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + }, + "expires_at": { + "name": "expires_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_external_thread_control_delivery_identity": { + "name": "idx_external_thread_control_delivery_identity", + "columns": [ + "pairing_id", + "authority_epoch", + "delivery_id" + ], + "isUnique": true + }, + "idx_external_thread_control_delivery_retention": { + "name": "idx_external_thread_control_delivery_retention", + "columns": [ + "pairing_id", + "status", + "expires_at" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "external_thread_control_pairings": { + "name": "external_thread_control_pairings", + "columns": { + "pairing_id": { + "name": "pairing_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "integration_id": { + "name": "integration_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "credential_hash": { + "name": "credential_hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "workspace_ids_json": { + "name": "workspace_ids_json", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'[]'" + }, + "scopes_json": { + "name": "scopes_json", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'[]'" + }, + "calls_per_minute": { + "name": "calls_per_minute", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "max_active_threads": { + "name": "max_active_threads", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'active'" + }, + "authority_epoch": { + "name": "authority_epoch", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + }, + "replaced_by_pairing_id": { + "name": "replaced_by_pairing_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "replaces_pairing_id": { + "name": "replaces_pairing_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "external_thread_control_pairings_credential_hash_unique": { + "name": "external_thread_control_pairings_credential_hash_unique", + "columns": [ + "credential_hash" + ], + "isUnique": true + }, + "idx_external_thread_control_pairings_integration": { + "name": "idx_external_thread_control_pairings_integration", + "columns": [ + "integration_id", + "status" + ], + "isUnique": false + }, + "idx_external_thread_control_active_integration": { + "name": "idx_external_thread_control_active_integration", + "columns": [ + "integration_id" + ], + "isUnique": true, + "where": "status = 'active'" + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "hook_executions": { + "name": "hook_executions", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "message_id": { + "name": "message_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "hook_name": { + "name": "hook_name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "tool_name": { + "name": "tool_name", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "phase": { + "name": "phase", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "payload": { + "name": "payload", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'{}'" + }, + "duration_ms": { + "name": "duration_ms", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "did_block": { + "name": "did_block", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "started_at": { + "name": "started_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + }, + "ended_at": { + "name": "ended_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "sort_order": { + "name": "sort_order", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + } + }, + "indexes": { + "idx_hook_executions_message_sort_order_id": { + "name": "idx_hook_executions_message_sort_order_id", + "columns": [ + "message_id", + "sort_order", + "id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "hook_executions_message_id_messages_id_fk": { + "name": "hook_executions_message_id_messages_id_fk", + "tableFrom": "hook_executions", + "tableTo": "messages", + "columnsFrom": [ + "message_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "messages": { + "name": "messages", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "thread_id": { + "name": "thread_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "tool_calls": { + "name": "tool_calls", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "files_changed": { + "name": "files_changed", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "cost_usd": { + "name": "cost_usd", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "tokens_used": { + "name": "tokens_used", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "timestamp": { + "name": "timestamp", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + }, + "sequence": { + "name": "sequence", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "attachments": { + "name": "attachments", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "preview_annotations": { + "name": "preview_annotations", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "mentions": { + "name": "mentions", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "selected_text_comments": { + "name": "selected_text_comments", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "reply_to_message_id": { + "name": "reply_to_message_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "quoted_text": { + "name": "quoted_text", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "model": { + "name": "model", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "origin_type": { + "name": "origin_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'legacy'" + }, + "source_thread_id": { + "name": "source_thread_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "source_turn_id": { + "name": "source_turn_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "source_provider_id": { + "name": "source_provider_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "legacy_provenance": { + "name": "legacy_provenance", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "parent_agent_provenance": { + "name": "parent_agent_provenance", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "is_internal": { + "name": "is_internal", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "outcome": { + "name": "outcome", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "outcome_execution_id": { + "name": "outcome_execution_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "system_notice": { + "name": "system_notice", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "idx_messages_thread": { + "name": "idx_messages_thread", + "columns": [ + "thread_id" + ], + "isUnique": false + }, + "idx_messages_sequence": { + "name": "idx_messages_sequence", + "columns": [ + "thread_id", + "sequence" + ], + "isUnique": false + }, + "idx_messages_thread_sequence_id": { + "name": "idx_messages_thread_sequence_id", + "columns": [ + "thread_id", + "sequence", + "id" + ], + "isUnique": false + }, + "idx_messages_notice_session_sequence": { + "name": "idx_messages_notice_session_sequence", + "columns": [ + "thread_id", + "json_extract(\"system_notice\", '$.sessionId')", + "\"sequence\" desc" + ], + "isUnique": false + } + }, + "foreignKeys": { + "messages_thread_id_threads_id_fk": { + "name": "messages_thread_id_threads_id_fk", + "tableFrom": "messages", + "tableTo": "threads", + "columnsFrom": [ + "thread_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "messages_reply_to_message_id_messages_id_fk": { + "name": "messages_reply_to_message_id_messages_id_fk", + "tableFrom": "messages", + "tableTo": "messages", + "columnsFrom": [ + "reply_to_message_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "parent_assistant_text_checkpoint_chunks": { + "name": "parent_assistant_text_checkpoint_chunks", + "columns": { + "execution_id": { + "name": "execution_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "first_sequence": { + "name": "first_sequence", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "last_sequence": { + "name": "last_sequence", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "text": { + "name": "text", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "byte_length": { + "name": "byte_length", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_parent_assistant_text_checkpoint_chunks_sequence": { + "name": "idx_parent_assistant_text_checkpoint_chunks_sequence", + "columns": [ + "execution_id", + "first_sequence" + ], + "isUnique": true + } + }, + "foreignKeys": { + "parent_assistant_text_checkpoint_chunks_execution_id_parent_assistant_text_checkpoints_execution_id_fk": { + "name": "parent_assistant_text_checkpoint_chunks_execution_id_parent_assistant_text_checkpoints_execution_id_fk", + "tableFrom": "parent_assistant_text_checkpoint_chunks", + "tableTo": "parent_assistant_text_checkpoints", + "columnsFrom": [ + "execution_id" + ], + "columnsTo": [ + "execution_id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "parent_assistant_text_checkpoints": { + "name": "parent_assistant_text_checkpoints", + "columns": { + "execution_id": { + "name": "execution_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "thread_id": { + "name": "thread_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "turn_id": { + "name": "turn_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "last_sequence": { + "name": "last_sequence", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "retained_bytes": { + "name": "retained_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "retained_chunks": { + "name": "retained_chunks", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_parent_assistant_text_checkpoints_thread": { + "name": "idx_parent_assistant_text_checkpoints_thread", + "columns": [ + "thread_id", + "updated_at" + ], + "isUnique": false + } + }, + "foreignKeys": { + "parent_assistant_text_checkpoints_thread_id_canonical_agent_threads_id_fk": { + "name": "parent_assistant_text_checkpoints_thread_id_canonical_agent_threads_id_fk", + "tableFrom": "parent_assistant_text_checkpoints", + "tableTo": "canonical_agent_threads", + "columnsFrom": [ + "thread_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "parent_assistant_text_checkpoints_turn_id_canonical_agent_turns_id_fk": { + "name": "parent_assistant_text_checkpoints_turn_id_canonical_agent_turns_id_fk", + "tableFrom": "parent_assistant_text_checkpoints", + "tableTo": "canonical_agent_turns", + "columnsFrom": [ + "turn_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "plan_question_answers": { + "name": "plan_question_answers", + "columns": { + "assistant_message_id": { + "name": "assistant_message_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "thread_id": { + "name": "thread_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "answered_at": { + "name": "answered_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + } + }, + "indexes": { + "idx_plan_question_answers_thread_answered_at": { + "name": "idx_plan_question_answers_thread_answered_at", + "columns": [ + "thread_id", + "answered_at" + ], + "isUnique": false + } + }, + "foreignKeys": { + "plan_question_answers_assistant_message_id_messages_id_fk": { + "name": "plan_question_answers_assistant_message_id_messages_id_fk", + "tableFrom": "plan_question_answers", + "tableTo": "messages", + "columnsFrom": [ + "assistant_message_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "plan_question_answers_thread_id_threads_id_fk": { + "name": "plan_question_answers_thread_id_threads_id_fk", + "tableFrom": "plan_question_answers", + "tableTo": "threads", + "columnsFrom": [ + "thread_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "plans": { + "name": "plans", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "thread_id": { + "name": "thread_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "message_id": { + "name": "message_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "version": { + "name": "version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 1 + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "content_md": { + "name": "content_md", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "sections_json": { + "name": "sections_json", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "change_summary": { + "name": "change_summary", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'draft'" + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + } + }, + "indexes": { + "idx_plans_thread": { + "name": "idx_plans_thread", + "columns": [ + "thread_id" + ], + "isUnique": false + }, + "idx_plans_thread_version": { + "name": "idx_plans_thread_version", + "columns": [ + "thread_id", + "version" + ], + "isUnique": false + } + }, + "foreignKeys": { + "plans_thread_id_threads_id_fk": { + "name": "plans_thread_id_threads_id_fk", + "tableFrom": "plans", + "tableTo": "threads", + "columnsFrom": [ + "thread_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "plans_message_id_messages_id_fk": { + "name": "plans_message_id_messages_id_fk", + "tableFrom": "plans", + "tableTo": "messages", + "columnsFrom": [ + "message_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "project_action_runs": { + "name": "project_action_runs", + "columns": { + "thread_id": { + "name": "thread_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "action_id": { + "name": "action_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "run_id": { + "name": "run_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "revision": { + "name": "revision", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "terminal_session_id": { + "name": "terminal_session_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "action_name": { + "name": "action_name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "snapshot_json": { + "name": "snapshot_json", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "started_at": { + "name": "started_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "finished_at": { + "name": "finished_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "exit_code": { + "name": "exit_code", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "transcript": { + "name": "transcript", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "transcript_truncated": { + "name": "transcript_truncated", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + } + }, + "indexes": { + "idx_project_action_runs_slot": { + "name": "idx_project_action_runs_slot", + "columns": [ + "thread_id", + "action_id" + ], + "isUnique": true + }, + "idx_project_action_runs_thread": { + "name": "idx_project_action_runs_thread", + "columns": [ + "thread_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "project_action_runs_thread_id_threads_id_fk": { + "name": "project_action_runs_thread_id_threads_id_fk", + "tableFrom": "project_action_runs", + "tableTo": "threads", + "columnsFrom": [ + "thread_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "project_action_runs_workspace_id_workspaces_id_fk": { + "name": "project_action_runs_workspace_id_workspaces_id_fk", + "tableFrom": "project_action_runs", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "provider_catalog_snapshots": { + "name": "provider_catalog_snapshots", + "columns": { + "context_key": { + "name": "context_key", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "cwd": { + "name": "cwd", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "snapshot_json": { + "name": "snapshot_json", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + } + }, + "indexes": { + "idx_provider_catalog_snapshots_workspace": { + "name": "idx_provider_catalog_snapshots_workspace", + "columns": [ + "workspace_id" + ], + "isUnique": false + }, + "idx_provider_catalog_snapshots_provider": { + "name": "idx_provider_catalog_snapshots_provider", + "columns": [ + "provider_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "provider_catalog_snapshots_workspace_id_workspaces_id_fk": { + "name": "provider_catalog_snapshots_workspace_id_workspaces_id_fk", + "tableFrom": "provider_catalog_snapshots", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "provider_model_cache": { + "name": "provider_model_cache", + "columns": { + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "models_json": { + "name": "models_json", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "fetched_at": { + "name": "fetched_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + }, + "model_count": { + "name": "model_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "pull_request_review_links": { + "name": "pull_request_review_links", + "columns": { + "worktree_id": { + "name": "worktree_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "repository_node_id": { + "name": "repository_node_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "pull_request_number": { + "name": "pull_request_number", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "pr_url": { + "name": "pr_url", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "pr_state": { + "name": "pr_state", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "worktree_path": { + "name": "worktree_path", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "worktree_managed": { + "name": "worktree_managed", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 1 + }, + "head_repository_node_id": { + "name": "head_repository_node_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "head_repository_owner": { + "name": "head_repository_owner", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "head_repository_name": { + "name": "head_repository_name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "head_ref": { + "name": "head_ref", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "head_oid": { + "name": "head_oid", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "local_branch": { + "name": "local_branch", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "push_remote": { + "name": "push_remote", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "push_ref": { + "name": "push_ref", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "managed_remote_name": { + "name": "managed_remote_name", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "primary_thread_id": { + "name": "primary_thread_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + } + }, + "indexes": { + "idx_pull_request_review_links_identity": { + "name": "idx_pull_request_review_links_identity", + "columns": [ + "provider", + "repository_node_id", + "pull_request_number" + ], + "isUnique": true + }, + "idx_pull_request_review_links_primary_thread": { + "name": "idx_pull_request_review_links_primary_thread", + "columns": [ + "primary_thread_id" + ], + "isUnique": true + }, + "idx_pull_request_review_links_workspace": { + "name": "idx_pull_request_review_links_workspace", + "columns": [ + "workspace_id" + ], + "isUnique": false + }, + "idx_pull_request_review_links_worktree_path": { + "name": "idx_pull_request_review_links_worktree_path", + "columns": [ + "worktree_path" + ], + "isUnique": false + } + }, + "foreignKeys": { + "pull_request_review_links_workspace_id_workspaces_id_fk": { + "name": "pull_request_review_links_workspace_id_workspaces_id_fk", + "tableFrom": "pull_request_review_links", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "pull_request_review_links_primary_thread_id_threads_id_fk": { + "name": "pull_request_review_links_primary_thread_id_threads_id_fk", + "tableFrom": "pull_request_review_links", + "tableTo": "threads", + "columnsFrom": [ + "primary_thread_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "terminal_cleanup_ledger": { + "name": "terminal_cleanup_ledger", + "columns": { + "session_id": { + "name": "session_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "host_generation": { + "name": "host_generation", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "root_pid": { + "name": "root_pid", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "process_group_id": { + "name": "process_group_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "containment": { + "name": "containment", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "thought_segments": { + "name": "thought_segments", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "message_id": { + "name": "message_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "text": { + "name": "text", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "started_at": { + "name": "started_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + }, + "ended_at": { + "name": "ended_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "sort_order": { + "name": "sort_order", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "is_final_response": { + "name": "is_final_response", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + } + }, + "indexes": { + "idx_thought_segments_message_final_sort_order_id": { + "name": "idx_thought_segments_message_final_sort_order_id", + "columns": [ + "message_id", + "is_final_response", + "sort_order", + "id" + ], + "isUnique": false + }, + "idx_thought_segments_final_message_sort_order_id": { + "name": "idx_thought_segments_final_message_sort_order_id", + "columns": [ + "message_id", + "sort_order", + "id" + ], + "isUnique": false, + "where": "\"thought_segments\".\"is_final_response\" <> 0" + } + }, + "foreignKeys": { + "thought_segments_message_id_messages_id_fk": { + "name": "thought_segments_message_id_messages_id_fk", + "tableFrom": "thought_segments", + "tableTo": "messages", + "columnsFrom": [ + "message_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "thread_control_approvals": { + "name": "thread_control_approvals", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "thread_id": { + "name": "thread_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "prompt": { + "name": "prompt", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "execution_json": { + "name": "execution_json", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "placement_json": { + "name": "placement_json", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "turn_id": { + "name": "turn_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "caller_id": { + "name": "caller_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "source_thread_id": { + "name": "source_thread_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "source_turn_id": { + "name": "source_turn_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "source_provider_id": { + "name": "source_provider_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "operation": { + "name": "operation", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'thread_create_batch'" + }, + "operation_phase": { + "name": "operation_phase", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'pre_provision'" + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'pending'" + }, + "processing_started_at": { + "name": "processing_started_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + }, + "resolved_at": { + "name": "resolved_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "idx_thread_control_approvals_thread": { + "name": "idx_thread_control_approvals_thread", + "columns": [ + "thread_id", + "status" + ], + "isUnique": false + } + }, + "foreignKeys": { + "thread_control_approvals_thread_id_threads_id_fk": { + "name": "thread_control_approvals_thread_id_threads_id_fk", + "tableFrom": "thread_control_approvals", + "tableTo": "threads", + "columnsFrom": [ + "thread_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "thread_control_approvals_workspace_id_workspaces_id_fk": { + "name": "thread_control_approvals_workspace_id_workspaces_id_fk", + "tableFrom": "thread_control_approvals", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "thread_control_audit": { + "name": "thread_control_audit", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "caller_id": { + "name": "caller_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source_thread_id": { + "name": "source_thread_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "thread_id": { + "name": "thread_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "operation": { + "name": "operation", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "outcome": { + "name": "outcome", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + } + }, + "indexes": { + "idx_thread_control_audit_thread": { + "name": "idx_thread_control_audit_thread", + "columns": [ + "thread_id", + "created_at" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "thread_startups": { + "name": "thread_startups", + "columns": { + "startup_id": { + "name": "startup_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "phase": { + "name": "phase", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "steps_json": { + "name": "steps_json", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "transcript_json": { + "name": "transcript_json", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "cancellation": { + "name": "cancellation", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'none'" + }, + "revision": { + "name": "revision", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "request_fingerprint": { + "name": "request_fingerprint", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "thread_id": { + "name": "thread_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "error_json": { + "name": "error_json", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "block_json": { + "name": "block_json", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + } + }, + "indexes": { + "idx_thread_startups_workspace_updated": { + "name": "idx_thread_startups_workspace_updated", + "columns": [ + "workspace_id", + "\"updated_at\" desc" + ], + "isUnique": false + }, + "idx_thread_startups_state": { + "name": "idx_thread_startups_state", + "columns": [ + "state" + ], + "isUnique": false + } + }, + "foreignKeys": { + "thread_startups_workspace_id_workspaces_id_fk": { + "name": "thread_startups_workspace_id_workspaces_id_fk", + "tableFrom": "thread_startups", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "thread_startups_thread_id_threads_id_fk": { + "name": "thread_startups_thread_id_threads_id_fk", + "tableFrom": "thread_startups", + "tableTo": "threads", + "columnsFrom": [ + "thread_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "thread_tasks": { + "name": "thread_tasks", + "columns": { + "thread_id": { + "name": "thread_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "tasks_json": { + "name": "tasks_json", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + } + }, + "indexes": {}, + "foreignKeys": { + "thread_tasks_thread_id_threads_id_fk": { + "name": "thread_tasks_thread_id_threads_id_fk", + "tableFrom": "thread_tasks", + "tableTo": "threads", + "columnsFrom": [ + "thread_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "threads": { + "name": "threads", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'active'" + }, + "mode": { + "name": "mode", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'direct'" + }, + "worktree_path": { + "name": "worktree_path", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "branch": { + "name": "branch", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "checkout_state": { + "name": "checkout_state", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'named'" + }, + "base_branch": { + "name": "base_branch", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "issue_number": { + "name": "issue_number", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "pr_number": { + "name": "pr_number", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "pr_status": { + "name": "pr_status", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + }, + "deleted_at": { + "name": "deleted_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "user_completed_at": { + "name": "user_completed_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "scheduled_deletion_at": { + "name": "scheduled_deletion_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "cleanup_state": { + "name": "cleanup_state", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "cleanup_reason": { + "name": "cleanup_reason", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "model": { + "name": "model", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "worktree_managed": { + "name": "worktree_managed", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 1 + }, + "sdk_session_id": { + "name": "sdk_session_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_context_tokens": { + "name": "last_context_tokens", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "context_window": { + "name": "context_window", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'claude'" + }, + "reasoning_level": { + "name": "reasoning_level", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "interaction_mode": { + "name": "interaction_mode", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "orchestration_mode": { + "name": "orchestration_mode", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "permission_mode": { + "name": "permission_mode", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "parent_thread_id": { + "name": "parent_thread_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "forked_from_message_id": { + "name": "forked_from_message_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "delegation_coordinator_thread_id": { + "name": "delegation_coordinator_thread_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "delegation_creator_turn_id": { + "name": "delegation_creator_turn_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "delegation_creator_tool_call_id": { + "name": "delegation_creator_tool_call_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "delegation_creation_kind": { + "name": "delegation_creation_kind", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_by_integration_id": { + "name": "created_by_integration_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_compact_summary": { + "name": "last_compact_summary", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "copilot_agent": { + "name": "copilot_agent", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "devin_mode": { + "name": "devin_mode", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "context_window_mode": { + "name": "context_window_mode", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "thinking": { + "name": "thinking", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "codex_fast_mode": { + "name": "codex_fast_mode", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "default_open_in_app": { + "name": "default_open_in_app", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "has_file_changes": { + "name": "has_file_changes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "current_notice_session_id": { + "name": "current_notice_session_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "idx_threads_workspace": { + "name": "idx_threads_workspace", + "columns": [ + "workspace_id" + ], + "isUnique": false + }, + "idx_threads_workspace_deleted": { + "name": "idx_threads_workspace_deleted", + "columns": [ + "workspace_id", + "deleted_at" + ], + "isUnique": false + }, + "idx_threads_workspace_completed": { + "name": "idx_threads_workspace_completed", + "columns": [ + "workspace_id", + "user_completed_at" + ], + "isUnique": false + }, + "idx_threads_cleanup_due": { + "name": "idx_threads_cleanup_due", + "columns": [ + "cleanup_state", + "scheduled_deletion_at" + ], + "isUnique": false + }, + "idx_threads_workspace_recency": { + "name": "idx_threads_workspace_recency", + "columns": [ + "workspace_id", + "\"updated_at\" desc" + ], + "isUnique": false + }, + "idx_threads_status": { + "name": "idx_threads_status", + "columns": [ + "status" + ], + "isUnique": false + }, + "idx_threads_parent_thread_id": { + "name": "idx_threads_parent_thread_id", + "columns": [ + "parent_thread_id" + ], + "isUnique": false + }, + "idx_threads_forked_from_message_id": { + "name": "idx_threads_forked_from_message_id", + "columns": [ + "forked_from_message_id" + ], + "isUnique": false + }, + "idx_threads_delegation_coordinator": { + "name": "idx_threads_delegation_coordinator", + "columns": [ + "delegation_coordinator_thread_id" + ], + "isUnique": false + }, + "idx_threads_created_by_integration": { + "name": "idx_threads_created_by_integration", + "columns": [ + "created_by_integration_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "threads_workspace_id_workspaces_id_fk": { + "name": "threads_workspace_id_workspaces_id_fk", + "tableFrom": "threads", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "threads_delegation_coordinator_thread_id_threads_id_fk": { + "name": "threads_delegation_coordinator_thread_id_threads_id_fk", + "tableFrom": "threads", + "tableTo": "threads", + "columnsFrom": [ + "delegation_coordinator_thread_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "tool_call_records": { + "name": "tool_call_records", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "message_id": { + "name": "message_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "parent_tool_call_id": { + "name": "parent_tool_call_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "tool_name": { + "name": "tool_name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "display_name": { + "name": "display_name", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "provider_agent_key": { + "name": "provider_agent_key", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "subagent_identity_key": { + "name": "subagent_identity_key", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "subagent_provider_name": { + "name": "subagent_provider_name", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "subagent_prompt": { + "name": "subagent_prompt", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "subagent_type": { + "name": "subagent_type", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "subagent_agent_id": { + "name": "subagent_agent_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "subagent_duration_ms": { + "name": "subagent_duration_ms", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "model": { + "name": "model", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "reasoning_effort": { + "name": "reasoning_effort", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "input_summary": { + "name": "input_summary", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "output_summary": { + "name": "output_summary", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "output_truncated": { + "name": "output_truncated", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "output_total_bytes": { + "name": "output_total_bytes", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "output_artifact_path": { + "name": "output_artifact_path", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "exit_code": { + "name": "exit_code", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'running'" + }, + "started_at": { + "name": "started_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + }, + "completed_at": { + "name": "completed_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "sort_order": { + "name": "sort_order", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + } + }, + "indexes": { + "idx_tool_call_records_message_sort_order_id": { + "name": "idx_tool_call_records_message_sort_order_id", + "columns": [ + "message_id", + "sort_order", + "id" + ], + "isUnique": false + }, + "idx_tool_call_records_parent": { + "name": "idx_tool_call_records_parent", + "columns": [ + "parent_tool_call_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "tool_call_records_message_id_messages_id_fk": { + "name": "tool_call_records_message_id_messages_id_fk", + "tableFrom": "tool_call_records", + "tableTo": "messages", + "columnsFrom": [ + "message_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "turn_diff_snapshots": { + "name": "turn_diff_snapshots", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "message_id": { + "name": "message_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "thread_id": { + "name": "thread_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source": { + "name": "source", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "fidelity": { + "name": "fidelity", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "patch": { + "name": "patch", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "revision": { + "name": "revision", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + } + }, + "indexes": { + "idx_turn_diff_snapshots_message": { + "name": "idx_turn_diff_snapshots_message", + "columns": [ + "message_id" + ], + "isUnique": true + }, + "idx_turn_diff_snapshots_thread": { + "name": "idx_turn_diff_snapshots_thread", + "columns": [ + "thread_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "turn_diff_snapshots_message_id_messages_id_fk": { + "name": "turn_diff_snapshots_message_id_messages_id_fk", + "tableFrom": "turn_diff_snapshots", + "tableTo": "messages", + "columnsFrom": [ + "message_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "turn_diff_snapshots_thread_id_threads_id_fk": { + "name": "turn_diff_snapshots_thread_id_threads_id_fk", + "tableFrom": "turn_diff_snapshots", + "tableTo": "threads", + "columnsFrom": [ + "thread_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "turn_snapshots": { + "name": "turn_snapshots", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "message_id": { + "name": "message_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "thread_id": { + "name": "thread_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ref_before": { + "name": "ref_before", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ref_after": { + "name": "ref_after", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "files_changed": { + "name": "files_changed", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'[]'" + }, + "file_effects": { + "name": "file_effects", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'{\"revision\":0,\"fileCount\":0,\"additions\":0,\"deletions\":0,\"effects\":[]}'" + }, + "worktree_path": { + "name": "worktree_path", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + } + }, + "indexes": { + "idx_turn_snapshots_message": { + "name": "idx_turn_snapshots_message", + "columns": [ + "message_id" + ], + "isUnique": false + }, + "idx_turn_snapshots_thread": { + "name": "idx_turn_snapshots_thread", + "columns": [ + "thread_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "turn_snapshots_message_id_messages_id_fk": { + "name": "turn_snapshots_message_id_messages_id_fk", + "tableFrom": "turn_snapshots", + "tableTo": "messages", + "columnsFrom": [ + "message_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "turn_snapshots_thread_id_threads_id_fk": { + "name": "turn_snapshots_thread_id_threads_id_fk", + "tableFrom": "turn_snapshots", + "tableTo": "threads", + "columnsFrom": [ + "thread_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "workspace_environment_automatic_setup_attempts": { + "name": "workspace_environment_automatic_setup_attempts", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "thread_id": { + "name": "thread_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "reason": { + "name": "reason", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "launch_snapshot_json": { + "name": "launch_snapshot_json", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "outcome": { + "name": "outcome", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + }, + "started_at": { + "name": "started_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "finished_at": { + "name": "finished_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "exit_code": { + "name": "exit_code", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "output": { + "name": "output", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "output_truncated": { + "name": "output_truncated", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + } + }, + "indexes": { + "idx_workspace_environment_automatic_setup_attempts_thread": { + "name": "idx_workspace_environment_automatic_setup_attempts_thread", + "columns": [ + "thread_id", + "\"created_at\" desc" + ], + "isUnique": false + } + }, + "foreignKeys": { + "workspace_environment_automatic_setup_attempts_thread_id_threads_id_fk": { + "name": "workspace_environment_automatic_setup_attempts_thread_id_threads_id_fk", + "tableFrom": "workspace_environment_automatic_setup_attempts", + "tableTo": "threads", + "columnsFrom": [ + "thread_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "workspace_environment_command_approvals": { + "name": "workspace_environment_command_approvals", + "columns": { + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "command_id": { + "name": "command_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "fingerprint": { + "name": "fingerprint", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "approved_at": { + "name": "approved_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + } + }, + "indexes": { + "idx_workspace_environment_command_approvals_command": { + "name": "idx_workspace_environment_command_approvals_command", + "columns": [ + "workspace_id", + "command_id" + ], + "isUnique": true + } + }, + "foreignKeys": { + "workspace_environment_command_approvals_workspace_id_workspaces_id_fk": { + "name": "workspace_environment_command_approvals_workspace_id_workspaces_id_fk", + "tableFrom": "workspace_environment_command_approvals", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "workspace_environment_queued_turns": { + "name": "workspace_environment_queued_turns", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "thread_id": { + "name": "thread_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "message_id": { + "name": "message_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "queue_position": { + "name": "queue_position", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "submission_json": { + "name": "submission_json", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + }, + "released_at": { + "name": "released_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "dispatching_at": { + "name": "dispatching_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "dispatched_at": { + "name": "dispatched_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "idx_workspace_environment_queued_turns_state": { + "name": "idx_workspace_environment_queued_turns_state", + "columns": [ + "state", + "created_at" + ], + "isUnique": false + }, + "idx_workspace_environment_queued_turns_thread_state_position": { + "name": "idx_workspace_environment_queued_turns_thread_state_position", + "columns": [ + "thread_id", + "state", + "queue_position" + ], + "isUnique": false + } + }, + "foreignKeys": { + "workspace_environment_queued_turns_thread_id_threads_id_fk": { + "name": "workspace_environment_queued_turns_thread_id_threads_id_fk", + "tableFrom": "workspace_environment_queued_turns", + "tableTo": "threads", + "columnsFrom": [ + "thread_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "workspace_environment_setup_gates": { + "name": "workspace_environment_setup_gates", + "columns": { + "thread_id": { + "name": "thread_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "attempt_id": { + "name": "attempt_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + } + }, + "indexes": {}, + "foreignKeys": { + "workspace_environment_setup_gates_thread_id_threads_id_fk": { + "name": "workspace_environment_setup_gates_thread_id_threads_id_fk", + "tableFrom": "workspace_environment_setup_gates", + "tableTo": "threads", + "columnsFrom": [ + "thread_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "workspace_environment_storage_settings": { + "name": "workspace_environment_storage_settings", + "columns": { + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "storage_mode": { + "name": "storage_mode", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + } + }, + "indexes": {}, + "foreignKeys": { + "workspace_environment_storage_settings_workspace_id_workspaces_id_fk": { + "name": "workspace_environment_storage_settings_workspace_id_workspaces_id_fk", + "tableFrom": "workspace_environment_storage_settings", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "workspace_terminal_preferences": { + "name": "workspace_terminal_preferences", + "columns": { + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "default_profile_id": { + "name": "default_profile_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + } + }, + "indexes": {}, + "foreignKeys": { + "workspace_terminal_preferences_workspace_id_workspaces_id_fk": { + "name": "workspace_terminal_preferences_workspace_id_workspaces_id_fk", + "tableFrom": "workspace_terminal_preferences", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "workspace_worktrees": { + "name": "workspace_worktrees", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "canonical_path": { + "name": "canonical_path", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "label": { + "name": "label", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "branch": { + "name": "branch", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "base_ref": { + "name": "base_ref", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "managed": { + "name": "managed", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "last_seen_at": { + "name": "last_seen_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + }, + "stale": { + "name": "stale", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + } + }, + "indexes": { + "idx_workspace_worktrees_path": { + "name": "idx_workspace_worktrees_path", + "columns": [ + "workspace_id", + "canonical_path" + ], + "isUnique": true + }, + "idx_workspace_worktrees_workspace": { + "name": "idx_workspace_worktrees_workspace", + "columns": [ + "workspace_id", + "stale" + ], + "isUnique": false + } + }, + "foreignKeys": { + "workspace_worktrees_workspace_id_workspaces_id_fk": { + "name": "workspace_worktrees_workspace_id_workspaces_id_fk", + "tableFrom": "workspace_worktrees", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "workspaces": { + "name": "workspaces", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "path": { + "name": "path", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "provider_config": { + "name": "provider_config", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'{}'" + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + }, + "pinned": { + "name": "pinned", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "last_opened_at": { + "name": "last_opened_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "sort_order": { + "name": "sort_order", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "is_git_repo": { + "name": "is_git_repo", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 1 + }, + "deleted_at": { + "name": "deleted_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "workspaces_path_unique": { + "name": "workspaces_path_unique", + "columns": [ + "path" + ], + "isUnique": true + }, + "idx_workspaces_sort_order": { + "name": "idx_workspaces_sort_order", + "columns": [ + "\"sort_order\" asc" + ], + "isUnique": false + }, + "idx_workspaces_pinned_last_opened": { + "name": "idx_workspaces_pinned_last_opened", + "columns": [ + "\"pinned\" desc", + "\"last_opened_at\" desc" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + } + }, + "views": {}, + "enums": {}, + "_meta": { + "schemas": {}, + "tables": {}, + "columns": {} + }, + "internal": { + "indexes": { + "idx_messages_notice_session_sequence": { + "columns": { + "json_extract(\"system_notice\", '$.sessionId')": { + "isExpression": true + }, + "\"sequence\" desc": { + "isExpression": true + } + } + }, + "idx_thread_startups_workspace_updated": { + "columns": { + "\"updated_at\" desc": { + "isExpression": true + } + } + }, + "idx_threads_workspace_recency": { + "columns": { + "\"updated_at\" desc": { + "isExpression": true + } + } + }, + "idx_workspace_environment_automatic_setup_attempts_thread": { + "columns": { + "\"created_at\" desc": { + "isExpression": true + } + } + }, + "idx_workspaces_sort_order": { + "columns": { + "\"sort_order\" asc": { + "isExpression": true + } + } + }, + "idx_workspaces_pinned_last_opened": { + "columns": { + "\"pinned\" desc": { + "isExpression": true + }, + "\"last_opened_at\" desc": { + "isExpression": true + } + } + } + } + } +} \ No newline at end of file diff --git a/apps/server/drizzle/meta/_journal.json b/apps/server/drizzle/meta/_journal.json index af57bc677..9aef29df4 100644 --- a/apps/server/drizzle/meta/_journal.json +++ b/apps/server/drizzle/meta/_journal.json @@ -463,6 +463,13 @@ "when": 1790262706352, "tag": "0064_fuzzy_jimmy_woo", "breakpoints": true + }, + { + "idx": 65, + "version": "6", + "when": 1790275595252, + "tag": "0065_ambitious_blindfold", + "breakpoints": true } ] } \ No newline at end of file diff --git a/apps/server/src/features/agents/canonical/__tests__/canonical-agent-writer-client.test.ts b/apps/server/src/features/agents/canonical/__tests__/canonical-agent-writer-client.test.ts index 8fb23edce..4c170931a 100644 --- a/apps/server/src/features/agents/canonical/__tests__/canonical-agent-writer-client.test.ts +++ b/apps/server/src/features/agents/canonical/__tests__/canonical-agent-writer-client.test.ts @@ -8,6 +8,7 @@ import type { ParentNarrativeRecoveryItem } from "@mcode/contracts"; import { openDatabase } from "../../../../runtime/persistence/sqlite/database.js"; import type { CanonicalAgentEventDraft } from "../canonical-agent-boundary.js"; import { CanonicalAgentWriterClient } from "../canonical-agent-writer-client.js"; +import type { CanonicalWriterResponse } from "../canonical-agent-writer-protocol.js"; import { ParentAssistantTextCheckpointService } from "../../turns/parent-assistant-text-checkpoint-service.js"; const THREAD_ID = "writer-thread"; @@ -15,6 +16,27 @@ const TURN_ID = "writer-turn"; const EXECUTION_ID = "00000000-0000-4000-8000-000000000176"; const NOW = "2026-09-24T12:00:00.000Z"; +function workerDroppingReply(kind: CanonicalWriterResponse["kind"]): Worker { + const worker = new Worker(new URL("../canonical-agent-writer.worker.ts", import.meta.url), { type: "module" }); + return new Proxy(worker, { + set(target, property, value) { + if (property !== "onmessage" || typeof value !== "function") return Reflect.set(target, property, value); + target.onmessage = (message) => { + if (message.data.kind === kind) { + target.terminate(); + return; + } + value(message); + }; + return true; + }, + get(target, property) { + const value: unknown = Reflect.get(target, property, target); + return typeof value === "function" ? value.bind(target) : value; + }, + }); +} + function events(): CanonicalAgentEventDraft[] { const sourceIdentities = [{ providerId: "codex" as const, scope: "thread" as const, value: "native-writer-thread", provenance: "native" as const }]; return [ @@ -125,7 +147,7 @@ describe("canonical SQLite writer", () => { await NodeFSPromises.rm(tempDir, { recursive: true, force: true }); }); - it("acknowledges only committed events and deduplicates a replay", async () => { + it("replays the original committed receipt and full envelopes", async () => { writer = new CanonicalAgentWriterClient(dbPath); const batch = { threadId: THREAD_ID, turnId: TURN_ID, executionId: EXECUTION_ID, phase: "running", events: events() }; const first = await writer.commit("writer-operation-1", batch); @@ -135,8 +157,9 @@ describe("canonical SQLite writer", () => { expect(db.prepare("SELECT COUNT(*) AS count FROM canonical_agent_events").get()).toEqual({ count: 3 }); const replay = await writer.commit("writer-operation-1", batch); - expect(replay).toMatchObject({ outcome: "duplicate", acceptedThrough: 3, durableThrough: 3, events: [] }); + expect(replay).toEqual(first); expect(db.prepare("SELECT COUNT(*) AS count FROM canonical_agent_events").get()).toEqual({ count: 3 }); + expect(db.prepare("SELECT COUNT(*) AS count FROM canonical_writer_operation_receipts").get()).toEqual({ count: 1 }); }); it("rejects a database failure without reporting a durable receipt", async () => { @@ -177,6 +200,11 @@ describe("canonical SQLite writer", () => { })).toEqual({ recorded: true }); expect(db.prepare("SELECT COUNT(*) AS count FROM canonical_agent_items WHERE id = ?") .get("toolCall:writer-recovery-tool")).toEqual({ count: 0 }); + expect(await writer.recordParentNarrativeRecovery("narrative-discard", { + executionId: EXECUTION_ID, + items: [], + discardedItemIds: ["toolCall:writer-recovery-tool"], + })).toEqual({ recorded: true }); }); it("rejects a failed recovery write without a durability acknowledgement", async () => { @@ -216,8 +244,8 @@ describe("canonical SQLite writer", () => { .get(EXECUTION_ID)).toEqual({ count: 0 }); expect(db.prepare("SELECT COUNT(*) AS count FROM canonical_agent_items WHERE id = ?") .get("toolCall:writer-recovery-tool")).toEqual({ count: 1 }); - await expect(writer.classifyParentNarrativeRecovery("classification-1", input)) - .rejects.toThrow("Canonical writer write-failed"); + expect(await writer.classifyParentNarrativeRecovery("classification-1", input)) + .toEqual({ recorded: true, reset: true }); expect(db.prepare("SELECT COUNT(*) AS count FROM canonical_agent_items WHERE id = ?") .get("toolCall:writer-recovery-tool")).toEqual({ count: 1 }); }); @@ -262,7 +290,7 @@ describe("canonical SQLite writer", () => { } }); - it("rejects future writes if the worker exits", async () => { + it("restarts a closed worker and accepts later writes", async () => { let worker: Worker | undefined; writer = new CanonicalAgentWriterClient(dbPath, () => { worker = new Worker(new URL("../canonical-agent-writer.worker.ts", import.meta.url), { type: "module" }); @@ -273,6 +301,115 @@ describe("canonical SQLite writer", () => { const closed = new Promise((resolve) => worker?.addEventListener("close", resolve, { once: true })); worker?.terminate(); await closed; - await expect(writer.commit("writer-operation-4", batch)).rejects.toThrow("Canonical writer worker closed"); + expect(await writer.commit("writer-operation-4", batch)).toMatchObject({ outcome: "duplicate" }); + }); + + it("replays the committed envelope after a response is lost with the worker", async () => { + let created = 0; + writer = new CanonicalAgentWriterClient(dbPath, () => { + return created++ === 0 ? workerDroppingReply("committed") + : new Worker(new URL("../canonical-agent-writer.worker.ts", import.meta.url), { type: "module" }); + }); + const batch = { threadId: THREAD_ID, turnId: TURN_ID, executionId: EXECUTION_ID, phase: "running", events: events() }; + const receipt = await writer.commit("lost-committed-response", batch); + expect(receipt).toMatchObject({ outcome: "committed", acceptedThrough: 3 }); + expect(receipt.events.map((event) => event.eventId)).toEqual(batch.events.map((event) => event.eventId)); + expect(created).toBe(2); + expect(db.prepare("SELECT COUNT(*) AS count FROM canonical_agent_events").get()).toEqual({ count: 3 }); + }); + + it("replays a classification after its reset committed but the reply was lost", async () => { + writer = new CanonicalAgentWriterClient(dbPath); + await writer.commit("classification-start", { + threadId: THREAD_ID, turnId: TURN_ID, executionId: EXECUTION_ID, phase: "running", events: events(), + }); + new ParentAssistantTextCheckpointService(db).appendChunk([{ + executionId: EXECUTION_ID, threadId: THREAD_ID, turnId: TURN_ID, sequence: 1, text: "provisional", + }]); + await writer.close(); + let created = 0; + writer = new CanonicalAgentWriterClient(dbPath, () => { + return created++ === 0 ? workerDroppingReply("parent-narrative-recovery-classified") + : new Worker(new URL("../canonical-agent-writer.worker.ts", import.meta.url), { type: "module" }); + }); + expect(await writer.classifyParentNarrativeRecovery("lost-classification", { + executionId: EXECUTION_ID, items: [recoveryToolCall()], + })).toEqual({ recorded: true, reset: true }); + expect(created).toBe(2); + expect(db.prepare("SELECT COUNT(*) AS count FROM parent_assistant_text_checkpoints WHERE execution_id = ?") + .get(EXECUTION_ID)).toEqual({ count: 0 }); + expect(db.prepare("SELECT COUNT(*) AS count FROM canonical_agent_items WHERE id = ?") + .get("toolCall:writer-recovery-tool")).toEqual({ count: 1 }); + }); + + it("replays a recovery discard after its reply was lost", async () => { + writer = new CanonicalAgentWriterClient(dbPath); + await writer.commit("recovery-start", { + threadId: THREAD_ID, turnId: TURN_ID, executionId: EXECUTION_ID, phase: "running", events: events(), + }); + await writer.recordParentNarrativeRecovery("recovery-record", { + executionId: EXECUTION_ID, items: [recoveryToolCall()], + }); + await writer.close(); + let created = 0; + writer = new CanonicalAgentWriterClient(dbPath, () => created++ === 0 + ? workerDroppingReply("parent-narrative-recovery-recorded") + : new Worker(new URL("../canonical-agent-writer.worker.ts", import.meta.url), { type: "module" })); + expect(await writer.recordParentNarrativeRecovery("lost-recovery-discard", { + executionId: EXECUTION_ID, items: [], discardedItemIds: ["toolCall:writer-recovery-tool"], + })).toEqual({ recorded: true }); + expect(created).toBe(2); + expect(db.prepare("SELECT COUNT(*) AS count FROM canonical_agent_items WHERE id = ?") + .get("toolCall:writer-recovery-tool")).toEqual({ count: 0 }); }); + + it("stops after three lost replies and permits explicit replay later", async () => { + let created = 0; + writer = new CanonicalAgentWriterClient(dbPath, () => { + created++; + return workerDroppingReply("committed"); + }); + const batch = { threadId: THREAD_ID, turnId: TURN_ID, executionId: EXECUTION_ID, phase: "running", events: events() }; + await expect(writer.commit("lost-three-times", batch)).rejects.toThrow("Canonical writer worker closed"); + expect(created).toBe(3); + expect(db.prepare("SELECT COUNT(*) AS count FROM canonical_agent_events").get()).toEqual({ count: 3 }); + await writer.close(); + writer = new CanonicalAgentWriterClient(dbPath); + expect(await writer.commit("lost-three-times", batch)).toMatchObject({ outcome: "committed" }); + }); + + it("rejects reused operation IDs with changed phase or native cursor", async () => { + writer = new CanonicalAgentWriterClient(dbPath); + const batch = { threadId: THREAD_ID, turnId: TURN_ID, executionId: EXECUTION_ID, phase: "running", events: events() }; + await writer.commit("same-id", batch); + await expect(writer.commit("same-id", { ...batch, phase: "finalizing" })) + .rejects.toThrow("Canonical writer operation-conflict"); + await expect(writer.commit("same-id", { ...batch, nativeCursor: "later" })) + .rejects.toThrow("Canonical writer operation-conflict"); + }); + + it("caps outstanding receipts and resumes after acknowledgement", async () => { + writer = new CanonicalAgentWriterClient(dbPath); + const batch = { threadId: THREAD_ID, turnId: TURN_ID, executionId: EXECUTION_ID, phase: "running", events: events() }; + await writer.commit("capacity-start", batch); + const insert = db.prepare(`INSERT INTO canonical_writer_operation_receipts + (execution_id, operation_id, kind, input_hash, receipt_json, created_at) + VALUES (?, ?, 'commit', ?, '{}', ?)`); + db.transaction(() => { + for (let index = 1; index < 16_384; index++) { + insert.run(EXECUTION_ID, `seed-${index}`, "seed", NOW); + } + })(); + const before = db.prepare("SELECT COUNT(*) AS count, SUM(LENGTH(receipt_json)) AS bytes FROM canonical_writer_operation_receipts").get(); + expect(before).toEqual({ count: 16_384, bytes: expect.any(Number) }); + await expect(writer.commit("capacity-over", batch)).rejects.toThrow("Canonical writer receipt-capacity"); + await writer.acknowledgeOperation(EXECUTION_ID, "capacity-start"); + await writer.acknowledgeOperation(EXECUTION_ID, "capacity-start"); + expect(await writer.commit("capacity-over", batch)).toMatchObject({ outcome: "duplicate" }); + expect(db.prepare("SELECT COUNT(*) AS count FROM canonical_writer_operation_receipts").get()) + .toEqual({ count: 16_384 }); + db.prepare("DELETE FROM canonical_agent_turns WHERE execution_id = ?").run(EXECUTION_ID); + expect(db.prepare("SELECT COUNT(*) AS count FROM canonical_writer_operation_receipts").get()) + .toEqual({ count: 0 }); + }, 30_000); }); diff --git a/apps/server/src/features/agents/canonical/canonical-agent-writer-client.ts b/apps/server/src/features/agents/canonical/canonical-agent-writer-client.ts index 323a0f1f4..da7bdf302 100644 --- a/apps/server/src/features/agents/canonical/canonical-agent-writer-client.ts +++ b/apps/server/src/features/agents/canonical/canonical-agent-writer-client.ts @@ -11,6 +11,7 @@ import type { import type { ParentNarrativeRecoveryCommitInput } from "./canonical-agent-boundary.js"; const MAX_PENDING_WRITES = 64; +const MAX_WORKER_ATTEMPTS = 3; const WRITER_EXECUTION_ID = "writer:init"; interface PendingRequest { @@ -19,72 +20,49 @@ interface PendingRequest { reject(error: Error): void; } -/** Sends cloneable canonical batches to one dedicated SQLite worker with bounded admission. */ +class CanonicalWriterWorkerLost extends Error {} + +/** Sends cloneable canonical commands to a dedicated SQLite worker with bounded admission and crash retries. */ export class CanonicalAgentWriterClient { - private readonly worker: Worker; + private worker: Worker | undefined; + private workerReady: Promise; + private restart: Promise | undefined; private readonly pending = new Map(); - private readonly ready: Promise; - private readonly closed: Promise; - private markClosed: (() => void) | undefined; - private failure: Error | undefined; + private generation = 0; + private stopping = false; - constructor(dbPath: string, createWorker: () => Worker = defaultCreateWorker) { + constructor( + private readonly dbPath: string, + private readonly createWorker: () => Worker = defaultCreateWorker, + ) { if (!NodePath.isAbsolute(dbPath)) throw new Error("Canonical writer database path must be absolute"); - this.worker = createWorker(); - this.closed = new Promise((resolve) => { this.markClosed = resolve; }); - this.worker.onmessage = (message: MessageEvent) => this.receive(message.data); - this.worker.onerror = () => this.fail(new Error("Canonical writer worker failed")); - this.worker.addEventListener("close", () => { - this.markClosed?.(); - this.fail(new Error("Canonical writer worker closed")); - }); - this.ready = this.send({ - kind: "open", - requestId: NodeCrypto.randomUUID(), - operationId: "writer:open", - executionId: WRITER_EXECUTION_ID, - dbPath, - }).then((response) => { - if (response.kind !== "opened") throw new Error("Canonical writer did not open its database"); - }).catch((error: Error) => { - this.fail(error); - throw error; - }); + this.workerReady = this.startWorker(); } /** Waits until the worker has opened the already-migrated database. */ whenReady(): Promise { - return this.ready; + return this.workerReady; } /** Resolves with committed envelopes for main-loop publication; it never publishes them itself. */ async commit(operationId: string, input: CanonicalProviderWriteInput): Promise { if (!operationId || !input.executionId) throw new Error("Canonical writer operation and execution IDs are required"); - await this.ready; - const response = await this.send({ - kind: "commit", - requestId: NodeCrypto.randomUUID(), - operationId, - executionId: input.executionId, - input, + const response = await this.sendWithRetry({ + kind: "commit", requestId: NodeCrypto.randomUUID(), operationId, executionId: input.executionId, input, }); if (response.kind !== "committed") throw new Error("Canonical writer returned an unexpected response"); return response.receipt; } - /** Resolves after recovery writes finish. A lost reply requires a durable-state check before retrying discards. */ + /** Resolves after recovery writes finish; a lost reply replays the durable receipt. */ async recordParentNarrativeRecovery( operationId: string, input: ParentNarrativeRecoveryCommitInput, ): Promise { if (!operationId || !input.executionId) throw new Error("Canonical writer operation and execution IDs are required"); - await this.ready; - const response = await this.send({ - kind: "record-parent-narrative-recovery", - requestId: NodeCrypto.randomUUID(), - operationId, - executionId: input.executionId, - input, + const response = await this.sendWithRetry({ + kind: "record-parent-narrative-recovery", requestId: NodeCrypto.randomUUID(), operationId, + executionId: input.executionId, input, }); if (response.kind !== "parent-narrative-recovery-recorded") { throw new Error("Canonical writer returned an unexpected response"); @@ -98,13 +76,9 @@ export class CanonicalAgentWriterClient { input: ParentNarrativeRecoveryCommitInput, ): Promise { if (!operationId || !input.executionId) throw new Error("Canonical writer operation and execution IDs are required"); - await this.ready; - const response = await this.send({ - kind: "classify-parent-narrative-recovery", - requestId: NodeCrypto.randomUUID(), - operationId, - executionId: input.executionId, - input, + const response = await this.sendWithRetry({ + kind: "classify-parent-narrative-recovery", requestId: NodeCrypto.randomUUID(), operationId, + executionId: input.executionId, input, }); if (response.kind !== "parent-narrative-recovery-classified") { throw new Error("Canonical writer returned an unexpected response"); @@ -112,38 +86,95 @@ export class CanonicalAgentWriterClient { return response.receipt; } + /** Releases a receipt after its caller has finished publication or journal discard. Never acknowledge before that work. */ + async acknowledgeOperation(executionId: string, operationId: string): Promise { + if (!executionId || !operationId) throw new Error("Canonical writer operation and execution IDs are required"); + const response = await this.sendWithRetry({ + kind: "ack-operation", requestId: NodeCrypto.randomUUID(), operationId, executionId, + }); + if (response.kind !== "operation-acknowledged") { + throw new Error("Canonical writer returned an unexpected response"); + } + } + /** Closes the database after accepted writes settle, then releases the worker. */ async close(): Promise { + if (this.stopping) return; + this.stopping = true; try { - await this.ready; - if (!this.failure) { - await this.send({ - kind: "close", - requestId: NodeCrypto.randomUUID(), - operationId: "writer:close", + await this.workerReady; + if (this.worker) { + await this.sendRaw({ + kind: "close", requestId: NodeCrypto.randomUUID(), operationId: "writer:close", executionId: WRITER_EXECUTION_ID, }); } - } catch (error) { - if (!this.failure) throw error; + } catch { + // A failed worker is already closed; caller writes have their own visible failures. } finally { - this.fail(new Error("Canonical writer closed")); - await this.closed; + this.loseWorker(new Error("Canonical writer closed")); + } + } + + private async sendWithRetry(request: CanonicalWriterRequest): Promise { + for (let attempt = 1; attempt <= MAX_WORKER_ATTEMPTS; attempt++) { + if (this.stopping) throw new Error("Canonical writer closed"); + try { + await this.workerReady; + return await this.sendRaw(request); + } catch (error) { + if (!(error instanceof CanonicalWriterWorkerLost) || attempt === MAX_WORKER_ATTEMPTS) throw error; + try { + await this.restartWorker(); + } catch (restartError) { + if (!(restartError instanceof CanonicalWriterWorkerLost)) throw restartError; + } + } + } + throw new Error("Canonical writer retry limit reached"); + } + + private async restartWorker(): Promise { + if (this.stopping) throw new Error("Canonical writer closed"); + if (!this.restart) { + this.restart = (async () => { + if (!this.worker) this.workerReady = this.startWorker(); + await this.workerReady; + })().finally(() => { this.restart = undefined; }); } + await this.restart; + } + + private async startWorker(): Promise { + const generation = ++this.generation; + const worker = this.createWorker(); + this.worker = worker; + worker.onmessage = (message: MessageEvent) => { + if (this.generation === generation) this.receive(message.data); + }; + worker.onerror = () => this.loseWorker(new CanonicalWriterWorkerLost("Canonical writer worker failed"), generation); + worker.addEventListener("close", () => { + this.loseWorker(new CanonicalWriterWorkerLost("Canonical writer worker closed"), generation); + }); + const response = await this.sendRaw({ + kind: "open", requestId: NodeCrypto.randomUUID(), operationId: "writer:open", + executionId: WRITER_EXECUTION_ID, dbPath: this.dbPath, + }); + if (response.kind !== "opened") throw new Error("Canonical writer did not open its database"); } - private send(request: CanonicalWriterRequest): Promise { - if (this.failure) return Promise.reject(this.failure); + private sendRaw(request: CanonicalWriterRequest): Promise { + const worker = this.worker; + if (!worker) return Promise.reject(new CanonicalWriterWorkerLost("Canonical writer worker closed")); if (request.kind !== "open" && request.kind !== "close" && this.pending.size >= MAX_PENDING_WRITES) { return Promise.reject(new Error("Canonical writer admission is full")); } return new Promise((resolve, reject) => { this.pending.set(request.requestId, { request, resolve, reject }); try { - this.worker.postMessage(request); + worker.postMessage(request); } catch { - this.pending.delete(request.requestId); - reject(new Error("Canonical writer request could not be sent")); + this.loseWorker(new CanonicalWriterWorkerLost("Canonical writer request could not be sent")); } }); } @@ -153,7 +184,7 @@ export class CanonicalAgentWriterClient { if (!pending) return; if (pending.request.operationId !== response.operationId || pending.request.executionId !== response.executionId) { - this.fail(new Error("Canonical writer response identity mismatch")); + this.loseWorker(new Error("Canonical writer response identity mismatch")); return; } this.pending.delete(response.requestId); @@ -164,10 +195,11 @@ export class CanonicalAgentWriterClient { pending.resolve(response); } - private fail(error: Error): void { - if (this.failure) return; - this.failure = error; - this.worker.terminate(); + private loseWorker(error: Error, generation = this.generation): void { + if (generation !== this.generation) return; + const worker = this.worker; + this.worker = undefined; + worker?.terminate(); for (const pending of this.pending.values()) pending.reject(error); this.pending.clear(); } diff --git a/apps/server/src/features/agents/canonical/canonical-agent-writer-protocol.ts b/apps/server/src/features/agents/canonical/canonical-agent-writer-protocol.ts index ccf608cb6..ab7a53c3a 100644 --- a/apps/server/src/features/agents/canonical/canonical-agent-writer-protocol.ts +++ b/apps/server/src/features/agents/canonical/canonical-agent-writer-protocol.ts @@ -44,6 +44,7 @@ export type CanonicalWriterRequest = | (Correlation & { kind: "commit"; input: CanonicalProviderWriteInput }) | (Correlation & { kind: "record-parent-narrative-recovery"; input: ParentNarrativeRecoveryCommitInput }) | (Correlation & { kind: "classify-parent-narrative-recovery"; input: ParentNarrativeRecoveryCommitInput }) + | (Correlation & { kind: "ack-operation" }) | (Correlation & { kind: "close" }); export type CanonicalWriterResponse = @@ -51,5 +52,6 @@ export type CanonicalWriterResponse = | (Correlation & { kind: "committed"; receipt: CanonicalProviderWriteReceipt }) | (Correlation & { kind: "parent-narrative-recovery-recorded"; receipt: CanonicalParentNarrativeRecoveryReceipt }) | (Correlation & { kind: "parent-narrative-recovery-classified"; receipt: CanonicalParentNarrativeClassificationReceipt }) + | (Correlation & { kind: "operation-acknowledged" }) | (Correlation & { kind: "closed" }) - | (Correlation & { kind: "failed"; reason: "open-failed" | "write-failed" }); + | (Correlation & { kind: "failed"; reason: "open-failed" | "write-failed" | "operation-conflict" | "receipt-capacity" }); diff --git a/apps/server/src/features/agents/canonical/canonical-agent-writer-receipts.ts b/apps/server/src/features/agents/canonical/canonical-agent-writer-receipts.ts new file mode 100644 index 000000000..f991b30e9 --- /dev/null +++ b/apps/server/src/features/agents/canonical/canonical-agent-writer-receipts.ts @@ -0,0 +1,168 @@ +import type { Database } from "bun:sqlite"; +import * as NodeCrypto from "node:crypto"; +import { CanonicalAgentEventEnvelopeSchema } from "@mcode/contracts"; +import { and, count, eq } from "drizzle-orm"; +import { drizzle } from "drizzle-orm/bun-sqlite"; +import { z } from "zod"; +import { + canonicalAgentEvents, + canonicalWriterOperationReceipts, +} from "../../../runtime/persistence/sqlite/schema.js"; +import type { CanonicalWriterRequest, CanonicalWriterResponse } from "./canonical-agent-writer-protocol.js"; + +type WriteRequest = Extract; +type WriteResponse = Extract; + +const storedReceiptSchema = z.discriminatedUnion("kind", [ + z.object({ + kind: z.literal("committed"), + receipt: z.object({ + outcome: z.enum(["committed", "duplicate", "conflict", "terminal-outcome-confirmed", "ingest-overflow"]), + conversationRevision: z.number().int(), + rosterRevision: z.number().int(), + acceptedThrough: z.number().int(), + durableThrough: z.number().int(), + eventIds: z.array(z.string()), + }), + }), + z.object({ + kind: z.literal("parent-narrative-recovery-recorded"), + receipt: z.object({ recorded: z.boolean() }), + }), + z.object({ + kind: z.literal("parent-narrative-recovery-classified"), + receipt: z.object({ recorded: z.literal(true), reset: z.literal(true) }), + }), +]); + +/** An operation ID may be retried only with the same kind and semantic input. */ +export class CanonicalWriterOperationConflict extends Error {} + +/** The caller must acknowledge handled receipts before admitting more writes for an execution. */ +export class CanonicalWriterReceiptCapacity extends Error {} + +export const MAX_UNACKNOWLEDGED_RECEIPTS_PER_EXECUTION = 16_384; + +/** Atomically stores compact receipts so a restarted worker can replay lost acknowledgements. */ +export class CanonicalAgentWriterReceipts { + private readonly orm; + + constructor(private readonly db: Database) { + this.orm = drizzle(db); + } + + execute(request: WriteRequest, apply: () => WriteResponse): WriteResponse { + const inputHash = fingerprint(request); + return this.db.transaction(() => { + const existing = this.orm.select().from(canonicalWriterOperationReceipts) + .where(and( + eq(canonicalWriterOperationReceipts.executionId, request.executionId), + eq(canonicalWriterOperationReceipts.operationId, request.operationId), + )).get(); + if (existing) { + if (existing.kind !== request.kind || existing.inputHash !== inputHash) { + throw new CanonicalWriterOperationConflict("Canonical writer operation ID was reused with different input"); + } + return this.replay(request, existing.receiptJson); + } + const outstanding = this.orm.select({ count: count() }).from(canonicalWriterOperationReceipts) + .where(eq(canonicalWriterOperationReceipts.executionId, request.executionId)).get()?.count ?? 0; + if (outstanding >= MAX_UNACKNOWLEDGED_RECEIPTS_PER_EXECUTION) { + throw new CanonicalWriterReceiptCapacity("Canonical writer receipt capacity reached"); + } + const response = apply(); + if (response.kind === "parent-narrative-recovery-recorded" && !response.receipt.recorded) { + return response; + } + this.orm.insert(canonicalWriterOperationReceipts).values({ + executionId: request.executionId, + operationId: request.operationId, + kind: request.kind, + inputHash, + receiptJson: compactReceipt(response), + }).run(); + return response; + })(); + } + + /** Removes a receipt only after the caller has completed its dependent publication or journal discard. */ + acknowledge(executionId: string, operationId: string): void { + this.orm.delete(canonicalWriterOperationReceipts).where(and( + eq(canonicalWriterOperationReceipts.executionId, executionId), + eq(canonicalWriterOperationReceipts.operationId, operationId), + )).run(); + } + + private replay(request: WriteRequest, receiptJson: string): WriteResponse { + const stored = storedReceiptSchema.parse(JSON.parse(receiptJson)); + const correlation = { + requestId: request.requestId, + operationId: request.operationId, + executionId: request.executionId, + }; + if (stored.kind === "committed") { + return { + ...correlation, + kind: "committed", + receipt: { + outcome: stored.receipt.outcome, + conversationRevision: stored.receipt.conversationRevision, + rosterRevision: stored.receipt.rosterRevision, + acceptedThrough: stored.receipt.acceptedThrough, + durableThrough: stored.receipt.durableThrough, + events: stored.receipt.eventIds.map((eventId) => this.loadEvent(request.executionId, eventId)), + }, + }; + } + if (stored.kind === "parent-narrative-recovery-recorded") { + return { ...correlation, kind: stored.kind, receipt: stored.receipt }; + } + return { ...correlation, kind: stored.kind, receipt: stored.receipt }; + } + + private loadEvent(executionId: string, eventId: string) { + const row = this.orm.select({ envelopeJson: canonicalAgentEvents.envelopeJson }) + .from(canonicalAgentEvents) + .where(and( + eq(canonicalAgentEvents.executionId, executionId), + eq(canonicalAgentEvents.eventId, eventId), + )) + .get(); + if (!row) throw new Error(`Canonical writer receipt event was not found: ${eventId}`); + return CanonicalAgentEventEnvelopeSchema.parse(JSON.parse(row.envelopeJson)); + } +} + +function compactReceipt(response: WriteResponse): string { + if (response.kind !== "committed") { + return JSON.stringify({ kind: response.kind, receipt: response.receipt }); + } + const { events, ...receipt } = response.receipt; + return JSON.stringify({ + kind: response.kind, + receipt: { ...receipt, eventIds: events.map((event) => event.eventId) }, + }); +} + +function fingerprint(request: WriteRequest): string { + if (!request.operationId || request.operationId.length > 256 || !request.executionId) { + throw new Error("Canonical writer operation and execution IDs are required and bounded"); + } + const input = stableJson({ kind: request.kind, input: request.input }); + return NodeCrypto.createHash("sha256").update(input).digest("hex"); +} + +function stableJson(value: unknown): string { + return JSON.stringify(sortJsonValue(JSON.parse(JSON.stringify(value)))); +} + +function sortJsonValue(value: unknown): unknown { + if (Array.isArray(value)) return value.map(sortJsonValue); + if (value === null || typeof value !== "object") return value; + return Object.fromEntries(Object.entries(value).sort(([left], [right]) => left.localeCompare(right)) + .map(([key, entry]) => [key, sortJsonValue(entry)])); +} diff --git a/apps/server/src/features/agents/canonical/canonical-agent-writer.worker.ts b/apps/server/src/features/agents/canonical/canonical-agent-writer.worker.ts index c256fbc9e..556be5028 100644 --- a/apps/server/src/features/agents/canonical/canonical-agent-writer.worker.ts +++ b/apps/server/src/features/agents/canonical/canonical-agent-writer.worker.ts @@ -5,6 +5,7 @@ import * as NodePath from "node:path"; import { applySQLiteConnectionPolicy } from "../../../runtime/persistence/sqlite/sqlite-connection-policy.js"; import { CanonicalAgentBoundary } from "./canonical-agent-boundary.js"; import { ParentAssistantTextCheckpointService } from "../turns/parent-assistant-text-checkpoint-service.js"; +import { CanonicalAgentWriterReceipts, CanonicalWriterOperationConflict, CanonicalWriterReceiptCapacity } from "./canonical-agent-writer-receipts.js"; import type { CanonicalParentNarrativeClassificationReceipt, CanonicalProviderWriteInput, @@ -16,6 +17,7 @@ import type { ParentNarrativeRecoveryCommitInput } from "./canonical-agent-bound let db: Database | undefined; let boundary: CanonicalAgentBoundary | undefined; let assistantTextCheckpoints: ParentAssistantTextCheckpointService | undefined; +let receipts: CanonicalAgentWriterReceipts | undefined; function openDatabase(dbPath: string): void { if (boundary || !NodePath.isAbsolute(dbPath) || !NodeFS.existsSync(dbPath)) { @@ -26,10 +28,12 @@ function openDatabase(dbPath: string): void { applySQLiteConnectionPolicy(connection, true); boundary = new CanonicalAgentBoundary(connection, () => {}); assistantTextCheckpoints = new ParentAssistantTextCheckpointService(connection); + receipts = new CanonicalAgentWriterReceipts(connection); db = connection; } catch (error) { boundary = undefined; assistantTextCheckpoints = undefined; + receipts = undefined; connection.close(true); throw error; } @@ -88,10 +92,20 @@ function handle(request: CanonicalWriterRequest): CanonicalWriterResponse { if (request.kind === "close") { boundary = undefined; assistantTextCheckpoints = undefined; + receipts = undefined; db?.close(true); db = undefined; return { ...correlation, kind: "closed" }; } + if (request.kind === "ack-operation") { + try { + if (!receipts) throw new Error("Canonical writer has not opened its database"); + receipts.acknowledge(request.executionId, request.operationId); + return { ...correlation, kind: "operation-acknowledged" }; + } catch { + return { ...correlation, kind: "failed", reason: "write-failed" }; + } + } return handleWrite(request, correlation); } @@ -100,38 +114,48 @@ function handleWrite( correlation: Pick, ): CanonicalWriterResponse { try { - if (!boundary) throw new Error("Canonical writer has not opened its database"); - if (request.kind === "classify-parent-narrative-recovery") { - if (request.input.executionId !== request.executionId) { - throw new Error("Canonical writer classification execution mismatch"); - } - const receipt = classifyParentNarrativeRecovery(request.input); - return { ...correlation, kind: "parent-narrative-recovery-classified", receipt }; - } - if (request.kind === "record-parent-narrative-recovery") { - if (request.input.executionId !== request.executionId) { - throw new Error("Canonical writer recovery execution mismatch"); - } - const recorded = boundary.recordParentNarrativeRecovery(request.input); - return { - ...correlation, - kind: "parent-narrative-recovery-recorded", - receipt: { recorded }, - }; - } - assertRouting(request.input, request.executionId); - const result = boundary.commit(request.input); - const { outcome, conversationRevision, rosterRevision, acceptedThrough, durableThrough, events } = result; + const canonical = boundary; + if (!canonical || !receipts) throw new Error("Canonical writer has not opened its database"); + return receipts.execute(request, () => applyWrite(request, correlation, canonical)); + } catch (error) { return { ...correlation, - kind: "committed", - receipt: { outcome, conversationRevision, rosterRevision, acceptedThrough, durableThrough, events }, + kind: "failed", + reason: error instanceof CanonicalWriterOperationConflict ? "operation-conflict" + : error instanceof CanonicalWriterReceiptCapacity ? "receipt-capacity" : "write-failed", }; - } catch { - return { ...correlation, kind: "failed", reason: "write-failed" }; } } +function applyWrite( + request: Extract, + correlation: Pick, + canonical: CanonicalAgentBoundary, +): Extract { + if (request.kind === "classify-parent-narrative-recovery") { + if (request.input.executionId !== request.executionId) { + throw new Error("Canonical writer classification execution mismatch"); + } + const receipt = classifyParentNarrativeRecovery(request.input); + return { ...correlation, kind: "parent-narrative-recovery-classified", receipt }; + } + if (request.kind === "record-parent-narrative-recovery") { + if (request.input.executionId !== request.executionId) { + throw new Error("Canonical writer recovery execution mismatch"); + } + const recorded = canonical.recordParentNarrativeRecovery(request.input); + return { ...correlation, kind: "parent-narrative-recovery-recorded", receipt: { recorded } }; + } + assertRouting(request.input, request.executionId); + const result = canonical.commit(request.input); + const { outcome, conversationRevision, rosterRevision, acceptedThrough, durableThrough, events } = result; + return { + ...correlation, + kind: "committed", + receipt: { outcome, conversationRevision, rosterRevision, acceptedThrough, durableThrough, events }, + }; +} + globalThis.onmessage = (message: MessageEvent): void => { globalThis.postMessage(handle(message.data)); }; diff --git a/apps/server/src/runtime/persistence/sqlite/schema.ts b/apps/server/src/runtime/persistence/sqlite/schema.ts index b026777a7..31e59b947 100644 --- a/apps/server/src/runtime/persistence/sqlite/schema.ts +++ b/apps/server/src/runtime/persistence/sqlite/schema.ts @@ -5,7 +5,7 @@ import { sql } from "drizzle-orm"; import { asc, desc } from "drizzle-orm"; -import { type AnySQLiteColumn, index, integer, real, sqliteTable, text, uniqueIndex } from "drizzle-orm/sqlite-core"; +import { type AnySQLiteColumn, index, integer, primaryKey, real, sqliteTable, text, uniqueIndex } from "drizzle-orm/sqlite-core"; const timestampDefault = sql`(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))`; @@ -741,6 +741,21 @@ export const canonicalAgentEvents = sqliteTable( ], ); +/** Writer receipts survive a lost worker reply and follow their canonical turn on deletion. */ +export const canonicalWriterOperationReceipts = sqliteTable( + "canonical_writer_operation_receipts", + { + executionId: text("execution_id").notNull() + .references(() => canonicalAgentTurns.executionId, { onDelete: "cascade" }), + operationId: text("operation_id").notNull(), + kind: text("kind").notNull(), + inputHash: text("input_hash").notNull(), + receiptJson: text("receipt_json").notNull(), + createdAt: text("created_at").notNull().default(timestampDefault), + }, + (table) => [primaryKey({ columns: [table.executionId, table.operationId] })], +); + /** Durable accepted and committed progress for one canonical execution. */ export const canonicalAgentIngestCheckpoints = sqliteTable( "canonical_agent_ingest_checkpoints", From 060772eb1e2b966ea96dec9f9dccd38b54691472 Mon Sep 17 00:00:00 2001 From: "(CJ) Chukwudi Nwobodo" <142016413+chuks-qua@users.noreply.github.com> Date: Thu, 24 Sep 2026 19:57:25 +0100 Subject: [PATCH 016/136] fix(server): acknowledge handled canonical receipts --- .../__tests__/provider-host-ports.test.ts | 44 +++++++++++++++++-- .../composition/provider-host-ports.ts | 14 +++++- 2 files changed, 53 insertions(+), 5 deletions(-) diff --git a/apps/server/src/features/providers/composition/__tests__/provider-host-ports.test.ts b/apps/server/src/features/providers/composition/__tests__/provider-host-ports.test.ts index 28832b3d0..ab7af0d2c 100644 --- a/apps/server/src/features/providers/composition/__tests__/provider-host-ports.test.ts +++ b/apps/server/src/features/providers/composition/__tests__/provider-host-ports.test.ts @@ -84,6 +84,7 @@ describe("createProviderHostPorts", () => { }); const publishCanonicalEvents = vi.fn(() => deliveryOrder.push("publication")); const acceptCommitted = vi.fn(() => deliveryOrder.push("ingress")); + const acknowledgeOperation = vi.fn(async () => { deliveryOrder.push("acknowledge"); }); const ports = createProviderHostPorts({ runtime: { platform: "linux", architecture: "x64", nodeAbi: "127" }, envService: { getEnv: () => ({ PATH: "test" }) }, @@ -91,7 +92,7 @@ describe("createProviderHostPorts", () => { browser: {}, threadControl: {}, grants: {}, - events: { commit }, + events: { commit, acknowledgeOperation }, publishCanonicalEvents, ingress: { acceptCommitted }, } as never); @@ -111,7 +112,8 @@ describe("createProviderHostPorts", () => { expect(commit).toHaveBeenCalledWith(expect.any(String), { ...batch, nativeCursor: undefined }); expect(publishCanonicalEvents).toHaveBeenCalledWith(events); expect(acceptCommitted).toHaveBeenCalledWith(events); - expect(deliveryOrder).toEqual(["commit", "publication", "ingress"]); + expect(acknowledgeOperation).toHaveBeenCalledWith(EXECUTION_ID, expect.any(String)); + expect(deliveryOrder).toEqual(["commit", "publication", "ingress", "acknowledge"]); }); it("does not hand duplicate or failed commits to ingress", async () => { @@ -135,7 +137,7 @@ describe("createProviderHostPorts", () => { browser: {}, threadControl: {}, grants: {}, - events: { commit }, + events: { commit, acknowledgeOperation: vi.fn() }, publishCanonicalEvents: vi.fn(), ingress: { acceptCommitted }, } as never); @@ -165,7 +167,7 @@ describe("createProviderHostPorts", () => { browser: {}, threadControl: {}, grants: {}, - events: { commit }, + events: { commit, acknowledgeOperation: vi.fn() }, publishCanonicalEvents: vi.fn(), ingress: { acceptCommitted: vi.fn() }, } as never); @@ -178,4 +180,38 @@ describe("createProviderHostPorts", () => { expect(operationIds[0]).toBe(operationIds[1]); expect(operationIds[2]).not.toBe(operationIds[0]); }); + + it("retains the receipt when publication fails after a durable commit", async () => { + const events = [committedRuntimeEnvelope()]; + const acknowledgeOperation = vi.fn(); + const acceptCommitted = vi.fn(); + const ports = createProviderHostPorts({ + runtime: { platform: "linux", architecture: "x64", nodeAbi: "127" }, + envService: { getEnv: () => ({}) }, + jobObject: { isWindowsJob: false }, + browser: {}, + threadControl: {}, + grants: {}, + events: { + commit: vi.fn(async () => ({ + outcome: "committed", + conversationRevision: 1, + rosterRevision: 0, + acceptedThrough: 1, + durableThrough: 1, + events, + })), + acknowledgeOperation, + }, + publishCanonicalEvents: () => { throw new Error("push unavailable"); }, + ingress: { acceptCommitted }, + } as never); + + await expect(ports.events.submit({ + threadId: "thread-1", turnId: "turn-1", executionId: EXECUTION_ID, + phase: "streaming", events: [], + })).resolves.toMatchObject({ commit: { outcome: "committed" } }); + expect(acceptCommitted).toHaveBeenCalledWith(events); + expect(acknowledgeOperation).not.toHaveBeenCalled(); + }); }); diff --git a/apps/server/src/features/providers/composition/provider-host-ports.ts b/apps/server/src/features/providers/composition/provider-host-ports.ts index fb02dcdbc..dc37d7b7e 100644 --- a/apps/server/src/features/providers/composition/provider-host-ports.ts +++ b/apps/server/src/features/providers/composition/provider-host-ports.ts @@ -20,7 +20,7 @@ export interface ProviderHostPortDependencies { browser: BrowserAutomationSessionLease; threadControl: InternalThreadControlMcpRuntime; grants: ScopedPreGrantService; - events: Pick; + events: Pick; publishCanonicalEvents: CanonicalAgentEventPublisher; ingress: ProviderEventIngress; } @@ -103,10 +103,12 @@ export function createProviderHostPorts( nativeCursor: batch.nativeCursor, events: batch.events, }); + let published = true; if (result.outcome === "committed" && result.events.length > 0) { try { dependencies.publishCanonicalEvents(result.events); } catch { + published = false; logger.warn("Canonical provider publication deferred after durable commit", { threadId: batch.threadId, executionId: batch.executionId, @@ -114,6 +116,16 @@ export function createProviderHostPorts( } dependencies.ingress.acceptCommitted(result.events); } + if (published) { + try { + await dependencies.events.acknowledgeOperation(batch.executionId, operationId); + } catch { + logger.warn("Canonical provider receipt acknowledgement deferred", { + threadId: batch.threadId, + executionId: batch.executionId, + }); + } + } return { commit: { outcome: providerCommitOutcome(result.outcome), From fc0e86c1b33a044932bc9eff76d7bc86a9f37f48 Mon Sep 17 00:00:00 2001 From: "(CJ) Chukwudi Nwobodo" <142016413+chuks-qua@users.noreply.github.com> Date: Thu, 24 Sep 2026 19:59:02 +0100 Subject: [PATCH 017/136] feat(server): run execution mailboxes in Bun workers --- .../build-server-runtime-bundles.test.mjs | 42 ++++ apps/desktop/scripts/build-main.mjs | 2 +- .../execution-mailbox-scheduler.test.ts | 17 ++ .../execution-thread-worker-port.test.ts | 132 +++++++++++ .../execution-worker-handler.test.ts | 1 + .../__tests__/fixtures/exit.worker.ts | 2 + .../execution/execution-mailbox-protocol.ts | 1 + .../execution/execution-mailbox-scheduler.ts | 4 +- .../agents/execution/execution-worker-port.ts | 218 ++++++++++++++++++ .../agents/execution/execution.worker.ts | 114 +++++++++ scripts/build-server-dev-bundle.mjs | 10 +- 11 files changed, 539 insertions(+), 4 deletions(-) create mode 100644 apps/server/src/features/agents/execution/__tests__/execution-thread-worker-port.test.ts create mode 100644 apps/server/src/features/agents/execution/__tests__/fixtures/exit.worker.ts create mode 100644 apps/server/src/features/agents/execution/execution-worker-port.ts create mode 100644 apps/server/src/features/agents/execution/execution.worker.ts diff --git a/apps/desktop/scripts/__tests__/build-server-runtime-bundles.test.mjs b/apps/desktop/scripts/__tests__/build-server-runtime-bundles.test.mjs index 432fb08f9..fb319bc21 100644 --- a/apps/desktop/scripts/__tests__/build-server-runtime-bundles.test.mjs +++ b/apps/desktop/scripts/__tests__/build-server-runtime-bundles.test.mjs @@ -1,4 +1,5 @@ import { afterEach, describe, expect, it } from "vitest"; +import * as NodeChildProcess from "node:child_process"; import * as NodeFSPromises from "node:fs/promises"; import * as NodeOS from "node:os"; import * as NodePath from "node:path"; @@ -25,6 +26,7 @@ describe("buildServerRuntimeBundles", () => { const ptyHostOutFile = NodePath.join(outputRoot, "pty-host.cjs"); const providerEventWorkerOutFile = NodePath.join(outputRoot, "provider-event.worker.cjs"); const canonicalWriterWorkerOutFile = NodePath.join(outputRoot, "canonical-agent-writer.worker.cjs"); + const executionWorkerOutFile = NodePath.join(outputRoot, "execution.worker.cjs"); try { compileServerWithSwc(serverRoot); @@ -38,7 +40,10 @@ describe("buildServerRuntimeBundles", () => { expect(() => new NodeVM.Script(bundledEntry, { filename: "server.cjs" })).not.toThrow(); await NodeFSPromises.access(providerEventWorkerOutFile); await NodeFSPromises.access(canonicalWriterWorkerOutFile); + await NodeFSPromises.access(executionWorkerOutFile); + expect(await NodeFSPromises.readFile(executionWorkerOutFile, "utf8")).toContain("writer-request"); expect(await NodeFSPromises.readFile(serverOutFile, "utf8")).toContain("provider-event.worker.cjs"); + expect(startAndCloseBundledWorker(executionWorkerOutFile)).toBe("ready,closed"); } finally { await NodeFSPromises.rm(outputRoot, { recursive: true, force: true }); await NodeFSPromises.rm(distTsc, { recursive: true, force: true }); @@ -73,20 +78,31 @@ describe("buildServerRuntimeBundles", () => { "canonical", "canonical-agent-writer.worker.js", ); + const executionWorkerEntry = NodePath.join( + serverRoot, + "dist-tsc", + "features", + "agents", + "execution", + "execution.worker.js", + ); const serverOutFile = NodePath.join(fixtureRoot, "dist", "server.cjs"); const ptyHostOutFile = NodePath.join(fixtureRoot, "dist", "pty-host.cjs"); const providerEventWorkerOutFile = NodePath.join(fixtureRoot, "dist", "provider-event.worker.cjs"); const canonicalWriterWorkerOutFile = NodePath.join(fixtureRoot, "dist", "canonical-agent-writer.worker.cjs"); + const executionWorkerOutFile = NodePath.join(fixtureRoot, "dist", "execution.worker.cjs"); await Promise.all([ NodeFSPromises.mkdir(NodePath.dirname(ptyHostEntry), { recursive: true }), NodeFSPromises.mkdir(NodePath.dirname(providerEventWorkerEntry), { recursive: true }), NodeFSPromises.mkdir(NodePath.dirname(canonicalWriterWorkerEntry), { recursive: true }), + NodeFSPromises.mkdir(NodePath.dirname(executionWorkerEntry), { recursive: true }), ]); await NodeFSPromises.writeFile(serverEntry, 'console.log("server-entry");\n'); await NodeFSPromises.writeFile(ptyHostEntry, 'console.log("pty-host-entry");\n'); await NodeFSPromises.writeFile(providerEventWorkerEntry, 'console.log("provider-event-worker");\n'); await NodeFSPromises.writeFile(canonicalWriterWorkerEntry, 'console.log("canonical-writer-worker");\n'); + await NodeFSPromises.writeFile(executionWorkerEntry, 'console.log("execution-worker");\n'); await buildServerRuntimeBundles({ serverRoot, @@ -99,9 +115,35 @@ describe("buildServerRuntimeBundles", () => { await NodeFSPromises.access(ptyHostOutFile); await NodeFSPromises.access(providerEventWorkerOutFile); await NodeFSPromises.access(canonicalWriterWorkerOutFile); + await NodeFSPromises.access(executionWorkerOutFile); expect(await NodeFSPromises.readFile(serverOutFile, "utf8")).toContain("server-entry"); expect(await NodeFSPromises.readFile(ptyHostOutFile, "utf8")).toContain("pty-host-entry"); expect(await NodeFSPromises.readFile(providerEventWorkerOutFile, "utf8")).toContain("provider-event-worker"); expect(await NodeFSPromises.readFile(canonicalWriterWorkerOutFile, "utf8")).toContain("canonical-writer-worker"); + expect(await NodeFSPromises.readFile(executionWorkerOutFile, "utf8")).toContain("execution-worker"); }, 30_000); }); + +function startAndCloseBundledWorker(workerPath) { + const script = ` + import { pathToFileURL } from "node:url"; + const worker = new Worker(pathToFileURL(process.env.MCODE_TEST_EXECUTION_WORKER), { type: "module", ref: true }); + const received = []; + const timer = setTimeout(() => { worker.terminate(); process.exit(1); }, 3000); + worker.onmessage = (event) => { + received.push(event.data.kind); + if (event.data.kind === "ready") worker.postMessage({ kind: "close" }); + if (event.data.kind === "closed") { + clearTimeout(timer); + worker.terminate(); + process.stdout.write(received.join(",")); + } + }; + worker.onerror = () => { clearTimeout(timer); process.exit(1); }; + `; + return NodeChildProcess.execFileSync("bun", ["-e", script], { + encoding: "utf8", + timeout: 5_000, + env: { ...process.env, MCODE_TEST_EXECUTION_WORKER: workerPath }, + }); +} diff --git a/apps/desktop/scripts/build-main.mjs b/apps/desktop/scripts/build-main.mjs index 174c6e132..07e29409d 100644 --- a/apps/desktop/scripts/build-main.mjs +++ b/apps/desktop/scripts/build-main.mjs @@ -90,7 +90,7 @@ await buildServerRuntimeBundles({ production: true, }); -console.log("Server bundles complete: dist/server/server.cjs, dist/server/pty-host.cjs"); +console.log("Server bundles complete: dist/server/server.cjs, dist/server/pty-host.cjs, dist/server/execution.worker.cjs"); const drizzleSrc = NodePath.resolve(serverRoot, "drizzle"); const drizzleDst = NodePath.resolve(desktopRoot, "dist/server/drizzle"); diff --git a/apps/server/src/features/agents/execution/__tests__/execution-mailbox-scheduler.test.ts b/apps/server/src/features/agents/execution/__tests__/execution-mailbox-scheduler.test.ts index fc37c2fca..e7c80554b 100644 --- a/apps/server/src/features/agents/execution/__tests__/execution-mailbox-scheduler.test.ts +++ b/apps/server/src/features/agents/execution/__tests__/execution-mailbox-scheduler.test.ts @@ -34,6 +34,7 @@ const LIMITS: ExecutionMailboxLimits = { class FakeWorker implements ExecutionWorkerPort { onmessage: ((event: MessageEvent>) => void) | null = null; onerror: ((event: ErrorEvent) => void) | null = null; + onclose: (() => void) | null = null; readonly requests: ExecutionWorkerRequest[] = []; terminated = false; @@ -54,6 +55,10 @@ class FakeWorker implements ExecutionWorkerPort { crash(): void { this.onerror?.(new ErrorEvent("error")); } + + close(): void { + this.onclose?.(); + } } function execution(threadId: string, executionId = `execution-${threadId}`): ExecutionIdentity { @@ -307,6 +312,18 @@ describe("ExecutionMailboxScheduler", () => { scheduler.shutdown(); }); + it("revokes retained commands when a worker exits without an error", async () => { + const { scheduler, workers, lost } = fixture(); + const identity = execution("closed"); + const lease = claimed(scheduler, identity); + const event = admitted(scheduler, identity, lease, { kind: "event", sequence: 1 }); + workers[0]?.close(); + expect(await event.completion).toEqual({ kind: "worker-lost" }); + expect(lost).toEqual([[identity]]); + expect(scheduler.depth()).toMatchObject({ pending: 0, activeExecutions: 0 }); + scheduler.shutdown(); + }); + it("rejects a second active execution on the same thread and settles shutdown", async () => { const { scheduler } = fixture(); const identity = execution("one", "first"); diff --git a/apps/server/src/features/agents/execution/__tests__/execution-thread-worker-port.test.ts b/apps/server/src/features/agents/execution/__tests__/execution-thread-worker-port.test.ts new file mode 100644 index 000000000..4f2af48d9 --- /dev/null +++ b/apps/server/src/features/agents/execution/__tests__/execution-thread-worker-port.test.ts @@ -0,0 +1,132 @@ +import type { ProviderEventDraft } from "@mcode/providers"; +import { describe, expect, it } from "vitest"; + +import { ExecutionMailboxScheduler, type ExecutionMailboxLimits } from "../execution-mailbox-scheduler.js"; +import type { ExecutionIdentity, ExecutionLease } from "../execution-mailbox-protocol.js"; +import { ExecutionThreadWorkerPort } from "../execution-worker-port.js"; +import type { + ExecutionSemanticOperation, + ExecutionSemanticWriter, + ExecutionWorkCommand, + ExecutionWorkerResult, + ExecutionWriteReceipt, +} from "../execution-worker-handler.js"; + +const EXECUTION: ExecutionIdentity = { + threadId: "real-worker-thread", + turnId: "real-worker-turn", + executionId: "00000000-0000-4000-8000-000000000002", +}; + +const LIMITS: ExecutionMailboxLimits = { + maxPending: 8, + maxPendingBytes: 8_000, + reservedControl: 4, + reservedControlBytes: 4_000, + maxPerExecutionPending: 6, + maxPerExecutionBytes: 6_000, + reservedPerExecutionControl: 3, + reservedPerExecutionControlBytes: 3_000, +}; + +class AcknowledgingWriter implements ExecutionSemanticWriter { + readonly operations: ExecutionSemanticOperation[] = []; + + async transact(operation: ExecutionSemanticOperation): Promise { + this.operations.push(operation); + return { kind: "committed", operationId: operation.operationId, durableRevision: this.operations.length }; + } +} + +function fixture(writer: ExecutionSemanticWriter, workerUrl?: URL) { + const ports: ExecutionThreadWorkerPort[] = []; + const lost: ExecutionIdentity[][] = []; + const scheduler = new ExecutionMailboxScheduler({ + workerCount: 1, + limits: LIMITS, + createWorker: () => { + const port = new ExecutionThreadWorkerPort(writer, workerUrl); + ports.push(port); + return port; + }, + onWorkerLost: (executions) => lost.push([...executions]), + }); + const claim = scheduler.claim(EXECUTION, 1); + if (claim.kind !== "claimed") throw new Error(`Claim failed: ${claim.kind}`); + return { scheduler, port: ports[0]!, lost, lease: claim.lease }; +} + +function submit( + scheduler: ExecutionMailboxScheduler, + lease: ExecutionLease, + command: Parameters[0]["command"], +) { + const admission = scheduler.submit({ execution: EXECUTION, lease, command, byteLength: 1_000 }); + if (admission.kind !== "admitted") throw new Error(`Admission failed: ${admission.kind}`); + return admission.completion; +} + +function eventDraft(): ProviderEventDraft { + return { + eventId: "real-worker-event-1", + routing: { ...EXECUTION, itemId: "real-worker-item-1" }, + sourceProviderId: "codex", + sourceIdentities: [], + sourceSequence: 1, + payload: { type: "turn.started", startedAt: "2026-09-24T12:00:00.000Z" }, + }; +} + +describe("ExecutionThreadWorkerPort", () => { + it("starts and closes the real Bun Worker", async () => { + const port = new ExecutionThreadWorkerPort(new AcknowledgingWriter()); + try { + await expect(port.whenReady()).resolves.toBe(true); + await expect(port.close()).resolves.toBe(true); + } finally { + port.terminate(); + } + }, 10_000); + + it("runs a complete synthetic execution through a real Worker and writer RPC", async () => { + const writer = new AcknowledgingWriter(); + const { scheduler, port, lease } = fixture(writer); + try { + await expect(submit(scheduler, lease, { kind: "start", providerId: "codex" })) + .resolves.toMatchObject({ kind: "reply", result: { kind: "committed", durableRevision: 1 } }); + await expect(port.whenReady()).resolves.toBe(true); + await expect(submit(scheduler, lease, { kind: "event", events: [eventDraft()] })) + .resolves.toMatchObject({ kind: "reply", result: { kind: "committed", durableRevision: 2 } }); + await expect(submit(scheduler, lease, { kind: "stop", requestId: "stop-real-worker" })) + .resolves.toMatchObject({ kind: "reply", result: { kind: "committed", durableRevision: 3 } }); + await expect(submit(scheduler, lease, { kind: "checkpoint", phase: "stopping", nativeCursor: null })) + .resolves.toMatchObject({ kind: "reply", result: { kind: "committed", durableRevision: 4 } }); + await expect(submit(scheduler, lease, { kind: "finalize", outcome: "cancelled" })) + .resolves.toMatchObject({ kind: "reply", result: { kind: "committed", durableRevision: 5 } }); + await expect(submit(scheduler, lease, { kind: "release" })) + .resolves.toEqual({ kind: "reply", result: { kind: "released" } }); + expect(scheduler.release(EXECUTION, lease)).toBe(true); + expect(writer.operations.map((operation) => operation.mutation.kind)) + .toEqual(["begin", "append-events", "stop-requested", "checkpoint", "finish"]); + expect(writer.operations[2]?.mutation).toEqual({ + kind: "stop-requested", requestId: "stop-real-worker", lastAdmittedOrdinal: 2, + }); + } finally { + scheduler.shutdown(); + } + }, 10_000); + + it("treats a clean but unexpected Worker exit as ownership loss", async () => { + const exitWorkerUrl = new URL("./fixtures/exit.worker.ts", import.meta.url); + const { scheduler, port, lost, lease } = fixture(new AcknowledgingWriter(), exitWorkerUrl); + try { + await expect(port.whenReady()).resolves.toBe(true); + await expect(submit(scheduler, lease, { kind: "start", providerId: "codex" })) + .resolves.toEqual({ kind: "worker-lost" }); + expect(lost).toEqual([[EXECUTION]]); + expect(scheduler.depth()).toMatchObject({ pending: 0, activeExecutions: 0 }); + } finally { + scheduler.shutdown(); + } + }, 10_000); +}); diff --git a/apps/server/src/features/agents/execution/__tests__/execution-worker-handler.test.ts b/apps/server/src/features/agents/execution/__tests__/execution-worker-handler.test.ts index 0f25fd725..89b02f26a 100644 --- a/apps/server/src/features/agents/execution/__tests__/execution-worker-handler.test.ts +++ b/apps/server/src/features/agents/execution/__tests__/execution-worker-handler.test.ts @@ -59,6 +59,7 @@ class RecordingWriter implements ExecutionSemanticWriter { class InlineWorker implements ExecutionWorkerPort { onmessage: ((event: MessageEvent>) => void) | null = null; onerror: ((event: ErrorEvent) => void) | null = null; + onclose: (() => void) | null = null; readonly requests: ExecutionWorkerRequest[] = []; terminated = false; diff --git a/apps/server/src/features/agents/execution/__tests__/fixtures/exit.worker.ts b/apps/server/src/features/agents/execution/__tests__/fixtures/exit.worker.ts new file mode 100644 index 000000000..b7ffa9bc9 --- /dev/null +++ b/apps/server/src/features/agents/execution/__tests__/fixtures/exit.worker.ts @@ -0,0 +1,2 @@ +globalThis.onmessage = (): void => globalThis.close(); +globalThis.postMessage({ kind: "ready" }); diff --git a/apps/server/src/features/agents/execution/execution-mailbox-protocol.ts b/apps/server/src/features/agents/execution/execution-mailbox-protocol.ts index 27ae1e185..922ba2b2b 100644 --- a/apps/server/src/features/agents/execution/execution-mailbox-protocol.ts +++ b/apps/server/src/features/agents/execution/execution-mailbox-protocol.ts @@ -43,6 +43,7 @@ export type ExecutionMailboxCompletion = export interface ExecutionWorkerPort { onmessage: ((event: MessageEvent>) => void) | null; onerror: ((event: ErrorEvent) => void) | null; + onclose: (() => void) | null; postMessage(request: ExecutionWorkerRequest): void; terminate(): void; } diff --git a/apps/server/src/features/agents/execution/execution-mailbox-scheduler.ts b/apps/server/src/features/agents/execution/execution-mailbox-scheduler.ts index 9b6fe8685..cc3fee393 100644 --- a/apps/server/src/features/agents/execution/execution-mailbox-scheduler.ts +++ b/apps/server/src/features/agents/execution/execution-mailbox-scheduler.ts @@ -349,6 +349,7 @@ export class ExecutionMailboxScheduler this.receive(slot, generation, event.data); worker.onerror = () => this.workerLost(slot, generation); + worker.onclose = () => this.workerLost(slot, generation); slot.worker = worker; } @@ -366,9 +367,10 @@ export class ExecutionMailboxScheduler, Result>, generation: number): void { if (this.stopped || slot.generation !== generation) return; - slot.worker?.terminate(); + const worker = slot.worker; slot.worker = undefined; slot.generation += 1; + worker?.terminate(); const revoked = [...this.byThread.values()] .filter((assignment) => assignment.slot === slot) .map((assignment) => assignment.execution); diff --git a/apps/server/src/features/agents/execution/execution-worker-port.ts b/apps/server/src/features/agents/execution/execution-worker-port.ts new file mode 100644 index 000000000..e48fa1320 --- /dev/null +++ b/apps/server/src/features/agents/execution/execution-worker-port.ts @@ -0,0 +1,218 @@ +import type { + ExecutionWorkerPort, + ExecutionWorkerReply, + ExecutionWorkerRequest, +} from "./execution-mailbox-protocol.js"; +import type { ExecutionMailboxCommand } from "./execution-mailbox-scheduler.js"; +import type { + ExecutionSemanticOperation, + ExecutionSemanticWriter, + ExecutionWorkCommand, + ExecutionWorkerResult, + ExecutionWriteReceipt, +} from "./execution-worker-handler.js"; + +type Command = ExecutionMailboxCommand; +type Request = ExecutionWorkerRequest; +type Reply = ExecutionWorkerReply; + +/** Only data crosses the Worker boundary; the single writer remains on the host. */ +export type ExecutionWorkerInbound = + | { readonly kind: "command"; readonly request: Request } + | { readonly kind: "writer-receipt"; readonly rpcId: number; readonly receipt: ExecutionWriteReceipt } + | { readonly kind: "writer-failure"; readonly rpcId: number } + | { readonly kind: "close" }; + +/** The Worker requests one durable transaction and returns its acknowledged result. */ +export type ExecutionWorkerOutbound = + | { readonly kind: "ready" } + | { readonly kind: "writer-request"; readonly rpcId: number; readonly operation: ExecutionSemanticOperation } + | { readonly kind: "command-reply"; readonly reply: Reply } + | { readonly kind: "fatal" } + | { readonly kind: "closed" }; + +/** A real Bun Worker port for one fixed scheduler slot. */ +export class ExecutionThreadWorkerPort implements ExecutionWorkerPort { + onmessage: ((event: MessageEvent) => void) | null = null; + onerror: ((event: ErrorEvent) => void) | null = null; + onclose: (() => void) | null = null; + + private readonly worker: Worker; + private readonly readyPromise: Promise; + private resolveReady: (ready: boolean) => void = () => {}; + private closePromise: Promise | undefined; + private resolveClose: ((graceful: boolean) => void) | undefined; + private closeTimer: ReturnType | undefined; + private state: "starting" | "ready" | "closing" | "closed" = "starting"; + private queuedBeforeReady: Request | undefined; + private inFlight: Request | undefined; + private writerRpcPending = false; + private writerCalled = false; + + constructor( + private readonly writer: ExecutionSemanticWriter, + workerUrl: URL = defaultWorkerUrl(), + ) { + this.readyPromise = new Promise((resolve) => { this.resolveReady = resolve; }); + this.worker = new Worker(workerUrl, { type: "module" }); + this.worker.onmessage = (event: MessageEvent) => this.receive(event.data); + this.worker.onerror = (event) => this.fail(event); + this.worker.onmessageerror = () => this.fail(new ErrorEvent("error")); + this.worker.addEventListener("close", () => this.closed()); + } + + /** Startup resolves false if the Worker exits before it can accept commands. */ + whenReady(): Promise { + return this.readyPromise; + } + + postMessage(request: Request): void { + if (this.state === "closed" || this.state === "closing") throw new Error("Execution worker is closed"); + if (this.inFlight) throw new Error("Execution worker already has an in-flight command"); + this.inFlight = request; + this.writerCalled = false; + if (this.state === "starting") this.queuedBeforeReady = request; + else this.send({ kind: "command", request }); + } + + /** Ask the Worker to close, then force termination if it does not respond. */ + close(): Promise { + if (this.state === "closed") return Promise.resolve(true); + if (this.closePromise) return this.closePromise; + this.state = "closing"; + this.closePromise = new Promise((resolve) => { this.resolveClose = resolve; }); + this.closeTimer = setTimeout(() => this.terminate(), 2_000); + this.send({ kind: "close" }); + return this.closePromise; + } + + terminate(): void { + if (this.state === "closed") return; + this.state = "closed"; + this.worker.terminate(); + this.finishClose(false); + } + + private receive(message: ExecutionWorkerOutbound): void { + if (this.state === "closed") return; + switch (message.kind) { + case "ready": + this.ready(); + break; + case "writer-request": + void this.write(message.rpcId, message.operation); + break; + case "command-reply": + this.reply(message.reply); + break; + case "fatal": + this.fail(new ErrorEvent("error")); + break; + case "closed": + this.closed(); + break; + default: { + const exhaustive: never = message; + throw new Error(`Unknown execution worker message: ${String(exhaustive)}`); + } + } + } + + private ready(): void { + if (this.state !== "starting") return; + this.state = "ready"; + this.resolveReady(true); + const request = this.queuedBeforeReady; + this.queuedBeforeReady = undefined; + if (request) this.send({ kind: "command", request }); + } + + private async write(rpcId: number, operation: ExecutionSemanticOperation): Promise { + if (this.state !== "ready" || this.writerRpcPending || this.writerCalled || !matchesOperation(this.inFlight, operation)) { + this.fail(new ErrorEvent("error")); + return; + } + this.writerRpcPending = true; + this.writerCalled = true; + try { + const receipt = await this.writer.transact(operation); + if (this.state === "ready") this.send({ kind: "writer-receipt", rpcId, receipt }); + } catch { + if (this.state === "ready") this.send({ kind: "writer-failure", rpcId }); + } finally { + this.writerRpcPending = false; + } + } + + private reply(reply: Reply): void { + if (this.state !== "ready" || !matchesReply(this.inFlight, reply)) { + this.fail(new ErrorEvent("error")); + return; + } + this.inFlight = undefined; + this.onmessage?.(new MessageEvent("message", { data: reply })); + } + + private send(message: ExecutionWorkerInbound): void { + try { + this.worker.postMessage(message); + } catch { + this.fail(new ErrorEvent("error")); + } + } + + private fail(event: ErrorEvent): void { + if (this.state === "closed") return; + try { + this.onerror?.(event); + } finally { + this.terminate(); + } + } + + private closed(): void { + if (this.state === "closed") return; + const graceful = this.state === "closing"; + this.state = "closed"; + this.worker.terminate(); + this.finishClose(graceful); + this.onclose?.(); + } + + private finishClose(graceful: boolean): void { + if (this.closeTimer) clearTimeout(this.closeTimer); + this.closeTimer = undefined; + this.resolveReady(false); + this.resolveClose?.(graceful); + this.resolveClose = undefined; + } +} + +function defaultWorkerUrl(): URL { + const workerFile = import.meta.url.endsWith(".cjs") ? "./execution.worker.cjs" : "./execution.worker.ts"; + return new URL(workerFile, import.meta.url); +} + +function matchesOperation(request: Request | undefined, operation: ExecutionSemanticOperation): boolean { + return request !== undefined && request.ordinal === operation.ordinal + && request.execution.threadId === operation.execution.threadId + && request.execution.turnId === operation.execution.turnId + && request.execution.executionId === operation.execution.executionId + && request.lease.leaseId === operation.lease.leaseId + && request.lease.ownerEpoch === operation.lease.ownerEpoch + && request.lease.workerGeneration === operation.lease.workerGeneration + && request.lease.workerIndex === operation.lease.workerIndex + && operation.operationId === `${request.lease.leaseId}:${request.ordinal}`; +} + +function matchesReply(request: Request | undefined, reply: Reply): boolean { + return request !== undefined && request.requestId === reply.requestId + && request.ordinal === reply.ordinal + && request.execution.threadId === reply.execution.threadId + && request.execution.turnId === reply.execution.turnId + && request.execution.executionId === reply.execution.executionId + && request.lease.leaseId === reply.lease.leaseId + && request.lease.ownerEpoch === reply.lease.ownerEpoch + && request.lease.workerGeneration === reply.lease.workerGeneration + && request.lease.workerIndex === reply.lease.workerIndex; +} diff --git a/apps/server/src/features/agents/execution/execution.worker.ts b/apps/server/src/features/agents/execution/execution.worker.ts new file mode 100644 index 000000000..95342481d --- /dev/null +++ b/apps/server/src/features/agents/execution/execution.worker.ts @@ -0,0 +1,114 @@ +import { + ExecutionWorkerHandler, + type ExecutionSemanticOperation, + type ExecutionSemanticWriter, + type ExecutionWriteReceipt, +} from "./execution-worker-handler.js"; +import type { ExecutionWorkerInbound, ExecutionWorkerOutbound } from "./execution-worker-port.js"; + +interface PendingWrite { + readonly rpcId: number; + readonly operationId: string; + readonly resolve: (receipt: ExecutionWriteReceipt) => void; + readonly reject: () => void; +} + +let nextRpcId = 1; +let pendingWrite: PendingWrite | undefined; +let commandActive = false; +let closed = false; + +const writer: ExecutionSemanticWriter = { + transact: (operation: ExecutionSemanticOperation): Promise => { + if (closed || pendingWrite) return Promise.reject(new Error("Execution writer RPC unavailable")); + const rpcId = nextRpcId++; + return new Promise((resolve, reject) => { + pendingWrite = { + rpcId, + operationId: operation.operationId, + resolve, + reject: () => reject(new Error("Execution writer RPC failed")), + }; + post({ kind: "writer-request", rpcId, operation }); + }); + }, +}; + +const handler = new ExecutionWorkerHandler(writer); + +globalThis.onmessage = (event: MessageEvent): void => { + const message = event.data; + switch (message.kind) { + case "command": + handleCommand(message.request); + break; + case "writer-receipt": + finishWrite(message.rpcId, message.receipt); + break; + case "writer-failure": + failWrite(message.rpcId); + break; + case "close": + closeWorker(); + break; + default: { + const exhaustive: never = message; + throw new Error(`Unknown execution worker input: ${String(exhaustive)}`); + } + } +}; + +post({ kind: "ready" }); + +function handleCommand(request: Extract["request"]): void { + if (closed || commandActive) { + failWorker(); + return; + } + commandActive = true; + void handler.handle(request) + .then((reply) => { + if (!closed) post({ kind: "command-reply", reply }); + }) + .catch(() => failWorker()) + .finally(() => { commandActive = false; }); +} + +function finishWrite(rpcId: number, receipt: ExecutionWriteReceipt): void { + const pending = pendingWrite; + if (!pending || pending.rpcId !== rpcId || pending.operationId !== receipt.operationId) { + failWorker(); + return; + } + pendingWrite = undefined; + pending.resolve(receipt); +} + +function failWrite(rpcId: number): void { + const pending = pendingWrite; + if (!pending || pending.rpcId !== rpcId) { + failWorker(); + return; + } + pendingWrite = undefined; + pending.reject(); +} + +function failWorker(): void { + if (closed) return; + post({ kind: "fatal" }); + closeWorker(); +} + +function closeWorker(): void { + if (closed) return; + closed = true; + pendingWrite?.reject(); + pendingWrite = undefined; + post({ kind: "closed" }); + globalThis.close(); +} + +function post(message: ExecutionWorkerOutbound): void { + globalThis.postMessage(message); +} diff --git a/scripts/build-server-dev-bundle.mjs b/scripts/build-server-dev-bundle.mjs index 8af56652e..5ab323205 100644 --- a/scripts/build-server-dev-bundle.mjs +++ b/scripts/build-server-dev-bundle.mjs @@ -574,13 +574,14 @@ export function findCopilotSdkPath(serverCjsOut, platform, arch) { return NodeFS.existsSync(copilotIndex) && NodeFS.existsSync(platformEntry) ? copilotIndex : undefined; } -/** Bundle the server and its isolated PTY host from one compiled server tree. */ +/** Bundle the server and its isolated workers from one compiled server tree. */ export async function buildServerRuntimeBundles({ serverRoot, serverOutFile, ptyHostOutFile, providerEventWorkerOutFile = NodePath.resolve(NodePath.dirname(serverOutFile), "provider-event.worker.cjs"), canonicalWriterWorkerOutFile = NodePath.resolve(NodePath.dirname(serverOutFile), "canonical-agent-writer.worker.cjs"), + executionWorkerOutFile = NodePath.resolve(NodePath.dirname(serverOutFile), "execution.worker.cjs"), production = false, }) { const shared = { @@ -626,6 +627,11 @@ export async function buildServerRuntimeBundles({ entryPoints: [NodePath.resolve(serverRoot, "dist-tsc/features/agents/canonical/canonical-agent-writer.worker.js")], outfile: canonicalWriterWorkerOutFile, })); + bundles.push(build({ + ...shared, + entryPoints: [NodePath.resolve(serverRoot, "dist-tsc/features/agents/execution/execution.worker.js")], + outfile: executionWorkerOutFile, + })); await Promise.all(bundles); } @@ -663,5 +669,5 @@ export async function rebuildServerDevBundle(options = {}) { console.log(`[server-dev-bundle] Copied Drizzle migrations -> ${drizzleDst}`); } - console.log(`[server-dev-bundle] Complete: ${serverOutFile}, ${ptyHostOutFile}`); + console.log(`[server-dev-bundle] Complete: ${serverOutFile}, ${ptyHostOutFile}, execution.worker.cjs`); } From 93f70774c6d6d27c5e1eb4be6f4b5b1170e2843c Mon Sep 17 00:00:00 2001 From: "(CJ) Chukwudi Nwobodo" <142016413+chuks-qua@users.noreply.github.com> Date: Thu, 24 Sep 2026 20:00:56 +0100 Subject: [PATCH 018/136] feat(server): carry canonical turn inputs through worker commands --- .../execution-thread-worker-port.test.ts | 29 ++++++++- .../execution-worker-handler.test.ts | 52 ++++++++++++++-- .../execution/execution-worker-handler.ts | 59 +++++++++++++++---- 3 files changed, 122 insertions(+), 18 deletions(-) diff --git a/apps/server/src/features/agents/execution/__tests__/execution-thread-worker-port.test.ts b/apps/server/src/features/agents/execution/__tests__/execution-thread-worker-port.test.ts index 4f2af48d9..07b24f41e 100644 --- a/apps/server/src/features/agents/execution/__tests__/execution-thread-worker-port.test.ts +++ b/apps/server/src/features/agents/execution/__tests__/execution-thread-worker-port.test.ts @@ -1,6 +1,10 @@ import type { ProviderEventDraft } from "@mcode/providers"; import { describe, expect, it } from "vitest"; +import type { + DataOnlyParentTurnFinishInput, + DataOnlyParentTurnStartInput, +} from "../../canonical/canonical-parent-turn-write.js"; import { ExecutionMailboxScheduler, type ExecutionMailboxLimits } from "../execution-mailbox-scheduler.js"; import type { ExecutionIdentity, ExecutionLease } from "../execution-mailbox-protocol.js"; import { ExecutionThreadWorkerPort } from "../execution-worker-port.js"; @@ -18,6 +22,25 @@ const EXECUTION: ExecutionIdentity = { executionId: "00000000-0000-4000-8000-000000000002", }; +const START_INPUT: DataOnlyParentTurnStartInput = { + thread: { id: EXECUTION.threadId, workspaceId: "fixture-workspace", providerId: "codex", createdAt: "2026-09-24T12:00:00.000Z" }, + turnId: EXECUTION.turnId, + executionId: EXECUTION.executionId, + permissionMode: "full", + providerIdentities: [], + userMessage: { kind: "create", content: "Test", sequence: 1 }, +}; + +const FINISH_INPUT: DataOnlyParentTurnFinishInput = { + threadId: EXECUTION.threadId, + turnId: EXECUTION.turnId, + executionId: EXECUTION.executionId, + providerId: "codex", + providerIdentities: [], + outcome: "cancelled", + projection: { message: null, narrative: [] }, +}; + const LIMITS: ExecutionMailboxLimits = { maxPending: 8, maxPendingBytes: 8_000, @@ -92,7 +115,7 @@ describe("ExecutionThreadWorkerPort", () => { const writer = new AcknowledgingWriter(); const { scheduler, port, lease } = fixture(writer); try { - await expect(submit(scheduler, lease, { kind: "start", providerId: "codex" })) + await expect(submit(scheduler, lease, { kind: "start", providerId: "codex", input: START_INPUT })) .resolves.toMatchObject({ kind: "reply", result: { kind: "committed", durableRevision: 1 } }); await expect(port.whenReady()).resolves.toBe(true); await expect(submit(scheduler, lease, { kind: "event", events: [eventDraft()] })) @@ -101,7 +124,7 @@ describe("ExecutionThreadWorkerPort", () => { .resolves.toMatchObject({ kind: "reply", result: { kind: "committed", durableRevision: 3 } }); await expect(submit(scheduler, lease, { kind: "checkpoint", phase: "stopping", nativeCursor: null })) .resolves.toMatchObject({ kind: "reply", result: { kind: "committed", durableRevision: 4 } }); - await expect(submit(scheduler, lease, { kind: "finalize", outcome: "cancelled" })) + await expect(submit(scheduler, lease, { kind: "finalize", outcome: "cancelled", input: FINISH_INPUT })) .resolves.toMatchObject({ kind: "reply", result: { kind: "committed", durableRevision: 5 } }); await expect(submit(scheduler, lease, { kind: "release" })) .resolves.toEqual({ kind: "reply", result: { kind: "released" } }); @@ -121,7 +144,7 @@ describe("ExecutionThreadWorkerPort", () => { const { scheduler, port, lost, lease } = fixture(new AcknowledgingWriter(), exitWorkerUrl); try { await expect(port.whenReady()).resolves.toBe(true); - await expect(submit(scheduler, lease, { kind: "start", providerId: "codex" })) + await expect(submit(scheduler, lease, { kind: "start", providerId: "codex", input: START_INPUT })) .resolves.toEqual({ kind: "worker-lost" }); expect(lost).toEqual([[EXECUTION]]); expect(scheduler.depth()).toMatchObject({ pending: 0, activeExecutions: 0 }); diff --git a/apps/server/src/features/agents/execution/__tests__/execution-worker-handler.test.ts b/apps/server/src/features/agents/execution/__tests__/execution-worker-handler.test.ts index 89b02f26a..751e19b36 100644 --- a/apps/server/src/features/agents/execution/__tests__/execution-worker-handler.test.ts +++ b/apps/server/src/features/agents/execution/__tests__/execution-worker-handler.test.ts @@ -1,6 +1,10 @@ import type { ProviderEventDraft } from "@mcode/providers"; import { describe, expect, it } from "vitest"; +import type { + DataOnlyParentTurnFinishInput, + DataOnlyParentTurnStartInput, +} from "../../canonical/canonical-parent-turn-write.js"; import { ExecutionMailboxScheduler, type ExecutionMailboxCommand, @@ -30,6 +34,25 @@ const EXECUTION: ExecutionIdentity = { executionId: "00000000-0000-4000-8000-000000000001", }; +const START_INPUT: DataOnlyParentTurnStartInput = { + thread: { id: EXECUTION.threadId, workspaceId: "fixture-workspace", providerId: "codex", createdAt: "2026-09-24T12:00:00.000Z" }, + turnId: EXECUTION.turnId, + executionId: EXECUTION.executionId, + permissionMode: "full", + providerIdentities: [], + userMessage: { kind: "create", content: "Test", sequence: 1 }, +}; + +const FINISH_INPUT: DataOnlyParentTurnFinishInput = { + threadId: EXECUTION.threadId, + turnId: EXECUTION.turnId, + executionId: EXECUTION.executionId, + providerId: "codex", + providerIdentities: [], + outcome: "cancelled", + projection: { message: null, narrative: [] }, +}; + const LIMITS: ExecutionMailboxLimits = { maxPending: 12, maxPendingBytes: 12_000, @@ -126,7 +149,7 @@ async function committed(admission: ReturnType, revision: number) describe("ExecutionWorkerHandler through its scheduler", () => { it("runs one synthetic turn from start through Stop, checkpoint, and finalization", async () => { const { scheduler, worker, writer, lease } = fixture(); - await committed(submit(scheduler, lease, { kind: "start", providerId: "codex" }), 1); + await committed(submit(scheduler, lease, { kind: "start", providerId: "codex", input: START_INPUT }), 1); await committed(submit(scheduler, lease, { kind: "event", events: [eventDraft()] }), 2); const stop = submit(scheduler, lease, { kind: "stop", requestId: "stop-1" }); expect(scheduler.submit({ @@ -139,7 +162,7 @@ describe("ExecutionWorkerHandler through its scheduler", () => { await committed(submit(scheduler, lease, { kind: "checkpoint", phase: "stopping", nativeCursor: "cursor-1" }), 4); await committed(submit(scheduler, lease, { kind: "effect-result", effectId: "file-1", settled: true }), 5); await committed(submit(scheduler, lease, { kind: "provider-outcome", outcome: "cancelled" }), 6); - await committed(submit(scheduler, lease, { kind: "finalize", outcome: "cancelled" }), 7); + await committed(submit(scheduler, lease, { kind: "finalize", outcome: "cancelled", input: FINISH_INPUT }), 7); await expect(submit(scheduler, lease, { kind: "release" }).completion).resolves.toEqual({ kind: "reply", result: { kind: "released" }, @@ -164,7 +187,7 @@ describe("ExecutionWorkerHandler through its scheduler", () => { let releaseWrite: (() => void) | undefined; writer.beforeCommit = () => new Promise((resolve) => { releaseWrite = resolve; }); const { scheduler, worker, lease } = fixture(writer); - const start = submit(scheduler, lease, { kind: "start", providerId: "codex" }); + const start = submit(scheduler, lease, { kind: "start", providerId: "codex", input: START_INPUT }); const event = submit(scheduler, lease, { kind: "event", events: [eventDraft()] }); await Promise.resolve(); expect(worker.requests).toHaveLength(1); @@ -177,11 +200,30 @@ describe("ExecutionWorkerHandler through its scheduler", () => { scheduler.shutdown(); }); + it("rejects a start whose transaction input names another execution", async () => { + const { scheduler, writer, lease } = fixture(); + const input = { ...START_INPUT, executionId: "another-execution" }; + await expect(submit(scheduler, lease, { kind: "start", providerId: "codex", input }).completion) + .resolves.toEqual({ kind: "reply", result: { kind: "rejected", reason: "invalid-transition" } }); + expect(writer.operations).toEqual([]); + scheduler.shutdown(); + }); + + it("rejects finalization when its projected outcome disagrees with the command", async () => { + const { scheduler, writer, lease } = fixture(); + await committed(submit(scheduler, lease, { kind: "start", providerId: "codex", input: START_INPUT }), 1); + const input = { ...FINISH_INPUT, outcome: "completed" as const }; + await expect(submit(scheduler, lease, { kind: "finalize", outcome: "cancelled", input }).completion) + .resolves.toEqual({ kind: "reply", result: { kind: "rejected", reason: "invalid-transition" } }); + expect(writer.operations.map((operation) => operation.mutation.kind)).toEqual(["begin"]); + scheduler.shutdown(); + }); + it("rejects a writer conflict without reporting a durable command", async () => { const writer = new RecordingWriter(); writer.conflictKind = "append-events"; const { scheduler, lease } = fixture(writer); - await committed(submit(scheduler, lease, { kind: "start", providerId: "codex" }), 1); + await committed(submit(scheduler, lease, { kind: "start", providerId: "codex", input: START_INPUT }), 1); await expect(submit(scheduler, lease, { kind: "event", events: [eventDraft()] }).completion).resolves.toEqual({ kind: "reply", result: { kind: "rejected", reason: "writer-conflict" }, @@ -197,7 +239,7 @@ describe("ExecutionWorkerHandler through its scheduler", () => { if (operation.mutation.kind === "append-events") throw new Error("database unavailable"); }; const { scheduler, worker, lost, lease } = fixture(writer); - await committed(submit(scheduler, lease, { kind: "start", providerId: "codex" }), 1); + await committed(submit(scheduler, lease, { kind: "start", providerId: "codex", input: START_INPUT }), 1); await expect(submit(scheduler, lease, { kind: "event", events: [eventDraft()] }).completion) .resolves.toEqual({ kind: "worker-lost" }); expect(lost).toEqual([[EXECUTION]]); diff --git a/apps/server/src/features/agents/execution/execution-worker-handler.ts b/apps/server/src/features/agents/execution/execution-worker-handler.ts index b475239e0..891b88751 100644 --- a/apps/server/src/features/agents/execution/execution-worker-handler.ts +++ b/apps/server/src/features/agents/execution/execution-worker-handler.ts @@ -1,6 +1,10 @@ import type { TurnOutcome } from "@mcode/contracts"; import type { ProviderEventDraft } from "@mcode/providers"; +import type { + DataOnlyParentTurnFinishInput, + DataOnlyParentTurnStartInput, +} from "../canonical/canonical-parent-turn-write.js"; import type { ExecutionIdentity, ExecutionLease, @@ -11,13 +15,13 @@ import type { ExecutionMailboxCommand } from "./execution-mailbox-scheduler.js"; /** Data that an execution worker may receive without a server dependency container. */ export type ExecutionWorkCommand = - | { readonly kind: "start"; readonly providerId: string } + | { readonly kind: "start"; readonly providerId: string; readonly input: DataOnlyParentTurnStartInput } | { readonly kind: "resume"; readonly providerId: string; readonly checkpointId: string } | { readonly kind: "event"; readonly events: readonly ProviderEventDraft[] } | { readonly kind: "checkpoint"; readonly phase: string; readonly nativeCursor: unknown | null } | { readonly kind: "effect-result"; readonly effectId: string; readonly settled: boolean } | { readonly kind: "provider-outcome"; readonly outcome: TurnOutcome } - | { readonly kind: "finalize"; readonly outcome: TurnOutcome } + | { readonly kind: "finalize"; readonly outcome: TurnOutcome; readonly input: DataOnlyParentTurnFinishInput } | { readonly kind: "release" }; /** One complete semantic mutation. The single writer decides its SQL transaction. */ @@ -27,13 +31,14 @@ export interface ExecutionSemanticOperation { readonly lease: ExecutionLease; readonly ordinal: number; readonly mutation: - | { readonly kind: "begin"; readonly providerId: string; readonly checkpointId?: string } + | { readonly kind: "begin"; readonly providerId: string; readonly input: DataOnlyParentTurnStartInput } + | { readonly kind: "resume"; readonly providerId: string; readonly checkpointId: string } | { readonly kind: "append-events"; readonly events: readonly ProviderEventDraft[] } | { readonly kind: "checkpoint"; readonly phase: string; readonly nativeCursor: unknown | null } | { readonly kind: "stop-requested"; readonly requestId: string; readonly lastAdmittedOrdinal: number } | { readonly kind: "effect-result"; readonly effectId: string; readonly settled: boolean } | { readonly kind: "provider-outcome"; readonly outcome: TurnOutcome } - | { readonly kind: "finish"; readonly outcome: TurnOutcome }; + | { readonly kind: "finish"; readonly outcome: TurnOutcome; readonly input: DataOnlyParentTurnFinishInput }; } /** A writer reply is valid only after the semantic operation commits durably. */ @@ -61,6 +66,7 @@ type WorkerCommand = ExecutionMailboxCommand; interface ExecutionState { readonly execution: ExecutionIdentity; readonly lease: ExecutionLease; + readonly providerId: string; nextOrdinal: number; phase: "running" | "stopping" | "finalized"; durableRevision: number; @@ -126,11 +132,12 @@ export class ExecutionWorkerHandler { if (request.ordinal !== 1) return { kind: "rejected", reason: "out-of-order" }; const command = request.command; if (command.kind !== "start" && command.kind !== "resume") return { kind: "rejected", reason: "invalid-transition" }; - const mutation: ExecutionSemanticOperation["mutation"] = { - kind: "begin", - providerId: command.providerId, - ...(command.kind === "resume" ? { checkpointId: command.checkpointId } : {}), - }; + if (command.kind === "start" && !validStartInput(command, request.execution)) { + return { kind: "rejected", reason: "invalid-transition" }; + } + const mutation: ExecutionSemanticOperation["mutation"] = command.kind === "start" + ? { kind: "begin", providerId: command.providerId, input: command.input } + : { kind: "resume", providerId: command.providerId, checkpointId: command.checkpointId }; const receipt = await this.writer.transact(operationFor(request, mutation)); if (!isDurableReceipt(receipt, request, 0)) { return { kind: "rejected", reason: "writer-conflict" }; @@ -138,6 +145,7 @@ export class ExecutionWorkerHandler { this.states.set(request.execution.threadId, { execution: request.execution, lease: request.lease, + providerId: command.providerId, nextOrdinal: 2, phase: "running", durableRevision: receipt.durableRevision, @@ -169,7 +177,7 @@ function mutationFor( case "provider-outcome": return { kind: "provider-outcome", outcome: command.outcome }; case "finalize": - return { kind: "finish", outcome: command.outcome }; + return finishMutation(command, request.execution, state.providerId); case "start": case "resume": case "release": @@ -199,6 +207,15 @@ function stopMutation( return { kind: "stop-requested", requestId: command.requestId, lastAdmittedOrdinal: request.stopWatermark }; } +function finishMutation( + command: Extract, + execution: ExecutionIdentity, + providerId: string, +): ExecutionSemanticOperation["mutation"] | undefined { + if (!validFinishInput(command, execution, providerId)) return undefined; + return { kind: "finish", outcome: command.outcome, input: command.input }; +} + function commandRejection( request: ExecutionWorkerRequest, state: ExecutionState, @@ -221,6 +238,28 @@ function validEventRouting(events: readonly ProviderEventDraft[], execution: Exe && event.routing.turnId === execution.turnId && event.routing.executionId === execution.executionId); } +function validStartInput( + command: Extract, + execution: ExecutionIdentity, +): boolean { + return command.providerId === command.input.thread.providerId + && command.input.thread.id === execution.threadId + && command.input.turnId === execution.turnId + && command.input.executionId === execution.executionId; +} + +function validFinishInput( + command: Extract, + execution: ExecutionIdentity, + providerId: string, +): boolean { + return command.outcome === command.input.outcome + && command.input.providerId === providerId + && command.input.threadId === execution.threadId + && command.input.turnId === execution.turnId + && command.input.executionId === execution.executionId; +} + function operationFor( request: ExecutionWorkerRequest, mutation: ExecutionSemanticOperation["mutation"], From 39621f2007cf074d55ae4bf4c1e6f339b062a1ae Mon Sep 17 00:00:00 2001 From: "(CJ) Chukwudi Nwobodo" <142016413+chuks-qua@users.noreply.github.com> Date: Thu, 24 Sep 2026 20:01:23 +0100 Subject: [PATCH 019/136] fix(server): retry failed writer acknowledgements --- .../canonical-agent-writer-client.test.ts | 48 ++++++++ .../canonical-agent-writer-client.ts | 116 +++++++++++++++--- 2 files changed, 148 insertions(+), 16 deletions(-) diff --git a/apps/server/src/features/agents/canonical/__tests__/canonical-agent-writer-client.test.ts b/apps/server/src/features/agents/canonical/__tests__/canonical-agent-writer-client.test.ts index 4c170931a..a5eef21bc 100644 --- a/apps/server/src/features/agents/canonical/__tests__/canonical-agent-writer-client.test.ts +++ b/apps/server/src/features/agents/canonical/__tests__/canonical-agent-writer-client.test.ts @@ -412,4 +412,52 @@ describe("canonical SQLite writer", () => { expect(db.prepare("SELECT COUNT(*) AS count FROM canonical_writer_operation_receipts").get()) .toEqual({ count: 0 }); }, 30_000); + + it("retries a failed acknowledgement before the next write", async () => { + writer = new CanonicalAgentWriterClient(dbPath); + const batch = { threadId: THREAD_ID, turnId: TURN_ID, executionId: EXECUTION_ID, phase: "running", events: events() }; + await writer.commit("ack-retry-start", batch); + db.run(`CREATE TRIGGER reject_receipt_delete BEFORE DELETE ON canonical_writer_operation_receipts + BEGIN SELECT RAISE(FAIL, 'ack unavailable'); END`); + await expect(writer.acknowledgeOperation(EXECUTION_ID, "ack-retry-start")) + .rejects.toThrow("Canonical writer write-failed"); + expect(writer.pendingAcknowledgementCount).toBe(1); + db.run("DROP TRIGGER reject_receipt_delete"); + expect(await writer.commit("ack-retry-next", batch)).toMatchObject({ outcome: "duplicate" }); + expect(writer.pendingAcknowledgementCount).toBe(0); + expect(db.prepare("SELECT operation_id FROM canonical_writer_operation_receipts").all()) + .toEqual([{ operation_id: "ack-retry-next" }]); + }); + + it("retries failed acknowledgements on close", async () => { + writer = new CanonicalAgentWriterClient(dbPath); + await writer.commit("ack-close-start", { + threadId: THREAD_ID, turnId: TURN_ID, executionId: EXECUTION_ID, phase: "running", events: events(), + }); + db.run(`CREATE TRIGGER reject_receipt_delete BEFORE DELETE ON canonical_writer_operation_receipts + BEGIN SELECT RAISE(FAIL, 'ack unavailable'); END`); + await expect(writer.acknowledgeOperation(EXECUTION_ID, "ack-close-start")) + .rejects.toThrow("Canonical writer write-failed"); + db.run("DROP TRIGGER reject_receipt_delete"); + await writer.close(); + expect(writer.pendingAcknowledgementCount).toBe(0); + expect(db.prepare("SELECT COUNT(*) AS count FROM canonical_writer_operation_receipts").get()) + .toEqual({ count: 0 }); + }); + + it("bounds failed acknowledgement retention and reports unfinished close cleanup", async () => { + writer = new CanonicalAgentWriterClient(dbPath); + const batch = { threadId: THREAD_ID, turnId: TURN_ID, executionId: EXECUTION_ID, phase: "running", events: events() }; + for (let index = 0; index <= 64; index++) await writer.commit(`ack-full-${index}`, batch); + db.run(`CREATE TRIGGER reject_receipt_delete BEFORE DELETE ON canonical_writer_operation_receipts + BEGIN SELECT RAISE(FAIL, 'ack unavailable'); END`); + for (let index = 0; index < 64; index++) { + await expect(writer.acknowledgeOperation(EXECUTION_ID, `ack-full-${index}`)) + .rejects.toThrow("Canonical writer write-failed"); + } + await expect(writer.acknowledgeOperation(EXECUTION_ID, "ack-full-64")) + .rejects.toThrow("Canonical writer acknowledgement retry queue is full"); + expect(writer.pendingAcknowledgementCount).toBe(64); + await expect(writer.close()).rejects.toThrow("64 unacknowledged operations"); + }, 30_000); }); diff --git a/apps/server/src/features/agents/canonical/canonical-agent-writer-client.ts b/apps/server/src/features/agents/canonical/canonical-agent-writer-client.ts index da7bdf302..1c512c211 100644 --- a/apps/server/src/features/agents/canonical/canonical-agent-writer-client.ts +++ b/apps/server/src/features/agents/canonical/canonical-agent-writer-client.ts @@ -12,6 +12,9 @@ import type { ParentNarrativeRecoveryCommitInput } from "./canonical-agent-bound const MAX_PENDING_WRITES = 64; const MAX_WORKER_ATTEMPTS = 3; +const MAX_PENDING_ACKNOWLEDGEMENTS = 64; +const MAX_ACK_RETRIES_BEFORE_WRITE = 8; +const MAX_ACK_RETRIES_ON_CLOSE = 16; const WRITER_EXECUTION_ID = "writer:init"; interface PendingRequest { @@ -20,6 +23,11 @@ interface PendingRequest { reject(error: Error): void; } +interface PendingAcknowledgement { + executionId: string; + operationId: string; +} + class CanonicalWriterWorkerLost extends Error {} /** Sends cloneable canonical commands to a dedicated SQLite worker with bounded admission and crash retries. */ @@ -27,8 +35,11 @@ export class CanonicalAgentWriterClient { private worker: Worker | undefined; private workerReady: Promise; private restart: Promise | undefined; + private retryingAcknowledgements: Promise | undefined; private readonly pending = new Map(); + private readonly pendingAcknowledgements = new Map(); private generation = 0; + private closing = false; private stopping = false; constructor( @@ -47,6 +58,7 @@ export class CanonicalAgentWriterClient { /** Resolves with committed envelopes for main-loop publication; it never publishes them itself. */ async commit(operationId: string, input: CanonicalProviderWriteInput): Promise { if (!operationId || !input.executionId) throw new Error("Canonical writer operation and execution IDs are required"); + await this.retryPendingAcknowledgements(MAX_ACK_RETRIES_BEFORE_WRITE); const response = await this.sendWithRetry({ kind: "commit", requestId: NodeCrypto.randomUUID(), operationId, executionId: input.executionId, input, }); @@ -60,6 +72,7 @@ export class CanonicalAgentWriterClient { input: ParentNarrativeRecoveryCommitInput, ): Promise { if (!operationId || !input.executionId) throw new Error("Canonical writer operation and execution IDs are required"); + await this.retryPendingAcknowledgements(MAX_ACK_RETRIES_BEFORE_WRITE); const response = await this.sendWithRetry({ kind: "record-parent-narrative-recovery", requestId: NodeCrypto.randomUUID(), operationId, executionId: input.executionId, input, @@ -76,6 +89,7 @@ export class CanonicalAgentWriterClient { input: ParentNarrativeRecoveryCommitInput, ): Promise { if (!operationId || !input.executionId) throw new Error("Canonical writer operation and execution IDs are required"); + await this.retryPendingAcknowledgements(MAX_ACK_RETRIES_BEFORE_WRITE); const response = await this.sendWithRetry({ kind: "classify-parent-narrative-recovery", requestId: NodeCrypto.randomUUID(), operationId, executionId: input.executionId, input, @@ -89,19 +103,39 @@ export class CanonicalAgentWriterClient { /** Releases a receipt after its caller has finished publication or journal discard. Never acknowledge before that work. */ async acknowledgeOperation(executionId: string, operationId: string): Promise { if (!executionId || !operationId) throw new Error("Canonical writer operation and execution IDs are required"); - const response = await this.sendWithRetry({ - kind: "ack-operation", requestId: NodeCrypto.randomUUID(), operationId, executionId, - }); - if (response.kind !== "operation-acknowledged") { - throw new Error("Canonical writer returned an unexpected response"); + if (this.closing || this.stopping) throw new Error("Canonical writer closed"); + const key = acknowledgementKey(executionId, operationId); + try { + await this.sendAcknowledgement({ executionId, operationId }); + this.pendingAcknowledgements.delete(key); + } catch (error) { + if (!this.pendingAcknowledgements.has(key)) { + if (this.pendingAcknowledgements.size >= MAX_PENDING_ACKNOWLEDGEMENTS) { + throw new Error("Canonical writer acknowledgement retry queue is full", { cause: error }); + } + this.pendingAcknowledgements.set(key, { executionId, operationId }); + } + throw error; } } + /** Number of handled operations whose receipt deletion still needs a retry. */ + get pendingAcknowledgementCount(): number { + return this.pendingAcknowledgements.size; + } + /** Closes the database after accepted writes settle, then releases the worker. */ async close(): Promise { - if (this.stopping) return; - this.stopping = true; + if (this.closing || this.stopping) return; + this.closing = true; + let acknowledgementError: Error | undefined; try { + await this.retryPendingAcknowledgements(MAX_ACK_RETRIES_ON_CLOSE); + if (this.pendingAcknowledgements.size > 0) { + acknowledgementError = new Error( + `Canonical writer closed with ${this.pendingAcknowledgements.size} unacknowledged operations`, + ); + } await this.workerReady; if (this.worker) { await this.sendRaw({ @@ -112,30 +146,76 @@ export class CanonicalAgentWriterClient { } catch { // A failed worker is already closed; caller writes have their own visible failures. } finally { + this.stopping = true; this.loseWorker(new Error("Canonical writer closed")); } + if (acknowledgementError) throw acknowledgementError; + } + + private sendAcknowledgement(acknowledgement: PendingAcknowledgement, allowDuringClose = false): Promise { + return this.sendWithRetry({ + kind: "ack-operation", requestId: NodeCrypto.randomUUID(), ...acknowledgement, + }, allowDuringClose).then((response) => { + if (response.kind !== "operation-acknowledged") { + throw new Error("Canonical writer returned an unexpected response"); + } + }); } - private async sendWithRetry(request: CanonicalWriterRequest): Promise { + private retryPendingAcknowledgements(limit: number): Promise { + if (this.pendingAcknowledgements.size === 0) return Promise.resolve(); + if (!this.retryingAcknowledgements) { + this.retryingAcknowledgements = this.drainAcknowledgements(limit) + .finally(() => { this.retryingAcknowledgements = undefined; }); + } + return this.retryingAcknowledgements; + } + + private async drainAcknowledgements(limit: number): Promise { + for (const [key, acknowledgement] of [...this.pendingAcknowledgements].slice(0, limit)) { + try { + await this.sendAcknowledgement(acknowledgement, this.closing); + this.pendingAcknowledgements.delete(key); + } catch { + if (this.pendingAcknowledgements.get(key) !== acknowledgement) continue; + this.pendingAcknowledgements.delete(key); + this.pendingAcknowledgements.set(key, acknowledgement); + } + } + } + + private async sendWithRetry( + request: CanonicalWriterRequest, + allowDuringClose = false, + ): Promise { for (let attempt = 1; attempt <= MAX_WORKER_ATTEMPTS; attempt++) { - if (this.stopping) throw new Error("Canonical writer closed"); + this.assertCanSend(allowDuringClose); try { await this.workerReady; + this.assertCanSend(allowDuringClose); return await this.sendRaw(request); } catch (error) { if (!(error instanceof CanonicalWriterWorkerLost) || attempt === MAX_WORKER_ATTEMPTS) throw error; - try { - await this.restartWorker(); - } catch (restartError) { - if (!(restartError instanceof CanonicalWriterWorkerLost)) throw restartError; - } + await this.restartAfterLoss(allowDuringClose); } } throw new Error("Canonical writer retry limit reached"); } - private async restartWorker(): Promise { - if (this.stopping) throw new Error("Canonical writer closed"); + private assertCanSend(allowDuringClose: boolean): void { + if (this.stopping || (this.closing && !allowDuringClose)) throw new Error("Canonical writer closed"); + } + + private async restartAfterLoss(allowDuringClose: boolean): Promise { + try { + await this.restartWorker(allowDuringClose); + } catch (error) { + if (!(error instanceof CanonicalWriterWorkerLost)) throw error; + } + } + + private async restartWorker(allowDuringClose: boolean): Promise { + if (this.stopping || (this.closing && !allowDuringClose)) throw new Error("Canonical writer closed"); if (!this.restart) { this.restart = (async () => { if (!this.worker) this.workerReady = this.startWorker(); @@ -211,3 +291,7 @@ function defaultCreateWorker(): Worker { : "./canonical-agent-writer.worker.ts"; return new Worker(new URL(workerFile, import.meta.url), { type: "module" }); } + +function acknowledgementKey(executionId: string, operationId: string): string { + return JSON.stringify([executionId, operationId]); +} From aea91851041e64a2654f86a0524c41470d2c9b9e Mon Sep 17 00:00:00 2001 From: "(CJ) Chukwudi Nwobodo" <142016413+chuks-qua@users.noreply.github.com> Date: Thu, 24 Sep 2026 20:02:11 +0100 Subject: [PATCH 020/136] feat(server): checkpoint parent narration through writer receipts --- .../agents/composition/register-agents.ts | 8 + .../agent-service-narrative-persist.test.ts | 156 ++++++++++++--- .../__tests__/agent-service-test-harness.ts | 31 +++ ...ent-narrative-recovery-coordinator.test.ts | 90 ++++++--- .../parent-narrative-recovery-coordinator.ts | 131 ++++++++---- .../turns/provider-turn-event-application.ts | 187 +++++++++++++----- 6 files changed, 466 insertions(+), 137 deletions(-) diff --git a/apps/server/src/features/agents/composition/register-agents.ts b/apps/server/src/features/agents/composition/register-agents.ts index 1b7dca5e8..dece995d3 100644 --- a/apps/server/src/features/agents/composition/register-agents.ts +++ b/apps/server/src/features/agents/composition/register-agents.ts @@ -75,6 +75,11 @@ import { ThreadCreationCoordinator } from "../turns/thread-creation-coordinator. import { ThreadStartupService } from "../../thread-startup/thread-startup-service.js"; import { ProviderSessionCursorPersistence } from "../turns/provider-session-cursor-persistence.js"; import { ProviderTurnEventApplication } from "../turns/provider-turn-event-application.js"; +import { CanonicalAgentWriterClient } from "../canonical/canonical-agent-writer-client.js"; +import { + PARENT_NARRATIVE_RECOVERY_WRITER, + type ParentNarrativeRecoveryWriter, +} from "../turns/parent-narrative-recovery-coordinator.js"; import { TURN_RUNTIME_EVENT_CONTROL, type TurnRuntimeEventControl, @@ -242,6 +247,9 @@ export function registerAgentServices(container: DependencyContainer): void { container.register(TURN_RUNTIME_EVENT_CONTROL, { useFactory: (c) => c.resolve(TurnRuntimeController), }); + container.register(PARENT_NARRATIVE_RECOVERY_WRITER, { + useFactory: (c) => c.resolve(CanonicalAgentWriterClient), + }); container.register( ProviderTurnEventApplication, { useClass: ProviderTurnEventApplication }, diff --git a/apps/server/src/features/agents/orchestration/__tests__/agent-service-narrative-persist.test.ts b/apps/server/src/features/agents/orchestration/__tests__/agent-service-narrative-persist.test.ts index 1557663c3..0179e8e25 100644 --- a/apps/server/src/features/agents/orchestration/__tests__/agent-service-narrative-persist.test.ts +++ b/apps/server/src/features/agents/orchestration/__tests__/agent-service-narrative-persist.test.ts @@ -79,6 +79,7 @@ function providerRuntimeEventForNarrativeTest(eventName: string, event: unknown) import type { PlanQuestionAnswersRepo } from "../../planning/persistence/plan-question-answers-repo.js"; import { CodexCollaborationEventAdapter } from "../../collaboration/adapters/codex-collaboration-event-adapter.js"; import { ProviderEventIngress } from "../../../providers/composition/provider-event-ingress.js"; +import type { ParentNarrativeRecoveryWriter } from "../../turns/parent-narrative-recovery-coordinator.js"; vi.mock("../../../../application/transport/push.js", () => ({ broadcast: vi.fn() })); vi.mock("fs", async (importOriginal) => { @@ -164,6 +165,7 @@ function build(options: { messageRepo?: MessageRepo; parentAssistantTextCheckpoints?: ParentAssistantTextCheckpointService; onProviderEvent?: (event: AgentEvent) => void; + narrativeWriter?: ParentNarrativeRecoveryWriter; } = {}): Built { const thread = makeThread(); const providerEmitter = Object.assign(new NodeEvents.EventEmitter(), { @@ -316,6 +318,10 @@ function build(options: { undefined, undefined, new TaskPersistenceService(taskRepo, narrativeStore), + undefined, + undefined, + undefined, + options.narrativeWriter, ); startAgentServiceIngressForTest(service, options.onProviderEvent); // Provider adapters always stamp turn-scoped events with the active execution @@ -358,11 +364,105 @@ function build(options: { }; } +function buildCommittedNarrativeTurn( + writerFor: (sink: CanonicalAgentEventSink) => ParentNarrativeRecoveryWriter, + onProviderEvent: (event: AgentEvent) => void, +): Built { + const db = openMemoryDatabase(); + const now = "2026-08-24T10:00:00.000Z"; + db.prepare( + "INSERT INTO workspaces (id, name, path, created_at, updated_at) VALUES (?, ?, ?, ?, ?)", + ).run("ws-1", "Workspace", "/workspace", now, now); + db.prepare( + "INSERT INTO threads (id, workspace_id, title, branch, provider, status, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", + ).run(THREAD_ID, "ws-1", "Parent", "main", "claude", "active", now, now); + const canonicalSink = new CanonicalAgentEventSink(db, vi.fn()); + const built = build({ db, canonicalSink, onProviderEvent, narrativeWriter: writerFor(canonicalSink) }); + canonicalSink.startParentTurn({ + thread: { id: THREAD_ID, workspaceId: "ws-1", providerId: "claude", createdAt: now }, + turnId: "turn-async-narrative", + executionId: narrativeExecutionId(built.service), + permissionMode: "supervised", + providerIdentities: [], + projectUserMessage: () => new SqliteMessageRepo(db).create(THREAD_ID, "user", "start", 1), + }); + return built; +} + describe("AgentService narrative persistence", () => { beforeEach(() => { vi.clearAllMocks(); }); + it("waits for a narrative writer acknowledgement before publishing and finalizing Stop", async () => { + let releaseWrite!: () => void; + const writeGate = new Promise((resolve) => { releaseWrite = resolve; }); + const published: AgentEvent[] = []; + const { db, service, providerEmitter, canonicalSink } = buildCommittedNarrativeTurn((sink) => ({ + async recordParentNarrativeRecovery(_operationId, input) { + await writeGate; + return { recorded: sink.recordParentNarrativeRecovery(input) }; + }, + async classifyParentNarrativeRecovery() { throw new Error("Unexpected classification"); }, + async acknowledgeOperation() {}, + }), (event) => published.push(event)); + const finalize = vi.spyOn(finalizerForAgentServiceTest(service), "finalize"); + try { + providerEmitter.emit("event", { + type: AgentEventType.TextDelta, + threadId: THREAD_ID, + turnExecutionId: narrativeExecutionId(service), + isFinalResponse: false, + delta: "awaited thought", + }); + await waitForAgentServiceIngressForTest(service, THREAD_ID); + await service.stopSession(THREAD_ID); + + expect(published).toEqual([]); + expect(finalize).not.toHaveBeenCalled(); + releaseWrite(); + await vi.waitFor(() => expect(finalize).toHaveBeenCalledOnce()); + expect(published.filter((event) => event.type === AgentEventType.TextDelta)).toHaveLength(1); + expect(canonicalSink.loadParentNarrativeRecovery("turn-async-narrative")).toHaveLength(1); + } finally { + db.close(); + } + }); + + it("does not finalize a stopped execution after its narrative write fails", async () => { + let failWrite!: (error: Error) => void; + const writeGate = new Promise((_resolve, reject) => { failWrite = reject; }); + const published: AgentEvent[] = []; + const { db, service, providerEmitter } = buildCommittedNarrativeTurn(() => ({ + async recordParentNarrativeRecovery() { + await writeGate; + return { recorded: true }; + }, + async classifyParentNarrativeRecovery() { throw new Error("Unexpected classification"); }, + async acknowledgeOperation() {}, + }), (event) => published.push(event)); + const finalize = vi.spyOn(finalizerForAgentServiceTest(service), "finalize"); + try { + providerEmitter.emit("event", { + type: AgentEventType.TextDelta, + threadId: THREAD_ID, + turnExecutionId: narrativeExecutionId(service), + isFinalResponse: false, + delta: "uncommitted thought", + }); + await waitForAgentServiceIngressForTest(service, THREAD_ID); + failWrite(new Error("injected writer failure")); + await vi.waitFor(() => expect(providerEmitter.stopSession).toHaveBeenCalled()); + await service.stopSession(THREAD_ID); + await Promise.resolve(); + + expect(published).toEqual([]); + expect(finalize).not.toHaveBeenCalled(); + } finally { + db.close(); + } + }); + it("moves unclassified text to narration and clears its assistant checkpoint at a false boundary", async () => { const db = openMemoryDatabase(); const now = "2026-08-24T10:00:00.000Z"; @@ -418,15 +518,17 @@ describe("AgentService narrative persistence", () => { }); await waitForAgentServiceIngressForTest(service, THREAD_ID); - expect(published - .filter((event) => event.type === AgentEventType.TextDelta) - .map((event) => event.delta)) - .toEqual(["durable ", "text"]); - expect(db.prepare(` - SELECT first_sequence, last_sequence, text - FROM parent_assistant_text_checkpoint_chunks - WHERE execution_id = ? - `).all(executionId)).toEqual([]); + await vi.waitFor(() => { + expect(published + .filter((event) => event.type === AgentEventType.TextDelta) + .map((event) => event.delta)) + .toEqual(["durable ", "text"]); + expect(db.prepare(` + SELECT first_sequence, last_sequence, text + FROM parent_assistant_text_checkpoint_chunks + WHERE execution_id = ? + `).all(executionId)).toEqual([]); + }); } finally { vi.useRealTimers(); db.close(); @@ -481,12 +583,14 @@ describe("AgentService narrative persistence", () => { } await waitForAgentServiceIngressForTest(service, THREAD_ID); - expect(published - .filter((event) => event.type === AgentEventType.TextDelta) - .map((event) => event.delta.length)) - .toEqual([...firstSegment, ...secondSegment].map((delta) => delta.length)); - expect(published.filter((event) => event.type === AgentEventType.AssistantMessageBoundary)) - .toHaveLength(2); + await vi.waitFor(() => { + expect(published + .filter((event) => event.type === AgentEventType.TextDelta) + .map((event) => event.delta.length)) + .toEqual([...firstSegment, ...secondSegment].map((delta) => delta.length)); + expect(published.filter((event) => event.type === AgentEventType.AssistantMessageBoundary)) + .toHaveLength(2); + }); } finally { db.close(); } @@ -892,12 +996,12 @@ describe("AgentService narrative persistence", () => { expect(published).toEqual([]); continueAgentTurnWithoutSavingForTest(service, executionId); - await Promise.resolve(); - - expect(published.map((event) => event.type)).toEqual([ - AgentEventType.TextDelta, - AgentEventType.AssistantMessageBoundary, - ]); + await vi.waitFor(() => { + expect(published.map((event) => event.type)).toEqual([ + AgentEventType.TextDelta, + AgentEventType.AssistantMessageBoundary, + ]); + }); expect(narrativeStore.recoverySnapshot(THREAD_ID)).toEqual([ expect.objectContaining({ kind: "narrationSegment", @@ -948,7 +1052,7 @@ describe("AgentService narrative persistence", () => { }); await waitForAgentServiceIngressForTest(service, THREAD_ID); vi.advanceTimersByTime(250); - expect(published).toHaveLength(1); + await vi.waitFor(() => expect(published).toHaveLength(1)); published.length = 0; db.exec(` CREATE TRIGGER reject_narration_recovery @@ -1030,9 +1134,11 @@ describe("AgentService narrative persistence", () => { }); await waitForAgentServiceIngressForTest(service, THREAD_ID); - expect(observed.at(-1)).toMatchObject({ - type: AgentEventType.AssistantMessageBoundary, - persisted: expect.stringContaining("I will inspect the child."), + await vi.waitFor(() => { + expect(observed.at(-1)).toMatchObject({ + type: AgentEventType.AssistantMessageBoundary, + persisted: expect.stringContaining("I will inspect the child."), + }); }); expect(observed.at(-1)?.persisted).not.toContain("turnExecutionId"); db.close(); diff --git a/apps/server/src/features/agents/orchestration/__tests__/agent-service-test-harness.ts b/apps/server/src/features/agents/orchestration/__tests__/agent-service-test-harness.ts index b32a5cd6f..91a05a6cd 100644 --- a/apps/server/src/features/agents/orchestration/__tests__/agent-service-test-harness.ts +++ b/apps/server/src/features/agents/orchestration/__tests__/agent-service-test-harness.ts @@ -58,6 +58,10 @@ import { TurnFeatureEffects } from "../../turns/turn-feature-effects.js"; import { AgentEventPublicationRuntimePort, AgentTurnContinuationPort } from "../agent-runtime-internal-ports.js"; import { TurnRuntimeController } from "../turn-runtime-controller.js"; import { ProviderTurnEventApplication } from "../../turns/provider-turn-event-application.js"; +import { + PARENT_NARRATIVE_RECOVERY_WRITER, + type ParentNarrativeRecoveryWriter, +} from "../../turns/parent-narrative-recovery-coordinator.js"; import { TURN_RUNTIME_EVENT_CONTROL, type TurnRuntimeEventControl } from "../turn-runtime-event-control.js"; const testEventPublications = new WeakMap(); @@ -200,6 +204,7 @@ export function createAgentServiceForTest( threadBranching?: ThreadBranchingService, eventPublication?: AgentEventPublicationRegistry, threadStartups?: ThreadStartupService, + narrativeWriterOverride?: ParentNarrativeRecoveryWriter, ): AgentService { if (!parentDurability) throw new Error("Parent turn durability is required by the test harness"); const turnDiffs = new TurnDiffService(new TurnDiffRepo(db)); @@ -318,6 +323,9 @@ export function createAgentServiceForTest( testContainer.registerInstance(AgentReliabilityPort, reliability); testContainer.registerInstance(AgentEventPublicationRegistry, publication); testContainer.registerInstance(PARENT_TURN_DURABILITY, parentDurability); + testContainer.registerInstance(PARENT_NARRATIVE_RECOVERY_WRITER, narrativeWriterForTest( + narrativeWriterOverride, db, parentDurability, parentAssistantTextCheckpoints, + )); testContainer.registerInstance(NarrativeStore, narrativeStore); testContainer.registerInstance(ParentAssistantTextCheckpointService, parentAssistantTextCheckpoints); testContainer.registerInstance("Database", db); @@ -342,3 +350,26 @@ export function createAgentServiceForTest( testGoalLifecycles.set(service, resolvedGoals); return service; } + +function narrativeWriterForTest( + override: ParentNarrativeRecoveryWriter | undefined, + db: Database, + parentDurability: ParentTurnDurability, + parentAssistantTextCheckpoints: ParentAssistantTextCheckpointService, +): ParentNarrativeRecoveryWriter { + return override ?? { + async recordParentNarrativeRecovery(_operationId, input) { + return { recorded: parentDurability.recordParentNarrativeRecovery(input) }; + }, + async classifyParentNarrativeRecovery(_operationId, input) { + return db.transaction(() => { + const recorded = parentDurability.recordParentNarrativeRecovery(input); + if (!recorded) throw new Error("Canonical parent turn was not found"); + const reset = parentAssistantTextCheckpoints.resetInTransaction(input.executionId); + if (!reset) throw new Error("Provisional assistant text checkpoint was not reset"); + return { recorded, reset }; + })(); + }, + async acknowledgeOperation() {}, + }; +} diff --git a/apps/server/src/features/agents/turns/__tests__/parent-narrative-recovery-coordinator.test.ts b/apps/server/src/features/agents/turns/__tests__/parent-narrative-recovery-coordinator.test.ts index 2b05aadc5..cc3180f20 100644 --- a/apps/server/src/features/agents/turns/__tests__/parent-narrative-recovery-coordinator.test.ts +++ b/apps/server/src/features/agents/turns/__tests__/parent-narrative-recovery-coordinator.test.ts @@ -4,7 +4,10 @@ import { type AgentEvent, type ParentNarrativeRecoveryItem, } from "@mcode/contracts"; -import { ParentNarrativeRecoveryCoordinator } from "../parent-narrative-recovery-coordinator.js"; +import { + ParentNarrativeRecoveryCoordinator, + type ParentNarrativeRecoveryWriter, +} from "../parent-narrative-recovery-coordinator.js"; import type { NarrativeStore } from "../../conversation/narrative/narrative-store.js"; import type { ParentTurnDurability } from "../parent-turn-durability.js"; @@ -23,41 +26,84 @@ const recoveryItems = [ }, }, ] satisfies ParentNarrativeRecoveryItem[]; +const event: AgentEvent = { + type: AgentEventType.TextDelta, + threadId: THREAD_ID, + turnExecutionId: EXECUTION_ID, + delta: "Reasoning before the answer.", +}; + +function coordinatorFor(writer: ParentNarrativeRecoveryWriter): ParentNarrativeRecoveryCoordinator { + const narrativeStore = { + recoverySnapshot: vi.fn(() => recoveryItems), + } as unknown as NarrativeStore; + const durability = { + loadTurnByExecution: vi.fn(() => ({ id: "turn-1" })), + } as unknown as ParentTurnDurability; + return new ParentNarrativeRecoveryCoordinator(writer, narrativeStore, durability); +} describe("ParentNarrativeRecoveryCoordinator", () => { - it("retries an uncommitted parent recovery snapshot before deduplicating it", () => { + it("retries an uncommitted recovery snapshot with the same operation identity", async () => { const recordParentNarrativeRecovery = vi.fn() - .mockReturnValueOnce(false) - .mockReturnValue(true); - const durability = { - loadTurnByExecution: vi.fn(() => ({ id: "turn-1" })), + .mockResolvedValueOnce({ recorded: false }) + .mockResolvedValue({ recorded: true }); + const writer: ParentNarrativeRecoveryWriter = { recordParentNarrativeRecovery, - } as unknown as ParentTurnDurability; - const narrativeStore = { - recoverySnapshot: vi.fn(() => recoveryItems), - } as unknown as NarrativeStore; - const coordinator = new ParentNarrativeRecoveryCoordinator(durability, narrativeStore); - const event: AgentEvent = { - type: AgentEventType.TextDelta, - threadId: THREAD_ID, - turnExecutionId: EXECUTION_ID, - delta: "Reasoning before the answer.", + classifyParentNarrativeRecovery: vi.fn(), + acknowledgeOperation: vi.fn(async () => undefined), }; + const coordinator = coordinatorFor(writer); const expectedCommit = { executionId: EXECUTION_ID, items: recoveryItems, discardedItemIds: [], }; - expect(() => coordinator.checkpoint(event)).toThrow( + await expect(coordinator.checkpoint(event)).rejects.toThrow( `Canonical parent turn was not found: ${EXECUTION_ID}`, ); - - coordinator.checkpoint(event); - coordinator.checkpoint(event); + await coordinator.checkpoint(event); + await coordinator.checkpoint(event); expect(recordParentNarrativeRecovery).toHaveBeenCalledTimes(2); - expect(recordParentNarrativeRecovery).toHaveBeenNthCalledWith(1, expectedCommit); - expect(recordParentNarrativeRecovery).toHaveBeenNthCalledWith(2, expectedCommit); + const operationId = recordParentNarrativeRecovery.mock.calls[0]?.[0]; + expect(operationId).toMatch(/^narrative:/); + expect(recordParentNarrativeRecovery).toHaveBeenNthCalledWith(1, operationId, expectedCommit); + expect(recordParentNarrativeRecovery).toHaveBeenNthCalledWith(2, operationId, expectedCommit); + expect(writer.acknowledgeOperation).not.toHaveBeenCalled(); + coordinator.acknowledgeHandled(event); + await vi.waitFor(() => expect(writer.acknowledgeOperation).toHaveBeenCalledWith(EXECUTION_ID, operationId)); + }); + + it("retries the atomic narration classification without changing its input", async () => { + const classifyParentNarrativeRecovery = vi.fn() + .mockRejectedValueOnce(new Error("writer unavailable")) + .mockResolvedValue({ recorded: true, reset: true }); + const writer: ParentNarrativeRecoveryWriter = { + recordParentNarrativeRecovery: vi.fn(), + classifyParentNarrativeRecovery, + acknowledgeOperation: vi.fn(async () => undefined), + }; + const coordinator = coordinatorFor(writer); + const boundary: AgentEvent = { + type: AgentEventType.AssistantMessageBoundary, + threadId: THREAD_ID, + turnExecutionId: EXECUTION_ID, + isFinalResponse: false, + }; + + await expect(coordinator.classify(boundary, recoveryItems)).rejects.toThrow("writer unavailable"); + await coordinator.classify(boundary, recoveryItems); + await coordinator.checkpoint(boundary); + + expect(classifyParentNarrativeRecovery).toHaveBeenCalledTimes(2); + expect(classifyParentNarrativeRecovery.mock.calls[1]).toEqual(classifyParentNarrativeRecovery.mock.calls[0]); + expect(writer.acknowledgeOperation).not.toHaveBeenCalled(); + coordinator.acknowledgeHandled(boundary); + await vi.waitFor(() => expect(writer.acknowledgeOperation).toHaveBeenCalledWith( + EXECUTION_ID, + classifyParentNarrativeRecovery.mock.calls[0]?.[0], + )); }); }); diff --git a/apps/server/src/features/agents/turns/parent-narrative-recovery-coordinator.ts b/apps/server/src/features/agents/turns/parent-narrative-recovery-coordinator.ts index 2b0c2b6bf..e08a4d9c8 100644 --- a/apps/server/src/features/agents/turns/parent-narrative-recovery-coordinator.ts +++ b/apps/server/src/features/agents/turns/parent-narrative-recovery-coordinator.ts @@ -1,85 +1,137 @@ +import * as NodeCrypto from "node:crypto"; +import { logger } from "@mcode/shared"; import { AgentEventType, type AgentEvent, type ParentNarrativeRecoveryItem, } from "@mcode/contracts"; -import type { ParentTurnDurability } from "./parent-turn-durability.js"; import type { NarrativeStore } from "../conversation/narrative/narrative-store.js"; +import type { CanonicalAgentWriterClient } from "../canonical/canonical-agent-writer-client.js"; +import type { ParentNarrativeRecoveryCommit } from "./parent-turn-durability.js"; +import type { ParentTurnDurability } from "./parent-turn-durability.js"; import { serverWorkTrace } from "../diagnostics/server-work-trace.js"; +/** The single acknowledged writer used for recovery and text classification. */ +export const PARENT_NARRATIVE_RECOVERY_WRITER = Symbol("ParentNarrativeRecoveryWriter"); +export type ParentNarrativeRecoveryWriter = Pick; + interface ParentNarrativeRecoveryCheckpoint { - persist(): void; + operationId: string; + input: ParentNarrativeRecoveryCommit; + committed: boolean; confirm(): void; } /** Commits changed structured narrative records before AgentService publishes their source event. */ export class ParentNarrativeRecoveryCoordinator { private readonly fingerprintsByExecution = new Map>(); + private readonly preparedByEvent = new WeakMap(); + private readonly existingExecutions = new Set(); constructor( - private readonly canonicalSink: ParentTurnDurability, + private readonly writer: ParentNarrativeRecoveryWriter, private readonly narrativeStore: NarrativeStore, + private readonly canonicalSink: ParentTurnDurability, ) {} /** Commit only the semantic records changed by this accepted provider event. */ - checkpoint(event: AgentEvent): void { - if (!serverWorkTrace) { - const checkpoint = this.prepareCheckpoint(event); - if (!checkpoint) return; - checkpoint.persist(); - checkpoint.confirm(); - return; - } - const checkpoint = serverWorkTrace.measure("narrative-prepare", event.threadId, - event.turnExecutionId, () => this.prepareCheckpoint(event)); - if (!checkpoint) return; - serverWorkTrace.measure("narrative-persist", event.threadId, - event.turnExecutionId, () => checkpoint.persist()); - serverWorkTrace.measure("narrative-confirm", event.threadId, - event.turnExecutionId, () => checkpoint.confirm()); + checkpoint(event: AgentEvent): Promise | void { + const checkpoint = this.prepareCheckpoint(event); + if (!checkpoint || checkpoint.committed) return; + const persist = () => this.writer.recordParentNarrativeRecovery(checkpoint.operationId, checkpoint.input); + const pending = serverWorkTrace + ? serverWorkTrace.measure("narrative-persist", event.threadId, event.turnExecutionId, persist) + : persist(); + return pending.then((receipt) => { + if (!receipt.recorded) throw new Error(`Canonical parent turn was not found: ${event.turnExecutionId}`); + if (serverWorkTrace) { + serverWorkTrace.measure("narrative-confirm", event.threadId, event.turnExecutionId, checkpoint.confirm); + } else checkpoint.confirm(); + checkpoint.committed = true; + }); + } + + /** Commit staged narration and reset provisional assistant text in one writer transaction. */ + async classify( + event: AgentEvent, + snapshot: readonly ParentNarrativeRecoveryItem[], + ): Promise { + const checkpoint = this.prepareCheckpoint(event, snapshot, true); + if (!checkpoint) throw new Error("Narrative classification requires a turn execution"); + await this.writer.classifyParentNarrativeRecovery(checkpoint.operationId, checkpoint.input); + checkpoint.confirm(); + checkpoint.committed = true; } - /** Prepare a recovery commit whose dedupe state advances only after its transaction commits. */ - prepareCheckpoint( + /** Release a receipt only after its source event was actually published or its recovery journal retired. */ + acknowledgeHandled(event: AgentEvent): void { + const checkpoint = this.preparedByEvent.get(event as object); + if (!checkpoint?.committed) return; + this.preparedByEvent.delete(event as object); + const { executionId } = checkpoint.input; + void this.writer.acknowledgeOperation(executionId, checkpoint.operationId).catch(() => { + logger.warn("Canonical narrative receipt acknowledgement failed", { + executionId, + operationId: checkpoint.operationId, + }); + }); + } + + private prepareCheckpoint( event: AgentEvent, snapshot?: readonly ParentNarrativeRecoveryItem[], + force = false, ): ParentNarrativeRecoveryCheckpoint | null { if (!this.requiresStructuredRecovery(event) || !event.turnExecutionId) return null; - if (!this.canonicalSink.loadTurnByExecution(event.turnExecutionId)) return null; - const snapshots = snapshot ?? this.narrativeStore.recoverySnapshot(event.threadId); + if (!force && !this.existingExecutions.has(event.turnExecutionId)) { + if (!this.canonicalSink.loadTurnByExecution(event.turnExecutionId)) return null; + this.existingExecutions.add(event.turnExecutionId); + } + const existing = this.preparedByEvent.get(event as object); + if (existing) return existing; + const executionId = event.turnExecutionId; + const prepare = () => this.buildCheckpoint(event, executionId, snapshot, force); + return serverWorkTrace + ? serverWorkTrace.measure("narrative-prepare", event.threadId, event.turnExecutionId, prepare) + : prepare(); + } - const fingerprints = this.fingerprintsByExecution.get(event.turnExecutionId) ?? new Map(); + private buildCheckpoint( + event: AgentEvent, + executionId: string, + snapshot: readonly ParentNarrativeRecoveryItem[] | undefined, + force: boolean, + ): ParentNarrativeRecoveryCheckpoint | null { + const snapshots = snapshot ?? this.narrativeStore.recoverySnapshot(event.threadId); + const fingerprints = this.fingerprintsByExecution.get(executionId) ?? new Map(); const nextFingerprints = new Map(); const currentKeys = new Set(snapshots.map((item) => `${item.kind}:${item.record.id}`)); const changed = snapshots.filter((item) => { const key = `${item.kind}:${item.record.id}`; const fingerprint = JSON.stringify(item); nextFingerprints.set(key, fingerprint); - if (fingerprints.get(key) === fingerprint) return false; - return true; + return fingerprints.get(key) !== fingerprint; }); const discardedItemIds = [...fingerprints.keys()] .filter((key) => !currentKeys.has(key)) .map((key) => this.canonicalItemId(key)); - if (changed.length === 0 && discardedItemIds.length === 0) return null; - return { - persist: () => { - const committed = this.canonicalSink.recordParentNarrativeRecovery({ - executionId: event.turnExecutionId!, - items: changed, - discardedItemIds, - }); - if (!committed) { - throw new Error(`Canonical parent turn was not found: ${event.turnExecutionId}`); - } - }, - confirm: () => this.fingerprintsByExecution.set(event.turnExecutionId!, nextFingerprints), + if (!force && changed.length === 0 && discardedItemIds.length === 0) return null; + const prepared = { + operationId: `narrative:${NodeCrypto.randomUUID()}`, + input: { executionId, items: changed, discardedItemIds }, + committed: false, + confirm: () => this.fingerprintsByExecution.set(executionId, nextFingerprints), }; + this.preparedByEvent.set(event as object, prepared); + return prepared; } /** Forget volatile dedupe state after a terminal turn releases its buffers. */ clear(executionId: string | undefined): void { - if (executionId) this.fingerprintsByExecution.delete(executionId); + if (!executionId) return; + this.fingerprintsByExecution.delete(executionId); + this.existingExecutions.delete(executionId); } private requiresStructuredRecovery(event: AgentEvent): boolean { @@ -98,5 +150,4 @@ export class ParentNarrativeRecoveryCoordinator { if (!kind || !id) throw new Error(`Invalid narrative recovery identity: ${key}`); return kind === "toolCall" ? `toolCall:${id}` : `${kind}:${id}`; } - } diff --git a/apps/server/src/features/agents/turns/provider-turn-event-application.ts b/apps/server/src/features/agents/turns/provider-turn-event-application.ts index 844120ca8..05f33c806 100644 --- a/apps/server/src/features/agents/turns/provider-turn-event-application.ts +++ b/apps/server/src/features/agents/turns/provider-turn-event-application.ts @@ -1,4 +1,3 @@ -import type { Database } from "bun:sqlite"; import { inject, injectable } from "tsyringe"; import { AgentEventType, @@ -17,7 +16,11 @@ import { TURN_FINALIZER, TurnFinalizer } from "./turn-finalizer.js"; import { TURN_FILE_EFFECTS, TurnFileEffects } from "./turn-file-effects.js"; import { ParentAssistantTextCheckpointService } from "./parent-assistant-text-checkpoint-service.js"; import { ParentAssistantTextCoordinator } from "./parent-assistant-text-coordinator.js"; -import { ParentNarrativeRecoveryCoordinator } from "./parent-narrative-recovery-coordinator.js"; +import { + PARENT_NARRATIVE_RECOVERY_WRITER, + ParentNarrativeRecoveryCoordinator, + type ParentNarrativeRecoveryWriter, +} from "./parent-narrative-recovery-coordinator.js"; import { PARENT_TURN_DURABILITY, type ParentTurnDurability } from "./parent-turn-durability.js"; import { TurnConversationProjectionService } from "./turn-conversation-projection-service.js"; import { PostTerminalHookCompletionEffect } from "./post-terminal-hook-completion-effect.js"; @@ -55,6 +58,11 @@ export class ProviderTurnEventApplication implements TurnEventApplication { private readonly lastContextByThread = new Map(); private readonly lastContextWindowByThread = new Map(); private readonly terminalProjectionByThread = new Map>(); + private readonly narrativeApplicationsByThread = new Map; + }>(); + private readonly failedNarrativeExecutionByThread = new Map(); constructor( @inject(TURN_FINALIZER) private readonly finalizer: TurnFinalizer, @@ -72,7 +80,7 @@ export class ProviderTurnEventApplication implements TurnEventApplication { @inject(NarrativeStore) private readonly narrative: NarrativeStore, @inject(ParentAssistantTextCheckpointService) checkpoints: ParentAssistantTextCheckpointService, @inject(TURN_FEATURE_EFFECTS) private readonly featureEffects: TurnFeatureEffects, - @inject("Database") private readonly db: Database, + @inject(PARENT_NARRATIVE_RECOVERY_WRITER) narrativeWriter: ParentNarrativeRecoveryWriter, @inject(TURN_RUNTIME_EVENT_CONTROL) private readonly runtime: TurnRuntimeEventControl, @inject(AgentEventPublicationRegistry) private readonly publication: AgentEventPublicationRegistry, @@ -87,7 +95,7 @@ export class ProviderTurnEventApplication implements TurnEventApplication { } }, ); - this.parentNarrativeRecovery = new ParentNarrativeRecoveryCoordinator(parentDurability, narrative); + this.parentNarrativeRecovery = new ParentNarrativeRecoveryCoordinator(narrativeWriter, narrative, parentDurability); this.browserNarrativeEventSanitizer = new BrowserNarrativeEventSanitizer( (threadId, toolCallId) => this.narrative.getBufferedToolCalls(threadId) .find((toolCall) => toolCall.toolCallId === toolCallId) @@ -106,7 +114,7 @@ export class ProviderTurnEventApplication implements TurnEventApplication { } /** Apply one queue-admitted provider event and publish it only after its durable prerequisites complete. */ - apply(input: ProviderEventIngressEvent, event: AgentEvent, publish: boolean): boolean { + apply(input: ProviderEventIngressEvent, event: AgentEvent, publish: boolean): boolean | Promise { const queued = this.queueVisibleAssistantText(input, event, publish); if (queued !== undefined) return queued; return this.applyPreparedEvent(input, event, publish); @@ -147,13 +155,25 @@ export class ProviderTurnEventApplication implements TurnEventApplication { if (this.terminalFinalizedThreads.has(command.threadId)) return null; this.terminalFinalizedThreads.add(command.threadId); const executionId = this.runtime.snapshot(command.threadId)?.turnExecutionId; - const finalization = this.fileEffects.finalize( + const finalizeEffects = () => this.fileEffects.finalize( command.threadId, command.outcome, executionId ?? undefined, command.source, ); - void finalization.finally(() => this.clearFinalizedEventState(command.threadId, executionId)); + const narrative = this.narrativeApplicationsByThread.get(command.threadId); + const failedExecutionId = this.failedNarrativeExecutionByThread.get(command.threadId); + const failed = this.failedNarrativeExecutionByThread.has(command.threadId) + && (failedExecutionId === undefined || failedExecutionId === executionId); + const finalization = failed + ? Promise.resolve(false) + : narrative && narrative.executionId === executionId + ? narrative.pending.then((ready) => ready ? finalizeEffects() : false) + : finalizeEffects(); + void finalization.then( + () => this.clearFinalizedEventState(command.threadId, executionId), + () => this.clearFinalizedEventState(command.threadId, executionId), + ); return finalization; } @@ -249,15 +269,55 @@ export class ProviderTurnEventApplication implements TurnEventApplication { event: AgentEvent, publish: boolean, textIsDurable = false, - ): boolean { + ): boolean | Promise { const terminal = isTerminal(event); const preparation = this.prepareEventForApplication(event, publish, textIsDurable, terminal); + if (preparation instanceof Promise) { + return preparation.then((ready) => ready === undefined + ? this.applyReadyEvent(input, event, publish, terminal) + : ready); + } if (preparation !== undefined) return preparation; + return this.applyReadyEvent(input, event, publish, terminal); + } + + private applyReadyEvent( + input: ProviderEventIngressEvent, + event: AgentEvent, + publish: boolean, + terminal: boolean, + ): boolean | Promise { this.recordDiagnostic(input, event); const accepted = this.applyEvent(input.providerId, event, publish); + if (accepted instanceof Promise) { + return accepted.then((result) => this.finishAppliedEvent(event, publish, terminal, result)); + } + return this.finishAppliedEvent(event, publish, terminal, accepted); + } + + private finishAppliedEvent( + event: AgentEvent, + publish: boolean, + terminal: boolean, + accepted: EventApplicationResult, + ): boolean | Promise { if (accepted === false) return true; if (terminal && accepted !== true) return false; - if (!this.checkpointNarrative(event, publish)) return false; + const durable = this.checkpointNarrative(event, publish); + if (durable instanceof Promise) { + return durable.then((ready) => this.publishAppliedEvent(event, publish, terminal, accepted, ready)); + } + return this.publishAppliedEvent(event, publish, terminal, accepted, durable); + } + + private publishAppliedEvent( + event: AgentEvent, + publish: boolean, + terminal: boolean, + accepted: EventApplicationResult, + durable: boolean, + ): boolean { + if (!durable) return false; if (publish && accepted === OWNED_LATE_HOOK_COMPLETION) return true; if (publish) this.publishAfterDurability(event, terminal); return true; @@ -271,16 +331,34 @@ export class ProviderTurnEventApplication implements TurnEventApplication { if (!publish || !this.publication.isBound() || event.type !== AgentEventType.TextDelta) return undefined; if (event.isFinalResponse === false || !event.turnExecutionId) return undefined; const queued = this.queueParentAssistantText(event, () => { - void this.applyPreparedEvent(input, event, true, true); + this.applyQueuedVisibleText(input, event); }); return queued === "blocked" ? false : queued; } - private applyEvent(providerId: ProviderId, event: AgentEvent, publish: boolean): EventApplicationResult { + private applyQueuedVisibleText(input: ProviderEventIngressEvent, event: AgentEvent): void { + const executionId = event.turnExecutionId; + if (!executionId) return; + const prior = this.narrativeApplicationsByThread.get(event.threadId)?.pending ?? Promise.resolve(true); + const pending = prior.then((ready) => ready ? this.applyPreparedEvent(input, event, true, true) : false) + .catch(() => this.failNarrativeCheckpoint(event)); + this.narrativeApplicationsByThread.set(event.threadId, { executionId, pending }); + void pending.then((ready) => { + if (ready && this.narrativeApplicationsByThread.get(event.threadId)?.pending === pending) { + this.narrativeApplicationsByThread.delete(event.threadId); + } + }); + } + + private applyEvent( + providerId: ProviderId, + event: AgentEvent, + publish: boolean, + ): EventApplicationResult | Promise { return this.applyNarrativeEvent(providerId, event) ?? this.applyLifecycleEvent(providerId, event, publish); } - private applyNarrativeEvent(providerId: ProviderId, event: AgentEvent): EventApplicationResult { + private applyNarrativeEvent(providerId: ProviderId, event: AgentEvent): EventApplicationResult | Promise { switch (event.type) { case AgentEventType.TextDelta: return this.applyTextDelta(event); case AgentEventType.GeneratedAttachment: return this.applyGeneratedAttachment(event); @@ -315,12 +393,14 @@ export class ProviderTurnEventApplication implements TurnEventApplication { publish: boolean, textIsDurable: boolean, terminal: boolean, - ): boolean | undefined { + ): boolean | Promise | undefined { if (!this.prepareTerminalText(event, publish, terminal)) return false; - if (!publish || textIsDurable || this.parentAssistantText.prepareSemanticBoundary(event.threadId)) { - return undefined; + if (!publish || textIsDurable) return undefined; + if (!this.parentAssistantText.prepareSemanticBoundary(event.threadId)) { + return this.parentAssistantText.hasThreadStoppedForStorageFailure(event.threadId); } - return this.parentAssistantText.hasThreadStoppedForStorageFailure(event.threadId); + const narrative = this.narrativeApplicationsByThread.get(event.threadId); + return narrative ? narrative.pending.then((ready) => ready ? undefined : false) : undefined; } private applyTextDelta(event: Extract): boolean { @@ -360,7 +440,7 @@ export class ProviderTurnEventApplication implements TurnEventApplication { return true; } - private applyAssistantMessageBoundary(event: Extract): boolean { + private applyAssistantMessageBoundary(event: Extract): boolean | Promise { if (event.isFinalResponse === true) { const finalText = this.narrative.takeOpenThought(event.threadId); if (finalText) this.finalizer.appendStreamingText(event.threadId, finalText); @@ -586,20 +666,29 @@ export class ProviderTurnEventApplication implements TurnEventApplication { return false; } - private checkpointNarrative(event: AgentEvent, publish: boolean): boolean { + private checkpointNarrative(event: AgentEvent, publish: boolean): boolean | Promise { if (!publish || this.isUnsavedNarrationBoundary(event)) return true; try { - if (serverWorkTrace) { - serverWorkTrace.measure("narrative-checkpoint", event.threadId, event.turnExecutionId, - () => this.parentNarrativeRecovery.checkpoint(event)); - } else this.parentNarrativeRecovery.checkpoint(event); - return true; + const checkpoint = () => this.parentNarrativeRecovery.checkpoint(event); + const pending = serverWorkTrace + ? serverWorkTrace.measure("narrative-checkpoint", event.threadId, event.turnExecutionId, checkpoint) + : checkpoint(); + if (!pending) return true; + return pending.then( + () => true, + () => this.failNarrativeCheckpoint(event), + ); } catch { - this.runtime.stopForEventApplicationFailure(event, "Parent narrative recovery checkpoint failed"); - return false; + return this.failNarrativeCheckpoint(event); } } + private failNarrativeCheckpoint(event: AgentEvent): false { + this.failedNarrativeExecutionByThread.set(event.threadId, event.turnExecutionId); + this.runtime.stopForEventApplicationFailure(event, "Parent narrative recovery checkpoint failed"); + return false; + } + private publishAfterDurability(event: AgentEvent, terminal: boolean): void { if (!terminal && !this.isLateHook(event)) { this.publish(event); @@ -621,6 +710,7 @@ export class ProviderTurnEventApplication implements TurnEventApplication { ? { ...event, outcome: "interrupted" } : event, ); + this.parentNarrativeRecovery.acknowledgeHandled(event); } /** Retain terminal durability from its scheduling point until every dependent publication observes it. */ @@ -702,7 +792,7 @@ export class ProviderTurnEventApplication implements TurnEventApplication { private classifyUnclassifiedAssistantTextAsNarration( event: Extract, - ): boolean { + ): boolean | Promise { const executionId = event.turnExecutionId; if (!executionId) return this.clearUnclassifiedAssistantText(event.threadId); const firstSequence = this.unclassifiedAssistantTextStartByExecution.get(executionId); @@ -720,30 +810,21 @@ export class ProviderTurnEventApplication implements TurnEventApplication { .map((chunk) => chunk.text) .join(""); const staged = this.narrative.stageNarrationSegment(event.threadId, text); - let confirm: (() => void) | undefined; - try { - this.db.transaction(() => { - const checkpoint = this.parentNarrativeRecovery.prepareCheckpoint( - event, - staged ? this.narrative.recoverySnapshotWithStagedNarration(event.threadId, staged) : undefined, - ); - checkpoint?.persist(); - if (!this.parentAssistantText.checkpoints.resetInTransaction(executionId)) { - throw new Error(`Unclassified assistant text checkpoint was not reset: ${executionId}`); - } - confirm = checkpoint?.confirm; - })(); - } catch { - this.runtime.stopForEventApplicationFailure(event, "Parent narrative recovery checkpoint failed"); - return false; - } - this.parentAssistantText.checkpoints.discardRecoveryJournal(executionId); - this.parentAssistantText.discard(executionId); - confirm?.(); - if (staged) this.narrative.applyStagedNarrationSegment(event.threadId, staged); - this.unclassifiedAssistantTextStartByExecution.delete(executionId); - this.finalizer.resetStreamingText(event.threadId); - return true; + const snapshot = staged + ? this.narrative.recoverySnapshotWithStagedNarration(event.threadId, staged) + : this.narrative.recoverySnapshot(event.threadId); + return this.parentNarrativeRecovery.classify(event, snapshot).then( + () => { + this.parentAssistantText.checkpoints.discardRecoveryJournal(executionId); + this.parentNarrativeRecovery.acknowledgeHandled(event); + this.parentAssistantText.discard(executionId); + if (staged) this.narrative.applyStagedNarrationSegment(event.threadId, staged); + this.unclassifiedAssistantTextStartByExecution.delete(executionId); + this.finalizer.resetStreamingText(event.threadId); + return true; + }, + () => this.failNarrativeCheckpoint(event), + ); } private clearUnclassifiedAssistantText(threadId: string): true { @@ -768,6 +849,12 @@ export class ProviderTurnEventApplication implements TurnEventApplication { } private clearFinalizedEventState(threadId: string, executionId: string | null | undefined): void { + if (this.failedNarrativeExecutionByThread.get(threadId) === executionId) { + this.failedNarrativeExecutionByThread.delete(threadId); + } + if (this.narrativeApplicationsByThread.get(threadId)?.executionId === executionId) { + this.narrativeApplicationsByThread.delete(threadId); + } if (executionId) { this.parentAssistantText.discard(executionId); this.unclassifiedAssistantTextStartByExecution.delete(executionId); From 9371eaa56a4bf497e9351ae792de281ffb1b6f6d Mon Sep 17 00:00:00 2001 From: "(CJ) Chukwudi Nwobodo" <142016413+chuks-qua@users.noreply.github.com> Date: Thu, 24 Sep 2026 20:03:14 +0100 Subject: [PATCH 021/136] fix(server): clear failed narrative guard after finalization --- .../features/agents/turns/provider-turn-event-application.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/apps/server/src/features/agents/turns/provider-turn-event-application.ts b/apps/server/src/features/agents/turns/provider-turn-event-application.ts index 05f33c806..a96548302 100644 --- a/apps/server/src/features/agents/turns/provider-turn-event-application.ts +++ b/apps/server/src/features/agents/turns/provider-turn-event-application.ts @@ -849,7 +849,9 @@ export class ProviderTurnEventApplication implements TurnEventApplication { } private clearFinalizedEventState(threadId: string, executionId: string | null | undefined): void { - if (this.failedNarrativeExecutionByThread.get(threadId) === executionId) { + const failedExecutionId = this.failedNarrativeExecutionByThread.get(threadId); + if (this.failedNarrativeExecutionByThread.has(threadId) + && (failedExecutionId === undefined || failedExecutionId === executionId)) { this.failedNarrativeExecutionByThread.delete(threadId); } if (this.narrativeApplicationsByThread.get(threadId)?.executionId === executionId) { From e410cb9387918de0ab2f4c46bea5b35580611160 Mon Sep 17 00:00:00 2001 From: "(CJ) Chukwudi Nwobodo" <142016413+chuks-qua@users.noreply.github.com> Date: Thu, 24 Sep 2026 20:01:56 +0100 Subject: [PATCH 022/136] fix(server): retain provider diagnostics after writer commits --- .../canonical/canonical-agent-boundary.ts | 5 ++ .../canonical-agent-diagnostics.test.ts | 28 +++++++++++ .../canonical-agent-diagnostics.ts | 47 ++++++++++++++++--- .../provider-composition-container.test.ts | 16 +++++++ .../__tests__/provider-host-ports.test.ts | 9 +++- .../composition/provider-host-ports.ts | 4 +- .../composition/register-providers.ts | 3 +- 7 files changed, 103 insertions(+), 9 deletions(-) diff --git a/apps/server/src/features/agents/canonical/canonical-agent-boundary.ts b/apps/server/src/features/agents/canonical/canonical-agent-boundary.ts index 7f67102d4..d3761df73 100644 --- a/apps/server/src/features/agents/canonical/canonical-agent-boundary.ts +++ b/apps/server/src/features/agents/canonical/canonical-agent-boundary.ts @@ -487,6 +487,11 @@ export class CanonicalAgentBoundary implements ParentTurnDurability, CodexCollab : this.eventStore.commit(this.toEventStoreCommitInput(input)); } + /** Retain diagnostics on the server after a provider writer acknowledges its durable events. */ + recordProviderCommitDiagnostics(events: readonly CanonicalAgentEventEnvelope[]): void { + this.recordCanonicalDiagnostics(events); + } + private commitInsideTransaction(input: CanonicalAgentCommitInput): CanonicalAgentCommitResult { return serverWorkTrace ? serverWorkTrace.measure("canonical-write", input.threadId, input.executionId, diff --git a/apps/server/src/features/agents/observability/__tests__/canonical-agent-diagnostics.test.ts b/apps/server/src/features/agents/observability/__tests__/canonical-agent-diagnostics.test.ts index 282854d8f..7928c580c 100644 --- a/apps/server/src/features/agents/observability/__tests__/canonical-agent-diagnostics.test.ts +++ b/apps/server/src/features/agents/observability/__tests__/canonical-agent-diagnostics.test.ts @@ -90,4 +90,32 @@ describe("CanonicalAgentDiagnostics", () => { confirmRaw: true, }).rawEvents).toEqual([]); }); + + it("records a replayed canonical event once while raw capture retains it", () => { + const diagnostics = new CanonicalAgentDiagnostics(); + diagnostics.startRawCapture({ turnId: "turn-1", consent: true, expiresInMs: 60_000 }); + const event = { eventId: "canonical-1", payload: { type: "item.recorded", content: "answer" } }; + const input = { turnId: "turn-1", executionId: "execution-1", source: "canonical" as const, event }; + diagnostics.record(input); + for (let index = 0; index < CANONICAL_DIAGNOSTIC_RING_CAPACITY; index += 1) { + diagnostics.record({ + turnId: "other-turn", + executionId: "other-execution", + source: "provider", + event: { type: "textDelta", delta: `other-${index}` }, + }); + } + diagnostics.record(input); + diagnostics.record({ + turnId: "turn-2", + executionId: "execution-2", + source: "canonical", + event, + }); + + const exported = diagnostics.exportTurn("turn-1", { includeRaw: true, confirmRaw: true }); + expect(exported.rawEvents).toEqual([event]); + expect(exported.truncation).toEqual({ droppedEntries: 1 }); + expect(diagnostics.exportTurn("turn-2").entries).toHaveLength(1); + }); }); diff --git a/apps/server/src/features/agents/observability/canonical-agent-diagnostics.ts b/apps/server/src/features/agents/observability/canonical-agent-diagnostics.ts index 8aca5e236..a22cb2760 100644 --- a/apps/server/src/features/agents/observability/canonical-agent-diagnostics.ts +++ b/apps/server/src/features/agents/observability/canonical-agent-diagnostics.ts @@ -66,6 +66,15 @@ function eventType(event: unknown): string { return typeof event.type === "string" ? event.type : "unknown"; } +function canonicalEventId(event: unknown): string | undefined { + if (!event || typeof event !== "object" || !("eventId" in event)) return undefined; + return typeof event.eventId === "string" ? event.eventId : undefined; +} + +function canonicalEventKey(turnId: string, eventId: string): string { + return JSON.stringify([turnId, eventId]); +} + interface ContentMeasureState { bytes: number; visited: number; @@ -162,6 +171,8 @@ function serializeRawEvent(event: unknown): { event: unknown; bytes: number } | /** Retains bounded redacted diagnostics and consent-scoped raw turn captures. */ export class CanonicalAgentDiagnostics { private readonly entries: CanonicalDiagnosticEntry[] = []; + private readonly canonicalEventIds = new Set(); + private readonly canonicalEventIdByEntry = new WeakMap(); private readonly droppedByTurn = new Map(); private readonly rawCaptures = new Map(); @@ -191,6 +202,8 @@ export class CanonicalAgentDiagnostics { /** Record one event without retaining content unless raw capture is active. */ record(input: CanonicalDiagnosticRecordInput): void { + const eventId = input.source === "canonical" ? canonicalEventId(input.event) : undefined; + if (eventId && this.hasRecordedCanonicalEvent(input.turnId, eventId)) return; const measuredContent = measureContent(input.event); const entry: CanonicalDiagnosticEntry = { turnId: input.turnId, @@ -203,13 +216,13 @@ export class CanonicalAgentDiagnostics { recordedAt: new Date(this.now()).toISOString(), }; this.entries.push(entry); - if (this.entries.length > CANONICAL_DIAGNOSTIC_RING_CAPACITY) { - const [dropped] = this.entries.splice(0, 1); - if (dropped) { - this.incrementDroppedEntries(dropped.turnId); - } - } + this.rememberCanonicalEvent(entry, eventId); + this.evictOldestEntryIfNeeded(); + + this.recordRawCapture(input); + } + private recordRawCapture(input: CanonicalDiagnosticRecordInput): void { const raw = this.rawCaptures.get(input.turnId); if (!raw) return; if (raw.expiresAt <= this.now()) { @@ -230,6 +243,28 @@ export class CanonicalAgentDiagnostics { if (input.terminal) raw.active = false; } + private hasRecordedCanonicalEvent(turnId: string, eventId: string): boolean { + if (this.canonicalEventIds.has(canonicalEventKey(turnId, eventId))) return true; + const raw = this.rawCaptures.get(turnId); + return raw?.events.some((event) => canonicalEventId(event) === eventId) ?? false; + } + + private rememberCanonicalEvent(entry: CanonicalDiagnosticEntry, eventId: string | undefined): void { + if (!eventId) return; + const key = canonicalEventKey(entry.turnId, eventId); + this.canonicalEventIds.add(key); + this.canonicalEventIdByEntry.set(entry, key); + } + + private evictOldestEntryIfNeeded(): void { + if (this.entries.length <= CANONICAL_DIAGNOSTIC_RING_CAPACITY) return; + const dropped = this.entries.shift(); + if (!dropped) return; + const eventId = this.canonicalEventIdByEntry.get(dropped); + if (eventId) this.canonicalEventIds.delete(eventId); + this.incrementDroppedEntries(dropped.turnId); + } + /** Export redacted diagnostics and, after separate confirmation, raw content. */ exportTurn( turnId: string, diff --git a/apps/server/src/features/providers/composition/__tests__/provider-composition-container.test.ts b/apps/server/src/features/providers/composition/__tests__/provider-composition-container.test.ts index 405c47486..d41c1ecd6 100644 --- a/apps/server/src/features/providers/composition/__tests__/provider-composition-container.test.ts +++ b/apps/server/src/features/providers/composition/__tests__/provider-composition-container.test.ts @@ -130,6 +130,16 @@ describe("provider composition container", () => { providerIdentities: [], projectUserMessage: () => messages.create("thread-1", "user", "Start", 1), }); + const baselineDiagnostics = canonical.exportTurnDiagnostics("turn-1").entries.length; + canonical.startRawTurnCapture({ turnId: "turn-1", consent: true, expiresInMs: 60_000 }); + const writer = container.resolve(CanonicalAgentWriterClient); + const commit = writer.commit.bind(writer); + let firstReceipt: Awaited> | undefined; + vi.spyOn(writer, "commit").mockImplementation(async (operationId, input) => { + if (firstReceipt) return firstReceipt; + firstReceipt = await commit(operationId, input); + return firstReceipt; + }); await expect(host.events.submit(runtimeBatch())).resolves.toMatchObject({ commit: { outcome: "committed", eventCount: 1 }, @@ -143,6 +153,12 @@ describe("provider composition container", () => { canonicalReceipt: expect.objectContaining({ eventId: "cursor:runtime-event-1" }), })]); }); + await host.events.submit(runtimeBatch()); + const diagnostics = canonical.exportTurnDiagnostics("turn-1", { includeRaw: true, confirmRaw: true }); + expect(diagnostics.entries).toHaveLength(baselineDiagnostics + 1); + expect(diagnostics.rawEvents).toEqual([ + expect.objectContaining({ eventId: "cursor:runtime-event-1" }), + ]); }); it("waits for provider cleanup even when another provider shutdown fails", async () => { diff --git a/apps/server/src/features/providers/composition/__tests__/provider-host-ports.test.ts b/apps/server/src/features/providers/composition/__tests__/provider-host-ports.test.ts index ab7af0d2c..d98c03422 100644 --- a/apps/server/src/features/providers/composition/__tests__/provider-host-ports.test.ts +++ b/apps/server/src/features/providers/composition/__tests__/provider-host-ports.test.ts @@ -54,6 +54,7 @@ describe("createProviderHostPorts", () => { threadControl: {}, grants: {}, events: {}, + diagnostics: {}, publishCanonicalEvents: vi.fn(), ingress: {}, } as never); @@ -82,6 +83,7 @@ describe("createProviderHostPorts", () => { events, }; }); + const recordProviderCommitDiagnostics = vi.fn(() => deliveryOrder.push("diagnostics")); const publishCanonicalEvents = vi.fn(() => deliveryOrder.push("publication")); const acceptCommitted = vi.fn(() => deliveryOrder.push("ingress")); const acknowledgeOperation = vi.fn(async () => { deliveryOrder.push("acknowledge"); }); @@ -93,6 +95,7 @@ describe("createProviderHostPorts", () => { threadControl: {}, grants: {}, events: { commit, acknowledgeOperation }, + diagnostics: { recordProviderCommitDiagnostics }, publishCanonicalEvents, ingress: { acceptCommitted }, } as never); @@ -110,10 +113,11 @@ describe("createProviderHostPorts", () => { delivery: { ingress: "queued" }, }); expect(commit).toHaveBeenCalledWith(expect.any(String), { ...batch, nativeCursor: undefined }); + expect(recordProviderCommitDiagnostics).toHaveBeenCalledWith(events); expect(publishCanonicalEvents).toHaveBeenCalledWith(events); expect(acceptCommitted).toHaveBeenCalledWith(events); expect(acknowledgeOperation).toHaveBeenCalledWith(EXECUTION_ID, expect.any(String)); - expect(deliveryOrder).toEqual(["commit", "publication", "ingress", "acknowledge"]); + expect(deliveryOrder).toEqual(["commit", "diagnostics", "publication", "ingress", "acknowledge"]); }); it("does not hand duplicate or failed commits to ingress", async () => { @@ -138,6 +142,7 @@ describe("createProviderHostPorts", () => { threadControl: {}, grants: {}, events: { commit, acknowledgeOperation: vi.fn() }, + diagnostics: { recordProviderCommitDiagnostics: vi.fn() }, publishCanonicalEvents: vi.fn(), ingress: { acceptCommitted }, } as never); @@ -168,6 +173,7 @@ describe("createProviderHostPorts", () => { threadControl: {}, grants: {}, events: { commit, acknowledgeOperation: vi.fn() }, + diagnostics: { recordProviderCommitDiagnostics: vi.fn() }, publishCanonicalEvents: vi.fn(), ingress: { acceptCommitted: vi.fn() }, } as never); @@ -203,6 +209,7 @@ describe("createProviderHostPorts", () => { })), acknowledgeOperation, }, + diagnostics: { recordProviderCommitDiagnostics: vi.fn() }, publishCanonicalEvents: () => { throw new Error("push unavailable"); }, ingress: { acceptCommitted }, } as never); diff --git a/apps/server/src/features/providers/composition/provider-host-ports.ts b/apps/server/src/features/providers/composition/provider-host-ports.ts index dc37d7b7e..6ea42d79b 100644 --- a/apps/server/src/features/providers/composition/provider-host-ports.ts +++ b/apps/server/src/features/providers/composition/provider-host-ports.ts @@ -5,7 +5,7 @@ import { logger } from "@mcode/shared"; import type { JobObject } from "../../../runtime/process/containment/job-object.js"; import type { EnvService } from "../../../runtime/environment/env-service.js"; import type { ScopedPreGrantService } from "../../agents/permissions/scoped-pre-grant.js"; -import type { CanonicalAgentEventPublisher } from "../../agents/canonical/canonical-agent-boundary.js"; +import type { CanonicalAgentBoundary, CanonicalAgentEventPublisher } from "../../agents/canonical/canonical-agent-boundary.js"; import type { CanonicalAgentWriterClient } from "../../agents/canonical/canonical-agent-writer-client.js"; import type { BrowserAutomationSessionLease } from "../../browser-automation/index.js"; import type { InternalThreadControlMcpRuntime } from "../../thread-control/index.js"; @@ -21,6 +21,7 @@ export interface ProviderHostPortDependencies { threadControl: InternalThreadControlMcpRuntime; grants: ScopedPreGrantService; events: Pick; + diagnostics: Pick; publishCanonicalEvents: CanonicalAgentEventPublisher; ingress: ProviderEventIngress; } @@ -105,6 +106,7 @@ export function createProviderHostPorts( }); let published = true; if (result.outcome === "committed" && result.events.length > 0) { + dependencies.diagnostics.recordProviderCommitDiagnostics(result.events); try { dependencies.publishCanonicalEvents(result.events); } catch { diff --git a/apps/server/src/features/providers/composition/register-providers.ts b/apps/server/src/features/providers/composition/register-providers.ts index 7ad724a34..afb5d236c 100644 --- a/apps/server/src/features/providers/composition/register-providers.ts +++ b/apps/server/src/features/providers/composition/register-providers.ts @@ -9,7 +9,7 @@ import { ProviderRegistry } from "./provider-registry.js"; import { createProviderHostPorts } from "./provider-host-ports.js"; import { BrowserAutomationSessionLease } from "../../browser-automation/index.js"; import { InternalThreadControlMcpRuntime } from "../../thread-control/index.js"; -import { publishCanonicalAgentEvents } from "../../agents/canonical/canonical-agent-boundary.js"; +import { CanonicalAgentBoundary, publishCanonicalAgentEvents } from "../../agents/canonical/canonical-agent-boundary.js"; import { CanonicalAgentWriterClient } from "../../agents/canonical/canonical-agent-writer-client.js"; import { ScopedPreGrantService } from "../../agents/permissions/scoped-pre-grant.js"; import { EnvService } from "../../../runtime/environment/env-service.js"; @@ -94,6 +94,7 @@ export function registerProviderAdapters(container: DependencyContainer): void { threadControl: c.resolve(InternalThreadControlMcpRuntime), grants: c.resolve(ScopedPreGrantService), events: c.resolve(CanonicalAgentWriterClient), + diagnostics: c.resolve(CanonicalAgentBoundary), publishCanonicalEvents: publishCanonicalAgentEvents, ingress: c.resolve(ProviderEventIngress), }), From 8f9fc3663c0ce753ddf54544549e5d3d2e3747e4 Mon Sep 17 00:00:00 2001 From: "(CJ) Chukwudi Nwobodo" <142016413+chuks-qua@users.noreply.github.com> Date: Thu, 24 Sep 2026 20:05:30 +0100 Subject: [PATCH 023/136] perf(server): avoid main thread recovery lookup --- .../agent-service-narrative-persist.test.ts | 4 +-- .../__tests__/agent-service-test-harness.ts | 1 + ...ent-narrative-recovery-coordinator.test.ts | 32 +++++++++++++------ .../parent-narrative-recovery-coordinator.ts | 17 +++------- .../turns/provider-turn-event-application.ts | 2 +- 5 files changed, 32 insertions(+), 24 deletions(-) diff --git a/apps/server/src/features/agents/orchestration/__tests__/agent-service-narrative-persist.test.ts b/apps/server/src/features/agents/orchestration/__tests__/agent-service-narrative-persist.test.ts index 0179e8e25..b2608718a 100644 --- a/apps/server/src/features/agents/orchestration/__tests__/agent-service-narrative-persist.test.ts +++ b/apps/server/src/features/agents/orchestration/__tests__/agent-service-narrative-persist.test.ts @@ -1218,13 +1218,13 @@ describe("AgentService narrative persistence", () => { }); await waitForAgentServiceIngressForTest(service, THREAD_ID); - expect(taskAppend).toHaveBeenCalledWith(THREAD_ID, { + await vi.waitFor(() => expect(taskAppend).toHaveBeenCalledWith(THREAD_ID, { id: "1", content: "Buy groceries - Pick up milk, eggs, bread", status: "pending", activeForm: "Buying groceries", group: "Tasks", - }); + })); }); it("does not persist a TaskCreate whose result errored", () => { diff --git a/apps/server/src/features/agents/orchestration/__tests__/agent-service-test-harness.ts b/apps/server/src/features/agents/orchestration/__tests__/agent-service-test-harness.ts index 91a05a6cd..5d6fbf70b 100644 --- a/apps/server/src/features/agents/orchestration/__tests__/agent-service-test-harness.ts +++ b/apps/server/src/features/agents/orchestration/__tests__/agent-service-test-harness.ts @@ -359,6 +359,7 @@ function narrativeWriterForTest( ): ParentNarrativeRecoveryWriter { return override ?? { async recordParentNarrativeRecovery(_operationId, input) { + if (!parentDurability.loadTurnByExecution(input.executionId)) return { recorded: false }; return { recorded: parentDurability.recordParentNarrativeRecovery(input) }; }, async classifyParentNarrativeRecovery(_operationId, input) { diff --git a/apps/server/src/features/agents/turns/__tests__/parent-narrative-recovery-coordinator.test.ts b/apps/server/src/features/agents/turns/__tests__/parent-narrative-recovery-coordinator.test.ts index cc3180f20..6a9e6e020 100644 --- a/apps/server/src/features/agents/turns/__tests__/parent-narrative-recovery-coordinator.test.ts +++ b/apps/server/src/features/agents/turns/__tests__/parent-narrative-recovery-coordinator.test.ts @@ -9,7 +9,6 @@ import { type ParentNarrativeRecoveryWriter, } from "../parent-narrative-recovery-coordinator.js"; import type { NarrativeStore } from "../../conversation/narrative/narrative-store.js"; -import type { ParentTurnDurability } from "../parent-turn-durability.js"; const EXECUTION_ID = "execution-1"; const THREAD_ID = "thread-1"; @@ -37,16 +36,13 @@ function coordinatorFor(writer: ParentNarrativeRecoveryWriter): ParentNarrativeR const narrativeStore = { recoverySnapshot: vi.fn(() => recoveryItems), } as unknown as NarrativeStore; - const durability = { - loadTurnByExecution: vi.fn(() => ({ id: "turn-1" })), - } as unknown as ParentTurnDurability; - return new ParentNarrativeRecoveryCoordinator(writer, narrativeStore, durability); + return new ParentNarrativeRecoveryCoordinator(writer, narrativeStore); } describe("ParentNarrativeRecoveryCoordinator", () => { it("retries an uncommitted recovery snapshot with the same operation identity", async () => { const recordParentNarrativeRecovery = vi.fn() - .mockResolvedValueOnce({ recorded: false }) + .mockRejectedValueOnce(new Error("writer unavailable")) .mockResolvedValue({ recorded: true }); const writer: ParentNarrativeRecoveryWriter = { recordParentNarrativeRecovery, @@ -60,9 +56,7 @@ describe("ParentNarrativeRecoveryCoordinator", () => { discardedItemIds: [], }; - await expect(coordinator.checkpoint(event)).rejects.toThrow( - `Canonical parent turn was not found: ${EXECUTION_ID}`, - ); + await expect(coordinator.checkpoint(event)).rejects.toThrow("writer unavailable"); await coordinator.checkpoint(event); await coordinator.checkpoint(event); @@ -76,6 +70,26 @@ describe("ParentNarrativeRecoveryCoordinator", () => { await vi.waitFor(() => expect(writer.acknowledgeOperation).toHaveBeenCalledWith(EXECUTION_ID, operationId)); }); + it("treats a missing execution as a normal receipt and retries its snapshot for later events", async () => { + const recordParentNarrativeRecovery = vi.fn() + .mockResolvedValueOnce({ recorded: false }) + .mockResolvedValue({ recorded: true }); + const writer: ParentNarrativeRecoveryWriter = { + recordParentNarrativeRecovery, + classifyParentNarrativeRecovery: vi.fn(), + acknowledgeOperation: vi.fn(async () => undefined), + }; + const coordinator = coordinatorFor(writer); + await coordinator.checkpoint(event); + coordinator.acknowledgeHandled(event); + const laterEvent = { ...event }; + await coordinator.checkpoint(laterEvent); + + expect(recordParentNarrativeRecovery).toHaveBeenCalledTimes(2); + expect(recordParentNarrativeRecovery.mock.calls[1]?.[1]).toEqual(recordParentNarrativeRecovery.mock.calls[0]?.[1]); + expect(recordParentNarrativeRecovery.mock.calls[1]?.[0]).not.toBe(recordParentNarrativeRecovery.mock.calls[0]?.[0]); + }); + it("retries the atomic narration classification without changing its input", async () => { const classifyParentNarrativeRecovery = vi.fn() .mockRejectedValueOnce(new Error("writer unavailable")) diff --git a/apps/server/src/features/agents/turns/parent-narrative-recovery-coordinator.ts b/apps/server/src/features/agents/turns/parent-narrative-recovery-coordinator.ts index e08a4d9c8..a6952b6d2 100644 --- a/apps/server/src/features/agents/turns/parent-narrative-recovery-coordinator.ts +++ b/apps/server/src/features/agents/turns/parent-narrative-recovery-coordinator.ts @@ -8,7 +8,6 @@ import { import type { NarrativeStore } from "../conversation/narrative/narrative-store.js"; import type { CanonicalAgentWriterClient } from "../canonical/canonical-agent-writer-client.js"; import type { ParentNarrativeRecoveryCommit } from "./parent-turn-durability.js"; -import type { ParentTurnDurability } from "./parent-turn-durability.js"; import { serverWorkTrace } from "../diagnostics/server-work-trace.js"; /** The single acknowledged writer used for recovery and text classification. */ @@ -27,12 +26,10 @@ interface ParentNarrativeRecoveryCheckpoint { export class ParentNarrativeRecoveryCoordinator { private readonly fingerprintsByExecution = new Map>(); private readonly preparedByEvent = new WeakMap(); - private readonly existingExecutions = new Set(); constructor( private readonly writer: ParentNarrativeRecoveryWriter, private readonly narrativeStore: NarrativeStore, - private readonly canonicalSink: ParentTurnDurability, ) {} /** Commit only the semantic records changed by this accepted provider event. */ @@ -44,10 +41,11 @@ export class ParentNarrativeRecoveryCoordinator { ? serverWorkTrace.measure("narrative-persist", event.threadId, event.turnExecutionId, persist) : persist(); return pending.then((receipt) => { - if (!receipt.recorded) throw new Error(`Canonical parent turn was not found: ${event.turnExecutionId}`); - if (serverWorkTrace) { - serverWorkTrace.measure("narrative-confirm", event.threadId, event.turnExecutionId, checkpoint.confirm); - } else checkpoint.confirm(); + if (receipt.recorded) { + if (serverWorkTrace) { + serverWorkTrace.measure("narrative-confirm", event.threadId, event.turnExecutionId, checkpoint.confirm); + } else checkpoint.confirm(); + } checkpoint.committed = true; }); } @@ -84,10 +82,6 @@ export class ParentNarrativeRecoveryCoordinator { force = false, ): ParentNarrativeRecoveryCheckpoint | null { if (!this.requiresStructuredRecovery(event) || !event.turnExecutionId) return null; - if (!force && !this.existingExecutions.has(event.turnExecutionId)) { - if (!this.canonicalSink.loadTurnByExecution(event.turnExecutionId)) return null; - this.existingExecutions.add(event.turnExecutionId); - } const existing = this.preparedByEvent.get(event as object); if (existing) return existing; const executionId = event.turnExecutionId; @@ -131,7 +125,6 @@ export class ParentNarrativeRecoveryCoordinator { clear(executionId: string | undefined): void { if (!executionId) return; this.fingerprintsByExecution.delete(executionId); - this.existingExecutions.delete(executionId); } private requiresStructuredRecovery(event: AgentEvent): boolean { diff --git a/apps/server/src/features/agents/turns/provider-turn-event-application.ts b/apps/server/src/features/agents/turns/provider-turn-event-application.ts index a96548302..7df4cda68 100644 --- a/apps/server/src/features/agents/turns/provider-turn-event-application.ts +++ b/apps/server/src/features/agents/turns/provider-turn-event-application.ts @@ -95,7 +95,7 @@ export class ProviderTurnEventApplication implements TurnEventApplication { } }, ); - this.parentNarrativeRecovery = new ParentNarrativeRecoveryCoordinator(narrativeWriter, narrative, parentDurability); + this.parentNarrativeRecovery = new ParentNarrativeRecoveryCoordinator(narrativeWriter, narrative); this.browserNarrativeEventSanitizer = new BrowserNarrativeEventSanitizer( (threadId, toolCallId) => this.narrative.getBufferedToolCalls(threadId) .find((toolCall) => toolCall.toolCallId === toolCallId) From 86b542d56c73933a1d017557881e38cec622198a Mon Sep 17 00:00:00 2001 From: "(CJ) Chukwudi Nwobodo" <142016413+chuks-qua@users.noreply.github.com> Date: Thu, 24 Sep 2026 20:13:00 +0100 Subject: [PATCH 024/136] perf(server): trace assistant text writes during active turns --- .../src/features/agents/diagnostics/server-work-trace.ts | 1 + .../turns/parent-assistant-text-checkpoint-service.ts | 6 +++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/apps/server/src/features/agents/diagnostics/server-work-trace.ts b/apps/server/src/features/agents/diagnostics/server-work-trace.ts index b2ffd912a..f0cde056b 100644 --- a/apps/server/src/features/agents/diagnostics/server-work-trace.ts +++ b/apps/server/src/features/agents/diagnostics/server-work-trace.ts @@ -5,6 +5,7 @@ import { logger } from "@mcode/shared"; export type ServerWorkPhase = | "provider-callback" | "worker-admission" | "worker-wait" | "mailbox-wait" | "canonical-write" | "event-apply" | "narrative-checkpoint" + | "assistant-text-write" | "narrative-prepare" | "narrative-persist" | "narrative-confirm" | "finalization" | "publication" | "terminal-create" | "thread-create" | "agent-send"; diff --git a/apps/server/src/features/agents/turns/parent-assistant-text-checkpoint-service.ts b/apps/server/src/features/agents/turns/parent-assistant-text-checkpoint-service.ts index f42594f9e..5fb229922 100644 --- a/apps/server/src/features/agents/turns/parent-assistant-text-checkpoint-service.ts +++ b/apps/server/src/features/agents/turns/parent-assistant-text-checkpoint-service.ts @@ -13,6 +13,7 @@ import { import { inject, injectable } from "tsyringe"; import { ACTIVE_TURN_WRITE_BATCH_LIMITS } from "../../../runtime/persistence/sqlite/bounded-write-batches.js"; import { PARENT_ASSISTANT_TEXT_RETAINED_LIMITS } from "./active-turn-recovery-retention-policy.js"; +import { serverWorkTrace } from "../diagnostics/server-work-trace.js"; export { PARENT_ASSISTANT_TEXT_RETAINED_LIMITS } from "./active-turn-recovery-retention-policy.js"; @@ -1045,7 +1046,10 @@ export class ParentAssistantTextCheckpointQueue { return false; } try { - const result = this.checkpoints.appendChunk(chunk.entries.map((entry) => entry.input)); + const append = () => this.checkpoints.appendChunk(chunk.entries.map((entry) => entry.input)); + const result = serverWorkTrace + ? serverWorkTrace.measure("assistant-text-write", state.threadId, executionId, append) + : append(); if (result.outcome === "overflow") { this.rejectChunk(executionId, state, chunk, "Parent assistant text recovery capacity reached"); return false; From bcebc7be5e6430f8d405d6f43c3246fdeaa13923 Mon Sep 17 00:00:00 2001 From: "(CJ) Chukwudi Nwobodo" <142016413+chuks-qua@users.noreply.github.com> Date: Thu, 24 Sep 2026 20:06:22 +0100 Subject: [PATCH 025/136] test(server): verify Stop isolation across six fixture tasks --- scripts/perf/seven-thread-live-harness.mjs | 200 +++++++++++++++--- .../perf/seven-thread-live-harness.test.mjs | 9 + 2 files changed, 179 insertions(+), 30 deletions(-) diff --git a/scripts/perf/seven-thread-live-harness.mjs b/scripts/perf/seven-thread-live-harness.mjs index 8a9137c16..8d0ebf669 100644 --- a/scripts/perf/seven-thread-live-harness.mjs +++ b/scripts/perf/seven-thread-live-harness.mjs @@ -26,6 +26,8 @@ import { renderFixtureWrapper } from "../../.agents/skills/verify-mcode/scripts/ import { openRuntimeVerificationSocket } from "../../.agents/skills/verify-mcode/scripts/runtime.mjs"; export const THREAD_COUNT = 7; +const STOP_ONE_THREAD_COUNT = 6; +const STOP_ONE_ORDINAL = 3; const TERMINAL_CONTROL_COUNT = 1; export const WORKLOAD_MODEL = "gpt-5.6-luna"; const PROVIDER_ID = "codex"; @@ -37,7 +39,7 @@ const CLEANUP_RPC_TIMEOUT_MS = 30_000; const PERFORMANCE_DIRECTORY = [".dev", "verification", "performance", "seven-thread-live"]; const FIXTURE_SOURCE = [".agents", "skills", "verify-mcode", "scripts", "transcript-provider-fixture.mjs"]; const FIXTURE_PROMPT = "Seven concurrent performance verifier long narrative"; -const FLAG_OPTIONS = new Set(["--run", "--confirm-run", "--confirm-cleanup"]); +const FLAG_OPTIONS = new Set(["--run", "--confirm-run", "--confirm-cleanup", "--stop-one"]); const VALUE_OPTIONS = new Set(["--label", "--cleanup", "--cleanup-receipt"]); const TERMINAL_TRANSPORTS = new Map([ ["legacy", { @@ -58,6 +60,7 @@ const HELP = `Seven concurrent thread live performance harness Usage: bun scripts/perf/seven-thread-live-harness.mjs --run --confirm-run --label + bun scripts/perf/seven-thread-live-harness.mjs --run --confirm-run --label after --stop-one bun scripts/perf/seven-thread-live-harness.mjs --cleanup --confirm-cleanup bun scripts/perf/seven-thread-live-harness.mjs --cleanup-receipt --confirm-run @@ -65,6 +68,8 @@ The run command creates exactly seven direct threads in this worktree's .dev/fixture-repo. It temporarily routes Codex through the checked-in transcript fixture and restores the previous setting during cleanup. It never falls back to a real Codex model or prints runtime credentials. +--stop-one creates six fixture threads, stops the third while all six are +active, and verifies the other five complete and reload durably. `; /** Parses the small explicit command surface before any runtime mutation. */ @@ -135,11 +140,13 @@ function parseRunCommand(flags, values) { } const label = values.get("--label"); if (label !== "before" && label !== "after") throw new Error("--run requires --label before or --label after"); - return { command: "run", label }; + return flags.has("--stop-one") + ? { command: "run", label, stopOne: true } + : { command: "run", label }; } function parseCleanupReceiptCommand(flags, values, receiptPath) { - if (!flags.has("--confirm-run") || flags.has("--confirm-cleanup") || values.has("--label")) { + if (!flags.has("--confirm-run") || flags.has("--confirm-cleanup") || flags.has("--stop-one") || values.has("--label")) { throw new Error("--cleanup-receipt requires --confirm-run and cannot include run options"); } return { command: "cleanup", receiptPath }; @@ -147,7 +154,7 @@ function parseCleanupReceiptCommand(flags, values, receiptPath) { function parseCleanupCommand(flags, values, receiptPath) { if (!flags.has("--confirm-cleanup")) throw new Error("--cleanup requires --confirm-cleanup"); - if (flags.has("--confirm-run") || values.has("--label")) throw new Error("--cleanup cannot include run options"); + if (flags.has("--confirm-run") || flags.has("--stop-one") || values.has("--label")) throw new Error("--cleanup cannot include run options"); return { command: "cleanup", receiptPath }; } @@ -276,13 +283,14 @@ function isValidServerStall(entry, startedAtMs, endedAtMs) { } /** Creates the stable names used to prove that only this harness owns a thread. */ -export function expectedThreadTitle(runId, ordinal) { - return `Seven-thread live performance ${runId} ${ordinal}/${THREAD_COUNT}`; +export function expectedThreadTitle(runId, ordinal, threadCount = THREAD_COUNT) { + const name = threadCount === STOP_ONE_THREAD_COUNT ? "Six-thread Stop verification" : "Seven-thread live performance"; + return `${name} ${runId} ${ordinal}/${threadCount}`; } /** Runs the fixed seven-thread workload and writes its recovery-capable receipt. */ -export async function runLiveHarness({ repoRoot = resolveRepoRoot(), label }) { - const context = createLiveHarnessContext(repoRoot, label); +export async function runLiveHarness({ repoRoot = resolveRepoRoot(), label, stopOne = false }) { + const context = createLiveHarnessContext(repoRoot, label, stopOne); startLiveHarness(context); try { await executeLiveHarness(context); @@ -295,16 +303,16 @@ export async function runLiveHarness({ repoRoot = resolveRepoRoot(), label }) { return { receiptPath: context.run.receiptPath, receipt: context.receipt }; } -function createLiveHarnessContext(repoRoot, label) { +function createLiveHarnessContext(repoRoot, label, stopOne) { const paths = getRuntimePaths(repoRoot); const ports = requireLocalRuntime(repoRoot); - const run = createRun(repoRoot, label); - const receipt = createReceipt(run, label); + const run = createRun(repoRoot, stopOne ? `${label}-stop-one` : label); + const receipt = createReceipt(run, label, stopOne); const eventLoop = new EventLoopStallSampler(EVENT_LOOP_INTERVAL_MS); const serverStalls = new ServerLogStallReader(paths.logsDir); const healthSampler = new HealthSampler(ports.healthUrl, receipt.metrics.health); const eventState = new Map(); - const terminalEvents = new TerminalEventWaiter(eventState); + const terminalEvents = new TerminalEventWaiter(eventState, stopOne); const turnStarts = new TurnStartWaiter(eventState); return { repoRoot, @@ -321,6 +329,7 @@ function createLiveHarnessContext(repoRoot, label) { socket: null, primaryError: null, workspace: null, + stopOne, }; } @@ -338,7 +347,11 @@ async function executeLiveHarness(context) { await createAndSubscribeThreads(context); const terminal = await preflightTerminalTransport(context); const { completed, sends } = await dispatchFixtureTurns(context); + if (context.stopOne) await waitForAllSixActive(context); + const stop = context.stopOne ? stopOneActiveTurn(context) : null; + void stop?.catch(() => undefined); await sampleActiveControls(context, terminal); + if (stop) await stop; throwFirstRejected(await sends, "Dispatching the controlled fixture turns failed"); context.receipt.state.phase = "waiting-for-events"; await completed; @@ -414,14 +427,14 @@ async function startWorkloadMeasurement(context) { async function createAndSubscribeThreads(context) { const branch = currentFixtureBranch(context.paths.fixtureRepoDir); context.receipt.state.phase = "creating-threads"; - const creations = await Promise.allSettled(Array.from({ length: THREAD_COUNT }, (_, index) => createOwnedThread(context, branch, index + 1))); + const creations = await Promise.allSettled(Array.from({ length: context.receipt.workload.threadCount }, (_, index) => createOwnedThread(context, branch, index + 1))); throwFirstRejected(creations, "Creating verifier-owned direct threads failed"); assertCreatedThreadCount(context.receipt); await subscribeToThreadEvents(context); } async function createOwnedThread(context, branch, ordinal) { - const title = expectedThreadTitle(context.run.id, ordinal); + const title = expectedThreadTitle(context.run.id, ordinal, context.receipt.workload.threadCount); const thread = await measuredRpc(context.socket, context.receipt.metrics.rpc, "thread.create", { workspaceId: context.workspace.id, title, @@ -437,7 +450,7 @@ async function createOwnedThread(context, branch, ordinal) { } function assertCreatedThreadCount(receipt) { - if (receipt.state.threads.length !== THREAD_COUNT) throw new Error("The harness did not create exactly seven threads"); + if (receipt.state.threads.length !== receipt.workload.threadCount) throw new Error("The harness did not create the requested fixture thread count"); } async function subscribeToThreadEvents(context) { @@ -467,6 +480,51 @@ async function dispatchFixtureTurns(context) { return { completed, sends }; } +async function waitForAllSixActive(context) { + const deadline = Date.now() + TURN_TIMEOUT_MS; + const threads = context.receipt.state.threads; + while (Date.now() < deadline) { + if (threads.some((thread) => thread.completedAtMs !== null || thread.persistedAtMs !== null)) { + throw new Error("A fixture turn ended before all six became active; Stop proof is inconclusive"); + } + const target = threads[STOP_ONE_ORDINAL - 1]; + if (threads.every((thread) => thread.startedAtMs !== null) + && target.events.filter((event) => event.type === "assistantMessageBoundary").length >= 10) return; + await new Promise((resolve) => setTimeout(resolve, 20)); + } + throw new Error("All six fixture turns and ten narrative boundaries in the stopped turn were not observed before the deadline"); +} + +async function stopOneActiveTurn(context) { + const target = context.receipt.state.threads[STOP_ONE_ORDINAL - 1]; + const launchedAtMs = NodePerfHooks.performance.now(); + const allSixActiveAtLaunch = context.receipt.state.threads.every(isActiveFixtureThread); + if (!allSixActiveAtLaunch) throw new Error("Stop did not launch while all six fixture turns were active"); + context.receipt.state.stop = { ordinal: STOP_ONE_ORDINAL, threadId: target.id, launchedAtMs, allSixActiveAtLaunch }; + writeReceipt(context.run.receiptPath, context.receipt, context.paths.devDir); + const result = await measuredRpc(context.socket, context.receipt.metrics.rpc, "agent.stop", { threadId: target.id }, undefined, CONTROL_RPC_TIMEOUT_MS); + context.receipt.state.stop = { + ...context.receipt.state.stop, + completedAtMs: NodePerfHooks.performance.now(), + status: result?.status, + dispatchState: result?.dispatchState, + turnExecutionId: result?.turnExecutionId, + snapshotPhase: result?.snapshot?.phase, + }; + if (!isCancelledStopResult(result, target.id)) { + throw new Error("Stop did not return the targeted execution's cancelled outcome"); + } + writeReceipt(context.run.receiptPath, context.receipt, context.paths.devDir); +} + +function isActiveFixtureThread(thread) { + return thread.startedAtMs !== null && thread.completedAtMs === null && thread.persistedAtMs === null; +} + +function isCancelledStopResult(result, threadId) { + return result?.status === "cancelled" && result?.snapshot?.phase === "cancelled" && result?.threadId === threadId; +} + async function sendFixtureTurn(context, thread) { thread.sentAtMs = NodePerfHooks.performance.now(); await measuredRpc(context.socket, context.receipt.metrics.rpc, "agent.send", { @@ -538,17 +596,34 @@ async function readDurableConversations(context) { async function readDurableConversation(context, thread) { const tail = await measuredRpc(context.socket, context.receipt.metrics.rpc, "conversation.tail", { threadId: thread.id, limit: 2 }); - thread.durable = auditConversationTail(tail); + const expectedOutcome = context.stopOne && thread.ordinal === STOP_ONE_ORDINAL ? "cancelled" : "completed"; + const page = expectedOutcome === "cancelled" + ? await measuredRpc(context.socket, context.receipt.metrics.rpc, "conversation.page", { threadId: thread.id, limit: 100 }) + : null; + thread.durable = context.stopOne + ? auditConversationOutcome(tail, page, expectedOutcome, context.receipt.state.stop?.turnExecutionId) + : auditConversationTail(tail); if (!thread.durable.ok) throw new Error(`Durable conversation proof failed for thread ${thread.ordinal}`); } async function verifyCompletedWorkload(context) { const activeAgents = await measuredRpc(context.socket, context.receipt.metrics.rpc, "agent.activeCount", {}); if (activeAgents !== 0) throw new Error("The controlled turns completed but the runtime still reports active agents"); + if (context.stopOne) await verifyStoppedRuntimeSnapshot(context); assertConclusiveControlSamples(context.receipt); if (!auditRun(context.receipt)) throw new Error("The controlled workload completed with event-loss, order, completion, or durability failures"); } +async function verifyStoppedRuntimeSnapshot(context) { + const snapshots = await measuredRpc(context.socket, context.receipt.metrics.rpc, "agent.listRunning", {}); + const stop = context.receipt.state.stop; + stop.reconnectSnapshot = Array.isArray(snapshots) && snapshots.some((snapshot) => + snapshot?.threadId === stop.threadId + && snapshot?.turnExecutionId === stop.turnExecutionId + && snapshot?.phase === "cancelled"); + if (!stop.reconnectSnapshot) throw new Error("The stopped execution did not retain its cancelled reconnect snapshot"); +} + function assertConclusiveControlSamples(receipt) { if (receipt.modelList?.inconclusive || receipt.terminalCapabilities?.inconclusive || receipt.state.terminalCreateInconclusive) { throw new Error("The active-turn control samples were inconclusive because all fixture turns finished before a required RPC could begin"); @@ -647,7 +722,7 @@ function createRun(repoRoot, label) { }; } -function createReceipt(run, label) { +function createReceipt(run, label, stopOne) { return { schemaVersion: 1, runId: run.id, @@ -656,7 +731,8 @@ function createReceipt(run, label) { completedAt: null, ok: false, workload: { - threadCount: THREAD_COUNT, + threadCount: stopOne ? STOP_ONE_THREAD_COUNT : THREAD_COUNT, + scenario: stopOne ? "stop-one" : "baseline", terminalControlCount: TERMINAL_CONTROL_COUNT, provider: PROVIDER_ID, model: WORKLOAD_MODEL, @@ -678,6 +754,7 @@ function createReceipt(run, label) { activeTurnAssertions: [], activeControlLaunch: null, terminalCreateInconclusive: false, + stop: null, threads: [], }, metrics: { @@ -827,6 +904,9 @@ function recordThreadPush(push, thread, turnStarts) { recordAgentEvent(push.data, thread, now, turnStarts); } else if (push.channel === "turn.persisted") { thread.persistedAtMs ??= now; + thread.persistedOutcome ??= push.data.outcome ?? null; + } else if (push.channel === "thread.status") { + thread.status = push.data.status; } } @@ -877,8 +957,9 @@ export function attributeControlLaunchToTurnCompletion(controlLaunch, threads) { } class TerminalEventWaiter { - constructor(state) { + constructor(state, stopOne = false) { this.state = state; + this.stopOne = stopOne; this.resolve = null; } @@ -889,7 +970,10 @@ class TerminalEventWaiter { reject(new Error("Not every controlled thread produced both turn completion and durable persistence events before the deadline")); }, timeoutMs); this.resolve = () => { - if (![...this.state.values()].every((thread) => thread.completedAtMs !== null && thread.persistedAtMs !== null)) return; + if (![...this.state.values()].every((thread) => + this.stopOne && thread.ordinal === STOP_ONE_ORDINAL + ? thread.status === "paused" || thread.status === "cancelled" + : thread.persistedAtMs !== null && thread.completedAtMs !== null)) return; this.resolve = null; clearTimeout(timer); resolve(); @@ -942,13 +1026,62 @@ function auditConversationTail(tail) { }; } +function auditConversationOutcome(tail, page, expectedOutcome, expectedExecutionId) { + const messages = Array.isArray(tail?.messages) ? tail.messages : []; + const assistant = messages.find((message) => message?.role === "assistant"); + const hasUserMessage = messages.some((message) => message?.role === "user"); + const assistantSummary = summarizeAssistantOutcome(assistant); + const narrativeSummary = summarizeNarrativeOutcome(page, assistant); + const summary = { messageCount: messages.length, hasUserMessage, ...assistantSummary, ...narrativeSummary }; + return { + ...summary, + ok: hasExpectedDurableOutcome(summary, expectedOutcome, expectedExecutionId), + }; +} + +function summarizeAssistantOutcome(assistant) { + return { + hasFixtureAssistantMessage: assistant?.content?.includes("Fixture answer:") === true, + assistantContentLength: assistant?.content?.length ?? 0, + outcome: assistant?.outcome ?? null, + outcomeExecutionId: assistant?.outcomeExecutionId ?? null, + }; +} + +function summarizeNarrativeOutcome(page, assistant) { + const narrativeByMessage = page?.narrativeByMessage ?? {}; + const allNarrative = Object.values(narrativeByMessage); + const narrative = assistant ? narrativeByMessage[assistant.id] : null; + return { + hasRetainedNarrative: narrative?.thoughts?.some((segment) => segment.text?.includes("Fixture step 1:")) === true, + narrativeMessageCount: allNarrative.length, + thoughtSegmentCount: allNarrative.reduce((count, batch) => count + (batch?.thoughts?.length ?? 0), 0), + toolCount: allNarrative.reduce((count, batch) => count + (batch?.tools?.length ?? 0), 0), + }; +} + +function hasExpectedDurableOutcome(summary, expectedOutcome, expectedExecutionId) { + if (!summary.hasUserMessage || summary.outcome !== expectedOutcome) return false; + if (expectedOutcome !== "cancelled") return summary.hasFixtureAssistantMessage; + return summary.outcomeExecutionId === expectedExecutionId + && summary.hasRetainedNarrative + && !summary.hasFixtureAssistantMessage; +} + function auditRun(receipt) { return receipt.state.threads.every((thread) => { thread.eventAudit = auditAgentEvents(thread.events); - return thread.eventAudit.sequenceValid && thread.eventAudit.completed && thread.durable?.ok === true; + return thread.eventAudit.sequenceValid && isExpectedTerminal(receipt, thread) && thread.durable?.ok === true; }); } +function isExpectedTerminal(receipt, thread) { + if (receipt.workload.scenario !== "stop-one" || thread.ordinal !== STOP_ONE_ORDINAL) return thread.eventAudit.completed; + return receipt.state.stop?.status === "cancelled" + && receipt.state.stop?.reconnectSnapshot === true + && (thread.status === "paused" || thread.status === "cancelled"); +} + function summarizeEventAudits(threads) { const audits = threads.map((thread) => thread.eventAudit ?? auditAgentEvents(thread.events ?? [])); for (let index = 0; index < threads.length; index += 1) threads[index].eventAudit = audits[index]; @@ -960,10 +1093,17 @@ function summarizeEventAudits(threads) { missingSequences: flatten("missingSequences"), duplicateSequences: flatten("duplicateSequences"), arrivalOrderViolations: flatten("arrivalOrderViolations"), - ok: audits.length === THREAD_COUNT && audits.every((audit) => audit.sequenceValid && audit.completed), + ok: audits.length === threads.length && audits.every((audit) => audit.sequenceValid) + && threads.every((thread) => thread.durable?.ok === true) + && threads.every((thread) => hasExpectedEventTerminal(thread, threads.length)), }; } +function hasExpectedEventTerminal(thread, threadCount) { + if (threadCount !== STOP_ONE_THREAD_COUNT || thread.ordinal !== STOP_ONE_ORDINAL) return thread.eventAudit.completed; + return thread.status === "paused" || thread.status === "cancelled"; +} + async function cleanupOwnedResources(socket, receipt, paths, repoRoot) { const cleanup = { ok: true, ptys: [], threads: [], settingsRestored: false, wrapperRemoved: false, failures: [] }; const threads = Array.isArray(receipt?.state?.threads) ? receipt.state.threads : []; @@ -1001,7 +1141,7 @@ function resolveCleanupTransport(receipt, threads, cleanup) { async function cleanupWorkloadResources(socket, receipt, threads, workspace, terminalTransport, cleanup) { if (cleanup.failures.length > 0) return; if (terminalTransport) await cleanupPtys(socket, threads, terminalTransport, cleanup); - await cleanupThreads(socket, threads, workspace.id, receipt.runId, cleanup); + await cleanupThreads(socket, threads, workspace.id, receipt.runId, receipt.workload.threadCount, cleanup); } async function cleanupPtys(socket, threads, transport, cleanup) { @@ -1091,11 +1231,11 @@ function cleanupRpc(socket, method, params) { return socket.rpc(method, params, cleanupDeadline()); } -async function cleanupThreads(socket, threads, workspaceId, runId, cleanup) { +async function cleanupThreads(socket, threads, workspaceId, runId, threadCount, cleanup) { const current = await currentWorkspaceThreads(socket, workspaceId, cleanup); if (current === null) return; for (const thread of threads) { - await cleanupThread(socket, current, thread, runId, cleanup); + await cleanupThread(socket, current, thread, runId, threadCount, cleanup); } await verifyThreadsRemoved(socket, threads, workspaceId, cleanup); } @@ -1109,14 +1249,14 @@ async function currentWorkspaceThreads(socket, workspaceId, cleanup) { } } -async function cleanupThread(socket, current, thread, runId, cleanup) { +async function cleanupThread(socket, current, thread, runId, threadCount, cleanup) { if (!isReceiptThread(thread)) return; const found = current.find((candidate) => candidate?.id === thread.id); if (!found) { cleanup.threads.push({ threadId: thread.id, outcome: "already-deleted" }); return; } - if (!matchesReceiptThread(found, thread, runId)) { + if (!matchesReceiptThread(found, thread, runId, threadCount)) { cleanup.failures.push(`Refusing to delete thread ${thread.id}: title does not match this receipt`); return; } @@ -1133,8 +1273,8 @@ function isReceiptThread(thread) { return typeof thread?.id === "string" && Number.isInteger(thread?.ordinal); } -function matchesReceiptThread(found, thread, runId) { - return found.title === expectedThreadTitle(runId, thread.ordinal) && found.title === thread.title; +function matchesReceiptThread(found, thread, runId, threadCount) { + return found.title === expectedThreadTitle(runId, thread.ordinal, threadCount) && found.title === thread.title; } async function verifyThreadsRemoved(socket, threads, workspaceId, cleanup) { @@ -1457,7 +1597,7 @@ async function main() { return; } const result = args.command === "run" - ? await runLiveHarness({ label: args.label }) + ? await runLiveHarness({ label: args.label, stopOne: args.stopOne }) : await cleanupReceipt({ receiptPath: args.receiptPath }); process.stdout.write(`${JSON.stringify({ ok: true, receiptPath: relativePath(resolveRepoRoot(), result.receiptPath) })}\n`); } diff --git a/scripts/perf/seven-thread-live-harness.test.mjs b/scripts/perf/seven-thread-live-harness.test.mjs index 5d697b0ff..49ac09b3a 100644 --- a/scripts/perf/seven-thread-live-harness.test.mjs +++ b/scripts/perf/seven-thread-live-harness.test.mjs @@ -20,6 +20,10 @@ NodeTest.test("requires an explicit before or after run confirmation", () => { parseArguments(["--run", "--confirm-run", "--label", "before"]), { command: "run", label: "before" }, ); + NodeAssertStrict.deepEqual( + parseArguments(["--run", "--confirm-run", "--label", "after", "--stop-one"]), + { command: "run", label: "after", stopOne: true }, + ); NodeAssertStrict.throws( () => parseArguments(["--run", "--label", "before"]), /requires --confirm-run/, @@ -36,6 +40,10 @@ NodeTest.test("requires an explicit before or after run confirmation", () => { () => parseArguments(["--cleanup-receipt", "receipt.json", "--confirm-cleanup"]), /requires --confirm-run/, ); + NodeAssertStrict.throws( + () => parseArguments(["--cleanup-receipt", "receipt.json", "--confirm-run", "--stop-one"]), + /cannot include run options/, + ); }); NodeTest.test("uses a stable nearest-rank latency summary", () => { @@ -82,6 +90,7 @@ NodeTest.test("names each of the exact verifier-owned threads", () => { const names = Array.from({ length: THREAD_COUNT }, (_, index) => expectedThreadTitle("run-id", index + 1)); NodeAssertStrict.equal(new Set(names).size, THREAD_COUNT); NodeAssertStrict.deepEqual(names.at(-1), "Seven-thread live performance run-id 7/7"); + NodeAssertStrict.equal(expectedThreadTitle("run-id", 3, 6), "Six-thread Stop verification run-id 3/6"); }); NodeTest.test("uses the same legacy and modern Terminal lifecycle families as the web transport", () => { From d55a582b2de6827153d9c77ae7a4acd98fc1bf3a Mon Sep 17 00:00:00 2001 From: "(CJ) Chukwudi Nwobodo" <142016413+chuks-qua@users.noreply.github.com> Date: Thu, 24 Sep 2026 20:13:17 +0100 Subject: [PATCH 026/136] fix(test): hydrate stopped turn narrative through public RPC --- scripts/perf/seven-thread-live-harness.mjs | 25 +++++++++++----------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/scripts/perf/seven-thread-live-harness.mjs b/scripts/perf/seven-thread-live-harness.mjs index 8d0ebf669..22ffad547 100644 --- a/scripts/perf/seven-thread-live-harness.mjs +++ b/scripts/perf/seven-thread-live-harness.mjs @@ -597,11 +597,12 @@ async function readDurableConversations(context) { async function readDurableConversation(context, thread) { const tail = await measuredRpc(context.socket, context.receipt.metrics.rpc, "conversation.tail", { threadId: thread.id, limit: 2 }); const expectedOutcome = context.stopOne && thread.ordinal === STOP_ONE_ORDINAL ? "cancelled" : "completed"; - const page = expectedOutcome === "cancelled" - ? await measuredRpc(context.socket, context.receipt.metrics.rpc, "conversation.page", { threadId: thread.id, limit: 100 }) + const assistant = tail?.messages?.find((message) => message?.role === "assistant"); + const narrative = expectedOutcome === "cancelled" && assistant + ? await measuredRpc(context.socket, context.receipt.metrics.rpc, "narrative.list", { messageId: assistant.id }) : null; thread.durable = context.stopOne - ? auditConversationOutcome(tail, page, expectedOutcome, context.receipt.state.stop?.turnExecutionId) + ? auditConversationOutcome(tail, narrative, expectedOutcome, context.receipt.state.stop?.turnExecutionId) : auditConversationTail(tail); if (!thread.durable.ok) throw new Error(`Durable conversation proof failed for thread ${thread.ordinal}`); } @@ -1026,12 +1027,12 @@ function auditConversationTail(tail) { }; } -function auditConversationOutcome(tail, page, expectedOutcome, expectedExecutionId) { +function auditConversationOutcome(tail, narrative, expectedOutcome, expectedExecutionId) { const messages = Array.isArray(tail?.messages) ? tail.messages : []; const assistant = messages.find((message) => message?.role === "assistant"); const hasUserMessage = messages.some((message) => message?.role === "user"); const assistantSummary = summarizeAssistantOutcome(assistant); - const narrativeSummary = summarizeNarrativeOutcome(page, assistant); + const narrativeSummary = summarizeNarrativeOutcome(narrative); const summary = { messageCount: messages.length, hasUserMessage, ...assistantSummary, ...narrativeSummary }; return { ...summary, @@ -1048,15 +1049,13 @@ function summarizeAssistantOutcome(assistant) { }; } -function summarizeNarrativeOutcome(page, assistant) { - const narrativeByMessage = page?.narrativeByMessage ?? {}; - const allNarrative = Object.values(narrativeByMessage); - const narrative = assistant ? narrativeByMessage[assistant.id] : null; +function summarizeNarrativeOutcome(narrative) { + const thoughts = narrative?.thoughts ?? []; + const tools = narrative?.tools ?? []; return { - hasRetainedNarrative: narrative?.thoughts?.some((segment) => segment.text?.includes("Fixture step 1:")) === true, - narrativeMessageCount: allNarrative.length, - thoughtSegmentCount: allNarrative.reduce((count, batch) => count + (batch?.thoughts?.length ?? 0), 0), - toolCount: allNarrative.reduce((count, batch) => count + (batch?.tools?.length ?? 0), 0), + hasRetainedNarrative: thoughts.some((segment) => segment.text?.includes("Fixture step 1:")), + thoughtSegmentCount: thoughts.length, + toolCount: tools.length, }; } From f8d2d06d607a840e06cffdb9d3400acce03fd0d0 Mon Sep 17 00:00:00 2001 From: "(CJ) Chukwudi Nwobodo" <142016413+chuks-qua@users.noreply.github.com> Date: Thu, 24 Sep 2026 20:16:34 +0100 Subject: [PATCH 027/136] fix(server): report exact leases after execution worker loss --- .../execution-mailbox-scheduler.test.ts | 26 ++++++++++++++----- .../execution-thread-worker-port.test.ts | 10 ++++--- .../execution-worker-handler.test.ts | 5 ++-- .../execution/execution-mailbox-scheduler.ts | 12 ++++++--- 4 files changed, 39 insertions(+), 14 deletions(-) diff --git a/apps/server/src/features/agents/execution/__tests__/execution-mailbox-scheduler.test.ts b/apps/server/src/features/agents/execution/__tests__/execution-mailbox-scheduler.test.ts index e7c80554b..940612ae8 100644 --- a/apps/server/src/features/agents/execution/__tests__/execution-mailbox-scheduler.test.ts +++ b/apps/server/src/features/agents/execution/__tests__/execution-mailbox-scheduler.test.ts @@ -3,6 +3,7 @@ import { describe, expect, it } from "vitest"; import { ExecutionMailboxScheduler, type ExecutionMailboxLimits, + type ExecutionLostAssignment, } from "../execution-mailbox-scheduler.js"; import type { ExecutionIdentity, @@ -67,7 +68,7 @@ function execution(threadId: string, executionId = `execution-${threadId}`): Exe function fixture(workerCount = 1, limits = LIMITS) { const workers: FakeWorker[] = []; - const lost: ExecutionIdentity[][] = []; + const lost: ExecutionLostAssignment[][] = []; const scheduler = new ExecutionMailboxScheduler({ workerCount, limits, @@ -258,7 +259,7 @@ describe("ExecutionMailboxScheduler", () => { workers[0]?.crash(); expect(await first.completion).toEqual({ kind: "worker-lost" }); expect(await queued.completion).toEqual({ kind: "worker-lost" }); - expect(lost).toEqual([[oldExecution]]); + expect(lost).toEqual([[{ execution: oldExecution, lease: oldLease }]]); expect(workers[0]?.terminated).toBe(true); expect(scheduler.claim(execution("unavailable"), 2)).toEqual({ kind: "worker-unavailable" }); expect(scheduler.submit({ execution: oldExecution, lease: oldLease, command: { kind: "stop", requestId: "stale" }, byteLength: 100 })) @@ -267,13 +268,19 @@ describe("ExecutionMailboxScheduler", () => { expect(scheduler.replaceWorker(0)).toBe(true); expect(workers[1]?.requests).toHaveLength(0); - const newExecution = execution("same-thread", "new-execution"); + const newExecution = oldExecution; const newLease = claimed(scheduler, newExecution, 2); + expect(newLease.workerGeneration).toBeGreaterThan(oldLease.workerGeneration); expect(scheduler.submit({ execution: newExecution, lease: oldLease, command: { kind: "start" }, byteLength: 100 })) .toEqual({ kind: "stale-execution" }); const newEvent = admitted(scheduler, newExecution, newLease, { kind: "event", sequence: 1 }); + let newEventSettled = false; + void newEvent.completion.then(() => { newEventSettled = true; }); workers[0]?.reply(oldRequest, 99); + await Promise.resolve(); + expect(newEventSettled).toBe(false); expect(scheduler.depth().pending).toBe(1); + expect(workers[1]?.terminated).toBe(false); workers[1]?.reply(request(workers[1]!, 0), 1); expect(await newEvent.completion).toEqual({ kind: "reply", result: { revision: 1 } }); expect(scheduler.release(oldExecution, oldLease)).toBe(false); @@ -290,7 +297,7 @@ describe("ExecutionMailboxScheduler", () => { data: { ...sent, execution: execution("other"), result: { revision: 1 } }, })); expect(await admission.completion).toEqual({ kind: "worker-lost" }); - expect(lost).toEqual([[identity]]); + expect(lost).toEqual([[{ execution: identity, lease }]]); scheduler.shutdown(); }); @@ -298,15 +305,22 @@ describe("ExecutionMailboxScheduler", () => { const { scheduler, workers, lost } = fixture(2); const failed = execution("failed"); const healthy = execution("healthy"); + const alsoFailed = execution("also-failed"); const failedLease = claimed(scheduler, failed); const healthyLease = claimed(scheduler, healthy); + const alsoFailedLease = claimed(scheduler, alsoFailed); const failedEvent = admitted(scheduler, failed, failedLease, { kind: "event", sequence: 1 }); const healthyEvent = admitted(scheduler, healthy, healthyLease, { kind: "event", sequence: 1 }); + const alsoFailedEvent = admitted(scheduler, alsoFailed, alsoFailedLease, { kind: "event", sequence: 1 }); workers[0]?.crash(); workers[1]?.reply(request(workers[1]!, 0), 1); expect(await failedEvent.completion).toEqual({ kind: "worker-lost" }); + expect(await alsoFailedEvent.completion).toEqual({ kind: "worker-lost" }); expect(await healthyEvent.completion).toEqual({ kind: "reply", result: { revision: 1 } }); - expect(lost).toEqual([[failed]]); + expect(lost).toEqual([[ + { execution: failed, lease: failedLease }, + { execution: alsoFailed, lease: alsoFailedLease }, + ]]); const another = execution("another"); expect(claimed(scheduler, another).workerIndex).toBe(1); scheduler.shutdown(); @@ -319,7 +333,7 @@ describe("ExecutionMailboxScheduler", () => { const event = admitted(scheduler, identity, lease, { kind: "event", sequence: 1 }); workers[0]?.close(); expect(await event.completion).toEqual({ kind: "worker-lost" }); - expect(lost).toEqual([[identity]]); + expect(lost).toEqual([[{ execution: identity, lease }]]); expect(scheduler.depth()).toMatchObject({ pending: 0, activeExecutions: 0 }); scheduler.shutdown(); }); diff --git a/apps/server/src/features/agents/execution/__tests__/execution-thread-worker-port.test.ts b/apps/server/src/features/agents/execution/__tests__/execution-thread-worker-port.test.ts index 07b24f41e..99ba3ac89 100644 --- a/apps/server/src/features/agents/execution/__tests__/execution-thread-worker-port.test.ts +++ b/apps/server/src/features/agents/execution/__tests__/execution-thread-worker-port.test.ts @@ -5,7 +5,11 @@ import type { DataOnlyParentTurnFinishInput, DataOnlyParentTurnStartInput, } from "../../canonical/canonical-parent-turn-write.js"; -import { ExecutionMailboxScheduler, type ExecutionMailboxLimits } from "../execution-mailbox-scheduler.js"; +import { + ExecutionMailboxScheduler, + type ExecutionMailboxLimits, + type ExecutionLostAssignment, +} from "../execution-mailbox-scheduler.js"; import type { ExecutionIdentity, ExecutionLease } from "../execution-mailbox-protocol.js"; import { ExecutionThreadWorkerPort } from "../execution-worker-port.js"; import type { @@ -63,7 +67,7 @@ class AcknowledgingWriter implements ExecutionSemanticWriter { function fixture(writer: ExecutionSemanticWriter, workerUrl?: URL) { const ports: ExecutionThreadWorkerPort[] = []; - const lost: ExecutionIdentity[][] = []; + const lost: ExecutionLostAssignment[][] = []; const scheduler = new ExecutionMailboxScheduler({ workerCount: 1, limits: LIMITS, @@ -146,7 +150,7 @@ describe("ExecutionThreadWorkerPort", () => { await expect(port.whenReady()).resolves.toBe(true); await expect(submit(scheduler, lease, { kind: "start", providerId: "codex", input: START_INPUT })) .resolves.toEqual({ kind: "worker-lost" }); - expect(lost).toEqual([[EXECUTION]]); + expect(lost).toEqual([[{ execution: EXECUTION, lease }]]); expect(scheduler.depth()).toMatchObject({ pending: 0, activeExecutions: 0 }); } finally { scheduler.shutdown(); diff --git a/apps/server/src/features/agents/execution/__tests__/execution-worker-handler.test.ts b/apps/server/src/features/agents/execution/__tests__/execution-worker-handler.test.ts index 751e19b36..bdeea3c97 100644 --- a/apps/server/src/features/agents/execution/__tests__/execution-worker-handler.test.ts +++ b/apps/server/src/features/agents/execution/__tests__/execution-worker-handler.test.ts @@ -9,6 +9,7 @@ import { ExecutionMailboxScheduler, type ExecutionMailboxCommand, type ExecutionMailboxLimits, + type ExecutionLostAssignment, } from "../execution-mailbox-scheduler.js"; import type { ExecutionIdentity, @@ -102,7 +103,7 @@ class InlineWorker implements ExecutionWorkerPort({ workerCount: 1, limits: LIMITS, @@ -242,7 +243,7 @@ describe("ExecutionWorkerHandler through its scheduler", () => { await committed(submit(scheduler, lease, { kind: "start", providerId: "codex", input: START_INPUT }), 1); await expect(submit(scheduler, lease, { kind: "event", events: [eventDraft()] }).completion) .resolves.toEqual({ kind: "worker-lost" }); - expect(lost).toEqual([[EXECUTION]]); + expect(lost).toEqual([[{ execution: EXECUTION, lease }]]); expect(worker.terminated).toBe(true); expect(scheduler.depth()).toMatchObject({ pending: 0, activeExecutions: 0 }); scheduler.shutdown(); diff --git a/apps/server/src/features/agents/execution/execution-mailbox-scheduler.ts b/apps/server/src/features/agents/execution/execution-mailbox-scheduler.ts index cc3fee393..9f6856651 100644 --- a/apps/server/src/features/agents/execution/execution-mailbox-scheduler.ts +++ b/apps/server/src/features/agents/execution/execution-mailbox-scheduler.ts @@ -60,12 +60,18 @@ export interface ExecutionMailboxLimits { readonly reservedPerExecutionControlBytes: number; } +/** Exact ownership that a host must revoke before replacing a lost worker. */ +export interface ExecutionLostAssignment { + readonly execution: ExecutionIdentity; + readonly lease: ExecutionLease; +} + /** The host supplies durable owner epochs and a worker factory for each fixed slot. */ export interface ExecutionMailboxOptions { readonly workerCount: number; readonly limits: ExecutionMailboxLimits; readonly createWorker: (workerIndex: number) => ExecutionWorkerPort, Result>; - readonly onWorkerLost: (executions: readonly ExecutionIdentity[]) => void; + readonly onWorkerLost: (assignments: readonly ExecutionLostAssignment[]) => void; } export type ExecutionClaim = @@ -373,9 +379,9 @@ export class ExecutionMailboxScheduler assignment.slot === slot) - .map((assignment) => assignment.execution); + .map((assignment) => ({ execution: assignment.execution, lease: assignment.lease })); this.settleSlot(slot, "worker-lost"); - for (const execution of revoked) this.byThread.delete(execution.threadId); + for (const assignment of revoked) this.byThread.delete(assignment.execution.threadId); slot.activeCount = 0; this.onWorkerLost(revoked); } From 2eeb2104661e97c6e7c98c3a7d7be3656709ace6 Mon Sep 17 00:00:00 2001 From: "(CJ) Chukwudi Nwobodo" <142016413+chuks-qua@users.noreply.github.com> Date: Thu, 24 Sep 2026 20:17:56 +0100 Subject: [PATCH 028/136] fix(test): read structured server work stalls --- scripts/perf/seven-thread-live-harness.mjs | 38 +++++++++++++------ .../perf/seven-thread-live-harness.test.mjs | 14 +++++++ 2 files changed, 40 insertions(+), 12 deletions(-) diff --git a/scripts/perf/seven-thread-live-harness.mjs b/scripts/perf/seven-thread-live-harness.mjs index 22ffad547..48eb64ce6 100644 --- a/scripts/perf/seven-thread-live-harness.mjs +++ b/scripts/perf/seven-thread-live-harness.mjs @@ -259,27 +259,41 @@ export function normalizeTerminalSessions(transport, sessions) { /** Extracts only Mcode server stall entries from JSONL server diagnostics. */ export function parseServerStallEntries(contents, startedAtMs, endedAtMs) { if (typeof contents !== "string") return []; - return contents.split(/\r?\n/).flatMap((line) => parseServerStallLine(line, startedAtMs, endedAtMs)); + const byTimestamp = new Map(); + for (const line of contents.split(/\r?\n/)) { + const sample = parseServerStallLine(line, startedAtMs, endedAtMs); + if (!sample) continue; + const previous = byTimestamp.get(sample.timestamp); + if (!previous || sample.source === "server-work-stall") byTimestamp.set(sample.timestamp, sample); + } + return [...byTimestamp.values()].map(({ timestamp, stalledMs }) => ({ timestamp, stalledMs })); } function parseServerStallLine(line, startedAtMs, endedAtMs) { - if (!line) return []; + if (!line) return null; try { const entry = JSON.parse(line); - if (!isValidServerStall(entry, startedAtMs, endedAtMs)) return []; - return [{ timestamp: entry.timestamp, stalledMs: entry.stalledMs }]; + const sample = serverStallSample(entry); + if (!sample || !isValidServerStallTime(entry.timestamp, startedAtMs, endedAtMs)) return null; + return { timestamp: entry.timestamp, ...sample }; } catch { - return []; + return null; + } +} + +function serverStallSample(entry) { + if (entry?.kind === "server-work-stall" && entry.message === "Server work trace" && Number.isFinite(entry.delayMs)) { + return { source: "server-work-stall", stalledMs: entry.delayMs }; + } + if (entry?.message === "Event loop stalled" && Number.isFinite(entry.stalledMs)) { + return { source: "event-loop-stalled", stalledMs: entry.stalledMs }; } + return null; } -function isValidServerStall(entry, startedAtMs, endedAtMs) { - const timestampMs = Date.parse(entry?.timestamp); - return entry?.message === "Event loop stalled" - && Number.isFinite(timestampMs) - && timestampMs >= startedAtMs - && timestampMs <= endedAtMs - && Number.isFinite(entry?.stalledMs); +function isValidServerStallTime(timestamp, startedAtMs, endedAtMs) { + const timestampMs = Date.parse(timestamp); + return Number.isFinite(timestampMs) && timestampMs >= startedAtMs && timestampMs <= endedAtMs; } /** Creates the stable names used to prove that only this harness owns a thread. */ diff --git a/scripts/perf/seven-thread-live-harness.test.mjs b/scripts/perf/seven-thread-live-harness.test.mjs index 49ac09b3a..ed8ab0a4e 100644 --- a/scripts/perf/seven-thread-live-harness.test.mjs +++ b/scripts/perf/seven-thread-live-harness.test.mjs @@ -176,3 +176,17 @@ NodeTest.test("keeps only server diagnostic stall entries inside the workload wi ].join("\n"), Date.parse("2026-09-24T10:00:00.500Z"), Date.parse("2026-09-24T10:00:02.500Z")); NodeAssertStrict.deepEqual(entries, [{ timestamp: "2026-09-24T10:00:02.000Z", stalledMs: 750 }]); }); + +NodeTest.test("reads structured server work stalls and prefers them over duplicate legacy warnings", () => { + const entries = parseServerStallEntries([ + JSON.stringify({ timestamp: "2026-09-24T19:07:27.116Z", message: "Event loop stalled", stalledMs: 240 }), + JSON.stringify({ timestamp: "2026-09-24T19:07:27.116Z", message: "Server work trace", kind: "server-work-stall", delayMs: 260.15 }), + JSON.stringify({ timestamp: "2026-09-24T19:07:37.927Z", message: "Server work trace", kind: "server-work-stall", delayMs: 3639.19 }), + JSON.stringify({ timestamp: "2026-09-24T19:07:39.550Z", message: "Server work trace", kind: "other", delayMs: 1603 }), + JSON.stringify({ timestamp: "2026-09-24T19:07:50.000Z", message: "Server work trace", kind: "server-work-stall", delayMs: 9000 }), + ].join("\n"), Date.parse("2026-09-24T19:07:26.326Z"), Date.parse("2026-09-24T19:07:46.625Z")); + NodeAssertStrict.deepEqual(entries, [ + { timestamp: "2026-09-24T19:07:27.116Z", stalledMs: 260.15 }, + { timestamp: "2026-09-24T19:07:37.927Z", stalledMs: 3639.19 }, + ]); +}); From e079d905caf049d5ebb89889d516c73a78a9f142 Mon Sep 17 00:00:00 2001 From: "(CJ) Chukwudi Nwobodo" <142016413+chuks-qua@users.noreply.github.com> Date: Thu, 24 Sep 2026 20:18:57 +0100 Subject: [PATCH 029/136] test(server): stop one task while six peers continue --- scripts/perf/seven-thread-live-harness.mjs | 56 ++++++++++--------- .../perf/seven-thread-live-harness.test.mjs | 2 +- 2 files changed, 30 insertions(+), 28 deletions(-) diff --git a/scripts/perf/seven-thread-live-harness.mjs b/scripts/perf/seven-thread-live-harness.mjs index 48eb64ce6..fe2e14ff3 100644 --- a/scripts/perf/seven-thread-live-harness.mjs +++ b/scripts/perf/seven-thread-live-harness.mjs @@ -26,7 +26,6 @@ import { renderFixtureWrapper } from "../../.agents/skills/verify-mcode/scripts/ import { openRuntimeVerificationSocket } from "../../.agents/skills/verify-mcode/scripts/runtime.mjs"; export const THREAD_COUNT = 7; -const STOP_ONE_THREAD_COUNT = 6; const STOP_ONE_ORDINAL = 3; const TERMINAL_CONTROL_COUNT = 1; export const WORKLOAD_MODEL = "gpt-5.6-luna"; @@ -68,8 +67,8 @@ The run command creates exactly seven direct threads in this worktree's .dev/fixture-repo. It temporarily routes Codex through the checked-in transcript fixture and restores the previous setting during cleanup. It never falls back to a real Codex model or prints runtime credentials. ---stop-one creates six fixture threads, stops the third while all six are -active, and verifies the other five complete and reload durably. +--stop-one creates seven fixture threads, stops the third while all seven are +active, and verifies the other six complete and reload durably. `; /** Parses the small explicit command surface before any runtime mutation. */ @@ -297,8 +296,10 @@ function isValidServerStallTime(timestamp, startedAtMs, endedAtMs) { } /** Creates the stable names used to prove that only this harness owns a thread. */ -export function expectedThreadTitle(runId, ordinal, threadCount = THREAD_COUNT) { - const name = threadCount === STOP_ONE_THREAD_COUNT ? "Six-thread Stop verification" : "Seven-thread live performance"; +export function expectedThreadTitle(runId, ordinal, threadCount = THREAD_COUNT, scenario = "baseline") { + const name = scenario === "stop-one" + ? `${threadCount === 6 ? "Six" : "Seven"}-thread Stop verification` + : "Seven-thread live performance"; return `${name} ${runId} ${ordinal}/${threadCount}`; } @@ -361,7 +362,7 @@ async function executeLiveHarness(context) { await createAndSubscribeThreads(context); const terminal = await preflightTerminalTransport(context); const { completed, sends } = await dispatchFixtureTurns(context); - if (context.stopOne) await waitForAllSixActive(context); + if (context.stopOne) await waitForAllSevenActive(context); const stop = context.stopOne ? stopOneActiveTurn(context) : null; void stop?.catch(() => undefined); await sampleActiveControls(context, terminal); @@ -448,7 +449,7 @@ async function createAndSubscribeThreads(context) { } async function createOwnedThread(context, branch, ordinal) { - const title = expectedThreadTitle(context.run.id, ordinal, context.receipt.workload.threadCount); + const title = expectedThreadTitle(context.run.id, ordinal, context.receipt.workload.threadCount, context.receipt.workload.scenario); const thread = await measuredRpc(context.socket, context.receipt.metrics.rpc, "thread.create", { workspaceId: context.workspace.id, title, @@ -494,27 +495,27 @@ async function dispatchFixtureTurns(context) { return { completed, sends }; } -async function waitForAllSixActive(context) { +async function waitForAllSevenActive(context) { const deadline = Date.now() + TURN_TIMEOUT_MS; const threads = context.receipt.state.threads; while (Date.now() < deadline) { if (threads.some((thread) => thread.completedAtMs !== null || thread.persistedAtMs !== null)) { - throw new Error("A fixture turn ended before all six became active; Stop proof is inconclusive"); + throw new Error("A fixture turn ended before all seven became active; Stop proof is inconclusive"); } const target = threads[STOP_ONE_ORDINAL - 1]; if (threads.every((thread) => thread.startedAtMs !== null) && target.events.filter((event) => event.type === "assistantMessageBoundary").length >= 10) return; await new Promise((resolve) => setTimeout(resolve, 20)); } - throw new Error("All six fixture turns and ten narrative boundaries in the stopped turn were not observed before the deadline"); + throw new Error("All seven fixture turns and ten narrative boundaries in the stopped turn were not observed before the deadline"); } async function stopOneActiveTurn(context) { const target = context.receipt.state.threads[STOP_ONE_ORDINAL - 1]; const launchedAtMs = NodePerfHooks.performance.now(); - const allSixActiveAtLaunch = context.receipt.state.threads.every(isActiveFixtureThread); - if (!allSixActiveAtLaunch) throw new Error("Stop did not launch while all six fixture turns were active"); - context.receipt.state.stop = { ordinal: STOP_ONE_ORDINAL, threadId: target.id, launchedAtMs, allSixActiveAtLaunch }; + const allSevenActiveAtLaunch = context.receipt.state.threads.every(isActiveFixtureThread); + if (!allSevenActiveAtLaunch) throw new Error("Stop did not launch while all seven fixture turns were active"); + context.receipt.state.stop = { ordinal: STOP_ONE_ORDINAL, threadId: target.id, launchedAtMs, allSevenActiveAtLaunch }; writeReceipt(context.run.receiptPath, context.receipt, context.paths.devDir); const result = await measuredRpc(context.socket, context.receipt.metrics.rpc, "agent.stop", { threadId: target.id }, undefined, CONTROL_RPC_TIMEOUT_MS); context.receipt.state.stop = { @@ -676,7 +677,7 @@ async function recordFinalReceiptMetrics(context) { receipt.metrics.turnCompletion = summarizeLatency(receipt.state.threads.map((thread) => elapsedSinceSend(thread, "completedAtMs"))); receipt.metrics.turnDurability = summarizeLatency(receipt.state.threads.map((thread) => elapsedSinceSend(thread, "persistedAtMs"))); receipt.state.activeControlLaunch = attributeControlLaunchToTurnCompletion(receipt.state.activeControlLaunch, receipt.state.threads); - receipt.metrics.events = summarizeEventAudits(receipt.state.threads); + receipt.metrics.events = summarizeEventAudits(receipt.state.threads, receipt.workload.scenario); receipt.metrics.harnessEventLoopStalls = { scope: "harness-process", intervalMs: EVENT_LOOP_INTERVAL_MS, @@ -746,7 +747,7 @@ function createReceipt(run, label, stopOne) { completedAt: null, ok: false, workload: { - threadCount: stopOne ? STOP_ONE_THREAD_COUNT : THREAD_COUNT, + threadCount: THREAD_COUNT, scenario: stopOne ? "stop-one" : "baseline", terminalControlCount: TERMINAL_CONTROL_COUNT, provider: PROVIDER_ID, @@ -1082,6 +1083,7 @@ function hasExpectedDurableOutcome(summary, expectedOutcome, expectedExecutionId } function auditRun(receipt) { + if (receipt.state.threads.length !== THREAD_COUNT) return false; return receipt.state.threads.every((thread) => { thread.eventAudit = auditAgentEvents(thread.events); return thread.eventAudit.sequenceValid && isExpectedTerminal(receipt, thread) && thread.durable?.ok === true; @@ -1095,7 +1097,7 @@ function isExpectedTerminal(receipt, thread) { && (thread.status === "paused" || thread.status === "cancelled"); } -function summarizeEventAudits(threads) { +function summarizeEventAudits(threads, scenario) { const audits = threads.map((thread) => thread.eventAudit ?? auditAgentEvents(thread.events ?? [])); for (let index = 0; index < threads.length; index += 1) threads[index].eventAudit = audits[index]; const flatten = (field) => audits.flatMap((audit) => audit[field] ?? []); @@ -1106,14 +1108,14 @@ function summarizeEventAudits(threads) { missingSequences: flatten("missingSequences"), duplicateSequences: flatten("duplicateSequences"), arrivalOrderViolations: flatten("arrivalOrderViolations"), - ok: audits.length === threads.length && audits.every((audit) => audit.sequenceValid) + ok: audits.length === THREAD_COUNT && audits.every((audit) => audit.sequenceValid) && threads.every((thread) => thread.durable?.ok === true) - && threads.every((thread) => hasExpectedEventTerminal(thread, threads.length)), + && threads.every((thread) => hasExpectedEventTerminal(thread, scenario)), }; } -function hasExpectedEventTerminal(thread, threadCount) { - if (threadCount !== STOP_ONE_THREAD_COUNT || thread.ordinal !== STOP_ONE_ORDINAL) return thread.eventAudit.completed; +function hasExpectedEventTerminal(thread, scenario) { + if (scenario !== "stop-one" || thread.ordinal !== STOP_ONE_ORDINAL) return thread.eventAudit.completed; return thread.status === "paused" || thread.status === "cancelled"; } @@ -1154,7 +1156,7 @@ function resolveCleanupTransport(receipt, threads, cleanup) { async function cleanupWorkloadResources(socket, receipt, threads, workspace, terminalTransport, cleanup) { if (cleanup.failures.length > 0) return; if (terminalTransport) await cleanupPtys(socket, threads, terminalTransport, cleanup); - await cleanupThreads(socket, threads, workspace.id, receipt.runId, receipt.workload.threadCount, cleanup); + await cleanupThreads(socket, threads, workspace.id, receipt.runId, receipt.workload.threadCount, receipt.workload.scenario, cleanup); } async function cleanupPtys(socket, threads, transport, cleanup) { @@ -1244,11 +1246,11 @@ function cleanupRpc(socket, method, params) { return socket.rpc(method, params, cleanupDeadline()); } -async function cleanupThreads(socket, threads, workspaceId, runId, threadCount, cleanup) { +async function cleanupThreads(socket, threads, workspaceId, runId, threadCount, scenario, cleanup) { const current = await currentWorkspaceThreads(socket, workspaceId, cleanup); if (current === null) return; for (const thread of threads) { - await cleanupThread(socket, current, thread, runId, threadCount, cleanup); + await cleanupThread(socket, current, thread, runId, threadCount, scenario, cleanup); } await verifyThreadsRemoved(socket, threads, workspaceId, cleanup); } @@ -1262,14 +1264,14 @@ async function currentWorkspaceThreads(socket, workspaceId, cleanup) { } } -async function cleanupThread(socket, current, thread, runId, threadCount, cleanup) { +async function cleanupThread(socket, current, thread, runId, threadCount, scenario, cleanup) { if (!isReceiptThread(thread)) return; const found = current.find((candidate) => candidate?.id === thread.id); if (!found) { cleanup.threads.push({ threadId: thread.id, outcome: "already-deleted" }); return; } - if (!matchesReceiptThread(found, thread, runId, threadCount)) { + if (!matchesReceiptThread(found, thread, runId, threadCount, scenario)) { cleanup.failures.push(`Refusing to delete thread ${thread.id}: title does not match this receipt`); return; } @@ -1286,8 +1288,8 @@ function isReceiptThread(thread) { return typeof thread?.id === "string" && Number.isInteger(thread?.ordinal); } -function matchesReceiptThread(found, thread, runId, threadCount) { - return found.title === expectedThreadTitle(runId, thread.ordinal, threadCount) && found.title === thread.title; +function matchesReceiptThread(found, thread, runId, threadCount, scenario) { + return found.title === expectedThreadTitle(runId, thread.ordinal, threadCount, scenario) && found.title === thread.title; } async function verifyThreadsRemoved(socket, threads, workspaceId, cleanup) { diff --git a/scripts/perf/seven-thread-live-harness.test.mjs b/scripts/perf/seven-thread-live-harness.test.mjs index ed8ab0a4e..2db017f2f 100644 --- a/scripts/perf/seven-thread-live-harness.test.mjs +++ b/scripts/perf/seven-thread-live-harness.test.mjs @@ -90,7 +90,7 @@ NodeTest.test("names each of the exact verifier-owned threads", () => { const names = Array.from({ length: THREAD_COUNT }, (_, index) => expectedThreadTitle("run-id", index + 1)); NodeAssertStrict.equal(new Set(names).size, THREAD_COUNT); NodeAssertStrict.deepEqual(names.at(-1), "Seven-thread live performance run-id 7/7"); - NodeAssertStrict.equal(expectedThreadTitle("run-id", 3, 6), "Six-thread Stop verification run-id 3/6"); + NodeAssertStrict.equal(expectedThreadTitle("run-id", 3, THREAD_COUNT, "stop-one"), "Seven-thread Stop verification run-id 3/7"); }); NodeTest.test("uses the same legacy and modern Terminal lifecycle families as the web transport", () => { From 671c2098d8f1956c3d827cd4830ec9919ccb706a Mon Sep 17 00:00:00 2001 From: "(CJ) Chukwudi Nwobodo" <142016413+chuks-qua@users.noreply.github.com> Date: Thu, 24 Sep 2026 20:19:24 +0100 Subject: [PATCH 030/136] fix(server): release SQLite locks between writer recovery batches --- .../canonical-agent-writer-client.test.ts | 64 ++++++++++++++- .../canonical/canonical-agent-boundary.ts | 43 ++++++++++- .../canonical-agent-writer-receipts.ts | 77 ++++++++++++++----- .../canonical-agent-writer.worker.ts | 42 +++++++--- .../sqlite/bounded-write-batches.ts | 8 +- 5 files changed, 194 insertions(+), 40 deletions(-) diff --git a/apps/server/src/features/agents/canonical/__tests__/canonical-agent-writer-client.test.ts b/apps/server/src/features/agents/canonical/__tests__/canonical-agent-writer-client.test.ts index a5eef21bc..92a1ea951 100644 --- a/apps/server/src/features/agents/canonical/__tests__/canonical-agent-writer-client.test.ts +++ b/apps/server/src/features/agents/canonical/__tests__/canonical-agent-writer-client.test.ts @@ -94,11 +94,11 @@ function events(): CanonicalAgentEventDraft[] { ]; } -function recoveryToolCall(): ParentNarrativeRecoveryItem { +function recoveryToolCall(id = "writer-recovery-tool"): ParentNarrativeRecoveryItem { return { kind: "toolCall", record: { - id: "writer-recovery-tool", + id, message_id: "", parent_tool_call_id: null, tool_name: "Read", @@ -219,6 +219,66 @@ describe("canonical SQLite writer", () => { })).rejects.toThrow("Canonical writer write-failed"); }); + it("converges after a failure between bounded recovery batches", async () => { + writer = new CanonicalAgentWriterClient(dbPath); + await writer.commit("batched-recovery-start", { + threadId: THREAD_ID, turnId: TURN_ID, executionId: EXECUTION_ID, phase: "running", events: events(), + }); + const input = { + executionId: EXECUTION_ID, + items: Array.from({ length: 65 }, (_, index) => recoveryToolCall(`batched-${index}`)), + }; + db.run(`CREATE TRIGGER reject_last_recovery BEFORE INSERT ON canonical_agent_items + WHEN NEW.id = 'toolCall:batched-64' BEGIN SELECT RAISE(FAIL, 'injected recovery failure'); END`); + await expect(writer.recordParentNarrativeRecovery("batched-recovery", input)) + .rejects.toThrow("Canonical writer write-failed"); + const partial = db.prepare("SELECT COUNT(*) AS count FROM canonical_agent_items WHERE id LIKE 'toolCall:batched-%'") + .get() as { count: number }; + expect(partial.count).toBeGreaterThan(0); + expect(partial.count).toBeLessThan(65); + expect(db.prepare("SELECT COUNT(*) AS count FROM canonical_writer_operation_receipts WHERE operation_id = ?") + .get("batched-recovery")).toEqual({ count: 0 }); + db.run("DROP TRIGGER reject_last_recovery"); + expect(await writer.recordParentNarrativeRecovery("batched-recovery", input)).toEqual({ recorded: true }); + expect(await writer.recordParentNarrativeRecovery("batched-recovery", input)).toEqual({ recorded: true }); + expect(db.prepare("SELECT COUNT(*) AS count FROM canonical_agent_items WHERE id LIKE 'toolCall:batched-%'") + .get()).toEqual({ count: 65 }); + expect(db.prepare("SELECT COUNT(*) AS count FROM canonical_writer_operation_receipts WHERE operation_id = ?") + .get("batched-recovery")).toEqual({ count: 1 }); + }, 30_000); + + it("lets a second SQLite writer proceed while recovery continues across batches", async () => { + writer = new CanonicalAgentWriterClient(dbPath); + await writer.commit("slow-recovery-start", { + threadId: THREAD_ID, turnId: TURN_ID, executionId: EXECUTION_ID, phase: "running", events: events(), + }); + db.run("CREATE TABLE writer_lock_probe (id TEXT PRIMARY KEY)"); + db.run(`CREATE TRIGGER slow_recovery BEFORE INSERT ON canonical_agent_items + WHEN NEW.id LIKE 'toolCall:slow-%' BEGIN SELECT randomblob(8000000); END`); + const input = { + executionId: EXECUTION_ID, + items: Array.from({ length: 80 }, (_, index) => recoveryToolCall(`slow-${index}`)), + }; + let settled = false; + const recovery = writer.recordParentNarrativeRecovery("slow-recovery", input) + .finally(() => { settled = true; }); + const count = db.prepare("SELECT COUNT(*) AS count FROM canonical_agent_items WHERE id LIKE 'toolCall:slow-%'"); + for (let attempts = 0; attempts < 500 && (count.get() as { count: number }).count === 0; attempts++) { + await new Promise((resolve) => setTimeout(resolve, 2)); + } + expect((count.get() as { count: number }).count).toBeGreaterThan(0); + expect(settled).toBe(false); + expect(db.prepare("SELECT COUNT(*) AS count FROM canonical_writer_operation_receipts WHERE operation_id = ?") + .get("slow-recovery")).toEqual({ count: 0 }); + const started = performance.now(); + db.prepare("INSERT INTO writer_lock_probe (id) VALUES (?)").run("main-write"); + const mainWriteMs = performance.now() - started; + expect(mainWriteMs).toBeLessThan(250); + await recovery; + expect(db.prepare("SELECT COUNT(*) AS count FROM canonical_writer_operation_receipts WHERE operation_id = ?") + .get("slow-recovery")).toEqual({ count: 1 }); + }, 30_000); + it("reports a missing execution without claiming recovery was recorded", async () => { writer = new CanonicalAgentWriterClient(dbPath); expect(await writer.recordParentNarrativeRecovery("missing-recovery", { diff --git a/apps/server/src/features/agents/canonical/canonical-agent-boundary.ts b/apps/server/src/features/agents/canonical/canonical-agent-boundary.ts index d3761df73..2360c92d9 100644 --- a/apps/server/src/features/agents/canonical/canonical-agent-boundary.ts +++ b/apps/server/src/features/agents/canonical/canonical-agent-boundary.ts @@ -57,6 +57,7 @@ import { ACTIVE_TURN_WRITE_BATCH_LIMITS, runBoundedWriteBatches, runBoundedWriteBatchesSync, + type RunBoundedWriteBatchesInput, type WriteBatchResult, } from "../../../runtime/persistence/sqlite/bounded-write-batches.js"; import { assertActiveTurnRecoveryRetention } from "../turns/active-turn-recovery-retention-policy.js"; @@ -198,6 +199,10 @@ export type CanonicalParentTurnFinishInput = ParentTurnFinishInput; /** Canonical alias for the structured parent recovery durability input. */ export type ParentNarrativeRecoveryCommitInput = ParentNarrativeRecoveryCommit; +type ParentNarrativeRecoveryOperation = + | { kind: "persist"; item: ParentNarrativeRecoveryItem } + | { kind: "discard"; itemId: string }; + export type { CanonicalChildTurnFinishInput, CodexChildDeliveryInput, @@ -1753,6 +1758,26 @@ export class CanonicalAgentBoundary implements ParentTurnDurability, CodexCollab return true; } + /** The writer worker yields between committed recovery batches so other SQLite writers can proceed. */ + async recordParentNarrativeRecoveryYielding( + input: ParentNarrativeRecoveryCommitInput, + ): Promise { + const turn = this.loadTurnByExecution(input.executionId); + if (!turn) return false; + const thread = this.loadThread(turn.threadId); + if (!thread) throw new Error(`Canonical parent thread not found: ${turn.threadId}`); + if (input.items.length === 0 && (input.discardedItemIds?.length ?? 0) === 0) return true; + const batch = this.parentNarrativeRecoveryBatchInput(input, thread, turn, new Date().toISOString()); + if (!batch) return true; + await runBoundedWriteBatches({ + ...batch, + beginImmediate: true, + // A macrotask-only yield can reacquire SQLite before another connection's busy waiter wakes. + yieldControl: () => new Promise((resolve) => setTimeout(resolve, 2)), + }); + return true; + } + /** Completes the canonical-to-display startup migration before provider recovery begins. */ async materializeConversationDisplay(): Promise { await this.displayMaterializer.runToCompletion(); @@ -1764,10 +1789,20 @@ export class CanonicalAgentBoundary implements ParentTurnDurability, CodexCollab turn: AgentTurn, now: string, ): void { + const batch = this.parentNarrativeRecoveryBatchInput(input, thread, turn, now); + if (batch) runBoundedWriteBatchesSync(batch); + } + + private parentNarrativeRecoveryBatchInput( + input: ParentNarrativeRecoveryCommitInput, + thread: AgentThread, + turn: AgentTurn, + now: string, + ): RunBoundedWriteBatchesInput | null { const checkpoint = this.loadCheckpoint(input.executionId); if (!checkpoint) throw new Error(`Canonical parent checkpoint was not found: ${input.executionId}`); - if (checkpoint.terminalOutcome) return; - const operations = [ + if (checkpoint.terminalOutcome) return null; + const operations: ParentNarrativeRecoveryOperation[] = [ ...input.items.map((item) => ({ kind: "persist" as const, item })), ...(input.discardedItemIds ?? []).map((itemId) => ({ kind: "discard" as const, itemId })), ]; @@ -1778,7 +1813,7 @@ export class CanonicalAgentBoundary implements ParentTurnDurability, CodexCollab operations.length, operations.reduce((total, operation) => total + byteLength(operation), 0), ); - runBoundedWriteBatchesSync({ + return { db: this.db, items: operations, limits: ACTIVE_TURN_WRITE_BATCH_LIMITS, @@ -1790,7 +1825,7 @@ export class CanonicalAgentBoundary implements ParentTurnDurability, CodexCollab } this.discardParentNarrativeRecoveryItem(operation.itemId, thread, turn); }, - }); + }; } private persistParentNarrativeRecoveryItem( diff --git a/apps/server/src/features/agents/canonical/canonical-agent-writer-receipts.ts b/apps/server/src/features/agents/canonical/canonical-agent-writer-receipts.ts index f991b30e9..2b773c788 100644 --- a/apps/server/src/features/agents/canonical/canonical-agent-writer-receipts.ts +++ b/apps/server/src/features/agents/canonical/canonical-agent-writer-receipts.ts @@ -13,6 +13,7 @@ import type { CanonicalWriterRequest, CanonicalWriterResponse } from "./canonica type WriteRequest = Extract; +type AtomicWriteRequest = Extract; type WriteResponse = Extract; @@ -55,29 +56,42 @@ export class CanonicalAgentWriterReceipts { this.orm = drizzle(db); } - execute(request: WriteRequest, apply: () => WriteResponse): WriteResponse { + execute(request: AtomicWriteRequest, apply: () => WriteResponse): WriteResponse { const inputHash = fingerprint(request); return this.db.transaction(() => { - const existing = this.orm.select().from(canonicalWriterOperationReceipts) - .where(and( - eq(canonicalWriterOperationReceipts.executionId, request.executionId), - eq(canonicalWriterOperationReceipts.operationId, request.operationId), - )).get(); - if (existing) { - if (existing.kind !== request.kind || existing.inputHash !== inputHash) { - throw new CanonicalWriterOperationConflict("Canonical writer operation ID was reused with different input"); - } - return this.replay(request, existing.receiptJson); - } - const outstanding = this.orm.select({ count: count() }).from(canonicalWriterOperationReceipts) - .where(eq(canonicalWriterOperationReceipts.executionId, request.executionId)).get()?.count ?? 0; - if (outstanding >= MAX_UNACKNOWLEDGED_RECEIPTS_PER_EXECUTION) { - throw new CanonicalWriterReceiptCapacity("Canonical writer receipt capacity reached"); - } + const existing = this.loadReceipt(request); + if (existing) return this.replayMatching(request, inputHash, existing); + this.assertCapacity(request.executionId); const response = apply(); - if (response.kind === "parent-narrative-recovery-recorded" && !response.receipt.recorded) { - return response; - } + this.orm.insert(canonicalWriterOperationReceipts).values({ + executionId: request.executionId, + operationId: request.operationId, + kind: request.kind, + inputHash, + receiptJson: compactReceipt(response), + }).run(); + return response; + })(); + } + + /** Recovery upserts are repeatable, so its bounded batches commit before a short final receipt transaction. */ + async executeBatchedRecovery( + request: Extract, + apply: () => Promise, + ): Promise { + const inputHash = fingerprint(request); + const existing = this.loadReceipt(request); + if (existing) return this.replayMatching(request, inputHash, existing); + this.assertCapacity(request.executionId); + const response = await apply(); + if (response.kind !== "parent-narrative-recovery-recorded") { + throw new Error("Canonical writer returned the wrong recovery response"); + } + if (!response.receipt.recorded) return response; + return this.db.transaction(() => { + const committed = this.loadReceipt(request); + if (committed) return this.replayMatching(request, inputHash, committed); + this.assertCapacity(request.executionId); this.orm.insert(canonicalWriterOperationReceipts).values({ executionId: request.executionId, operationId: request.operationId, @@ -97,6 +111,29 @@ export class CanonicalAgentWriterReceipts { )).run(); } + private loadReceipt(request: WriteRequest) { + return this.orm.select().from(canonicalWriterOperationReceipts) + .where(and( + eq(canonicalWriterOperationReceipts.executionId, request.executionId), + eq(canonicalWriterOperationReceipts.operationId, request.operationId), + )).get(); + } + + private replayMatching(request: WriteRequest, inputHash: string, existing: NonNullable>): WriteResponse { + if (existing.kind !== request.kind || existing.inputHash !== inputHash) { + throw new CanonicalWriterOperationConflict("Canonical writer operation ID was reused with different input"); + } + return this.replay(request, existing.receiptJson); + } + + private assertCapacity(executionId: string): void { + const outstanding = this.orm.select({ count: count() }).from(canonicalWriterOperationReceipts) + .where(eq(canonicalWriterOperationReceipts.executionId, executionId)).get()?.count ?? 0; + if (outstanding >= MAX_UNACKNOWLEDGED_RECEIPTS_PER_EXECUTION) { + throw new CanonicalWriterReceiptCapacity("Canonical writer receipt capacity reached"); + } + } + private replay(request: WriteRequest, receiptJson: string): WriteResponse { const stored = storedReceiptSchema.parse(JSON.parse(receiptJson)); const correlation = { diff --git a/apps/server/src/features/agents/canonical/canonical-agent-writer.worker.ts b/apps/server/src/features/agents/canonical/canonical-agent-writer.worker.ts index 556be5028..31abbed39 100644 --- a/apps/server/src/features/agents/canonical/canonical-agent-writer.worker.ts +++ b/apps/server/src/features/agents/canonical/canonical-agent-writer.worker.ts @@ -75,7 +75,7 @@ function eventMatchesRouting( && (event.routing.turnId === undefined || event.routing.turnId === input.turnId); } -function handle(request: CanonicalWriterRequest): CanonicalWriterResponse { +async function handle(request: CanonicalWriterRequest): Promise { const correlation = { requestId: request.requestId, operationId: request.operationId, @@ -109,13 +109,23 @@ function handle(request: CanonicalWriterRequest): CanonicalWriterResponse { return handleWrite(request, correlation); } -function handleWrite( +async function handleWrite( request: Extract, correlation: Pick, -): CanonicalWriterResponse { +): Promise { try { const canonical = boundary; if (!canonical || !receipts) throw new Error("Canonical writer has not opened its database"); + if (request.kind === "record-parent-narrative-recovery") { + if (request.input.executionId !== request.executionId) { + throw new Error("Canonical writer recovery execution mismatch"); + } + return await receipts.executeBatchedRecovery(request, async () => ({ + ...correlation, + kind: "parent-narrative-recovery-recorded", + receipt: { recorded: await canonical.recordParentNarrativeRecoveryYielding(request.input) }, + })); + } return receipts.execute(request, () => applyWrite(request, correlation, canonical)); } catch (error) { return { @@ -128,7 +138,7 @@ function handleWrite( } function applyWrite( - request: Extract, + request: Extract, correlation: Pick, canonical: CanonicalAgentBoundary, ): Extract { @@ -139,13 +149,6 @@ function applyWrite( const receipt = classifyParentNarrativeRecovery(request.input); return { ...correlation, kind: "parent-narrative-recovery-classified", receipt }; } - if (request.kind === "record-parent-narrative-recovery") { - if (request.input.executionId !== request.executionId) { - throw new Error("Canonical writer recovery execution mismatch"); - } - const recorded = canonical.recordParentNarrativeRecovery(request.input); - return { ...correlation, kind: "parent-narrative-recovery-recorded", receipt: { recorded } }; - } assertRouting(request.input, request.executionId); const result = canonical.commit(request.input); const { outcome, conversationRevision, rosterRevision, acceptedThrough, durableThrough, events } = result; @@ -156,8 +159,23 @@ function applyWrite( }; } +let requestTail = Promise.resolve(); globalThis.onmessage = (message: MessageEvent): void => { - globalThis.postMessage(handle(message.data)); + const request = message.data; + requestTail = requestTail.then(async () => { + try { + globalThis.postMessage(await handle(request)); + } catch (error) { + console.error("Canonical writer request failed unexpectedly", error); + globalThis.postMessage({ + requestId: request.requestId, + operationId: request.operationId, + executionId: request.executionId, + kind: "failed", + reason: "write-failed", + } satisfies CanonicalWriterResponse); + } + }); }; process.on("exit", () => db?.close(true)); diff --git a/apps/server/src/runtime/persistence/sqlite/bounded-write-batches.ts b/apps/server/src/runtime/persistence/sqlite/bounded-write-batches.ts index a6df31afd..92dc26e97 100644 --- a/apps/server/src/runtime/persistence/sqlite/bounded-write-batches.ts +++ b/apps/server/src/runtime/persistence/sqlite/bounded-write-batches.ts @@ -37,6 +37,8 @@ export interface RunBoundedWriteBatchesInput { onBatchStarted?: () => void; onBatchFinishing?: () => void; onBatchCommitted?: (result: WriteBatchResult) => void; + /** Acquire the write lock before a batch reads, avoiding failed read-to-write upgrades across connections. */ + beginImmediate?: boolean; } function assertPositiveLimit(value: number, name: keyof WriteBatchLimits): void { @@ -72,7 +74,8 @@ export async function runBoundedWriteBatches( batchRows = result.batchRows; input.onBatchFinishing?.(); }); - transaction(); + if (input.beginImmediate) transaction.immediate(); + else transaction(); batches += 1; totalBytes += batchBytes; totalRows += batchRows; @@ -110,7 +113,8 @@ export function runBoundedWriteBatchesSync( batchRows = result.batchRows; input.onBatchFinishing?.(); }); - transaction(); + if (input.beginImmediate) transaction.immediate(); + else transaction(); batches += 1; totalBytes += batchBytes; totalRows += batchRows; From 6873f2345b555a8f2a803265d5dc2ba3e73f3655 Mon Sep 17 00:00:00 2001 From: "(CJ) Chukwudi Nwobodo" <142016413+chuks-qua@users.noreply.github.com> Date: Thu, 24 Sep 2026 20:27:05 +0100 Subject: [PATCH 031/136] Revert "perf(server): avoid main thread recovery lookup" This reverts commit 8f9fc3663c0ce753ddf54544549e5d3d2e3747e4. --- .../agent-service-narrative-persist.test.ts | 4 +-- .../__tests__/agent-service-test-harness.ts | 1 - ...ent-narrative-recovery-coordinator.test.ts | 32 ++++++------------- .../parent-narrative-recovery-coordinator.ts | 17 +++++++--- .../turns/provider-turn-event-application.ts | 2 +- 5 files changed, 24 insertions(+), 32 deletions(-) diff --git a/apps/server/src/features/agents/orchestration/__tests__/agent-service-narrative-persist.test.ts b/apps/server/src/features/agents/orchestration/__tests__/agent-service-narrative-persist.test.ts index b2608718a..0179e8e25 100644 --- a/apps/server/src/features/agents/orchestration/__tests__/agent-service-narrative-persist.test.ts +++ b/apps/server/src/features/agents/orchestration/__tests__/agent-service-narrative-persist.test.ts @@ -1218,13 +1218,13 @@ describe("AgentService narrative persistence", () => { }); await waitForAgentServiceIngressForTest(service, THREAD_ID); - await vi.waitFor(() => expect(taskAppend).toHaveBeenCalledWith(THREAD_ID, { + expect(taskAppend).toHaveBeenCalledWith(THREAD_ID, { id: "1", content: "Buy groceries - Pick up milk, eggs, bread", status: "pending", activeForm: "Buying groceries", group: "Tasks", - })); + }); }); it("does not persist a TaskCreate whose result errored", () => { diff --git a/apps/server/src/features/agents/orchestration/__tests__/agent-service-test-harness.ts b/apps/server/src/features/agents/orchestration/__tests__/agent-service-test-harness.ts index 5d6fbf70b..91a05a6cd 100644 --- a/apps/server/src/features/agents/orchestration/__tests__/agent-service-test-harness.ts +++ b/apps/server/src/features/agents/orchestration/__tests__/agent-service-test-harness.ts @@ -359,7 +359,6 @@ function narrativeWriterForTest( ): ParentNarrativeRecoveryWriter { return override ?? { async recordParentNarrativeRecovery(_operationId, input) { - if (!parentDurability.loadTurnByExecution(input.executionId)) return { recorded: false }; return { recorded: parentDurability.recordParentNarrativeRecovery(input) }; }, async classifyParentNarrativeRecovery(_operationId, input) { diff --git a/apps/server/src/features/agents/turns/__tests__/parent-narrative-recovery-coordinator.test.ts b/apps/server/src/features/agents/turns/__tests__/parent-narrative-recovery-coordinator.test.ts index 6a9e6e020..cc3180f20 100644 --- a/apps/server/src/features/agents/turns/__tests__/parent-narrative-recovery-coordinator.test.ts +++ b/apps/server/src/features/agents/turns/__tests__/parent-narrative-recovery-coordinator.test.ts @@ -9,6 +9,7 @@ import { type ParentNarrativeRecoveryWriter, } from "../parent-narrative-recovery-coordinator.js"; import type { NarrativeStore } from "../../conversation/narrative/narrative-store.js"; +import type { ParentTurnDurability } from "../parent-turn-durability.js"; const EXECUTION_ID = "execution-1"; const THREAD_ID = "thread-1"; @@ -36,13 +37,16 @@ function coordinatorFor(writer: ParentNarrativeRecoveryWriter): ParentNarrativeR const narrativeStore = { recoverySnapshot: vi.fn(() => recoveryItems), } as unknown as NarrativeStore; - return new ParentNarrativeRecoveryCoordinator(writer, narrativeStore); + const durability = { + loadTurnByExecution: vi.fn(() => ({ id: "turn-1" })), + } as unknown as ParentTurnDurability; + return new ParentNarrativeRecoveryCoordinator(writer, narrativeStore, durability); } describe("ParentNarrativeRecoveryCoordinator", () => { it("retries an uncommitted recovery snapshot with the same operation identity", async () => { const recordParentNarrativeRecovery = vi.fn() - .mockRejectedValueOnce(new Error("writer unavailable")) + .mockResolvedValueOnce({ recorded: false }) .mockResolvedValue({ recorded: true }); const writer: ParentNarrativeRecoveryWriter = { recordParentNarrativeRecovery, @@ -56,7 +60,9 @@ describe("ParentNarrativeRecoveryCoordinator", () => { discardedItemIds: [], }; - await expect(coordinator.checkpoint(event)).rejects.toThrow("writer unavailable"); + await expect(coordinator.checkpoint(event)).rejects.toThrow( + `Canonical parent turn was not found: ${EXECUTION_ID}`, + ); await coordinator.checkpoint(event); await coordinator.checkpoint(event); @@ -70,26 +76,6 @@ describe("ParentNarrativeRecoveryCoordinator", () => { await vi.waitFor(() => expect(writer.acknowledgeOperation).toHaveBeenCalledWith(EXECUTION_ID, operationId)); }); - it("treats a missing execution as a normal receipt and retries its snapshot for later events", async () => { - const recordParentNarrativeRecovery = vi.fn() - .mockResolvedValueOnce({ recorded: false }) - .mockResolvedValue({ recorded: true }); - const writer: ParentNarrativeRecoveryWriter = { - recordParentNarrativeRecovery, - classifyParentNarrativeRecovery: vi.fn(), - acknowledgeOperation: vi.fn(async () => undefined), - }; - const coordinator = coordinatorFor(writer); - await coordinator.checkpoint(event); - coordinator.acknowledgeHandled(event); - const laterEvent = { ...event }; - await coordinator.checkpoint(laterEvent); - - expect(recordParentNarrativeRecovery).toHaveBeenCalledTimes(2); - expect(recordParentNarrativeRecovery.mock.calls[1]?.[1]).toEqual(recordParentNarrativeRecovery.mock.calls[0]?.[1]); - expect(recordParentNarrativeRecovery.mock.calls[1]?.[0]).not.toBe(recordParentNarrativeRecovery.mock.calls[0]?.[0]); - }); - it("retries the atomic narration classification without changing its input", async () => { const classifyParentNarrativeRecovery = vi.fn() .mockRejectedValueOnce(new Error("writer unavailable")) diff --git a/apps/server/src/features/agents/turns/parent-narrative-recovery-coordinator.ts b/apps/server/src/features/agents/turns/parent-narrative-recovery-coordinator.ts index a6952b6d2..e08a4d9c8 100644 --- a/apps/server/src/features/agents/turns/parent-narrative-recovery-coordinator.ts +++ b/apps/server/src/features/agents/turns/parent-narrative-recovery-coordinator.ts @@ -8,6 +8,7 @@ import { import type { NarrativeStore } from "../conversation/narrative/narrative-store.js"; import type { CanonicalAgentWriterClient } from "../canonical/canonical-agent-writer-client.js"; import type { ParentNarrativeRecoveryCommit } from "./parent-turn-durability.js"; +import type { ParentTurnDurability } from "./parent-turn-durability.js"; import { serverWorkTrace } from "../diagnostics/server-work-trace.js"; /** The single acknowledged writer used for recovery and text classification. */ @@ -26,10 +27,12 @@ interface ParentNarrativeRecoveryCheckpoint { export class ParentNarrativeRecoveryCoordinator { private readonly fingerprintsByExecution = new Map>(); private readonly preparedByEvent = new WeakMap(); + private readonly existingExecutions = new Set(); constructor( private readonly writer: ParentNarrativeRecoveryWriter, private readonly narrativeStore: NarrativeStore, + private readonly canonicalSink: ParentTurnDurability, ) {} /** Commit only the semantic records changed by this accepted provider event. */ @@ -41,11 +44,10 @@ export class ParentNarrativeRecoveryCoordinator { ? serverWorkTrace.measure("narrative-persist", event.threadId, event.turnExecutionId, persist) : persist(); return pending.then((receipt) => { - if (receipt.recorded) { - if (serverWorkTrace) { - serverWorkTrace.measure("narrative-confirm", event.threadId, event.turnExecutionId, checkpoint.confirm); - } else checkpoint.confirm(); - } + if (!receipt.recorded) throw new Error(`Canonical parent turn was not found: ${event.turnExecutionId}`); + if (serverWorkTrace) { + serverWorkTrace.measure("narrative-confirm", event.threadId, event.turnExecutionId, checkpoint.confirm); + } else checkpoint.confirm(); checkpoint.committed = true; }); } @@ -82,6 +84,10 @@ export class ParentNarrativeRecoveryCoordinator { force = false, ): ParentNarrativeRecoveryCheckpoint | null { if (!this.requiresStructuredRecovery(event) || !event.turnExecutionId) return null; + if (!force && !this.existingExecutions.has(event.turnExecutionId)) { + if (!this.canonicalSink.loadTurnByExecution(event.turnExecutionId)) return null; + this.existingExecutions.add(event.turnExecutionId); + } const existing = this.preparedByEvent.get(event as object); if (existing) return existing; const executionId = event.turnExecutionId; @@ -125,6 +131,7 @@ export class ParentNarrativeRecoveryCoordinator { clear(executionId: string | undefined): void { if (!executionId) return; this.fingerprintsByExecution.delete(executionId); + this.existingExecutions.delete(executionId); } private requiresStructuredRecovery(event: AgentEvent): boolean { diff --git a/apps/server/src/features/agents/turns/provider-turn-event-application.ts b/apps/server/src/features/agents/turns/provider-turn-event-application.ts index 7df4cda68..a96548302 100644 --- a/apps/server/src/features/agents/turns/provider-turn-event-application.ts +++ b/apps/server/src/features/agents/turns/provider-turn-event-application.ts @@ -95,7 +95,7 @@ export class ProviderTurnEventApplication implements TurnEventApplication { } }, ); - this.parentNarrativeRecovery = new ParentNarrativeRecoveryCoordinator(narrativeWriter, narrative); + this.parentNarrativeRecovery = new ParentNarrativeRecoveryCoordinator(narrativeWriter, narrative, parentDurability); this.browserNarrativeEventSanitizer = new BrowserNarrativeEventSanitizer( (threadId, toolCallId) => this.narrative.getBufferedToolCalls(threadId) .find((toolCall) => toolCall.toolCallId === toolCallId) From 2ff986a62a7eaa30faf9b9299262c38b811a21ff Mon Sep 17 00:00:00 2001 From: "(CJ) Chukwudi Nwobodo" <142016413+chuks-qua@users.noreply.github.com> Date: Thu, 24 Sep 2026 20:27:08 +0100 Subject: [PATCH 032/136] Revert "fix(server): retain provider diagnostics after writer commits" This reverts commit e410cb9387918de0ab2f4c46bea5b35580611160. --- .../canonical/canonical-agent-boundary.ts | 5 -- .../canonical-agent-diagnostics.test.ts | 28 ----------- .../canonical-agent-diagnostics.ts | 47 +++---------------- .../provider-composition-container.test.ts | 16 ------- .../__tests__/provider-host-ports.test.ts | 9 +--- .../composition/provider-host-ports.ts | 4 +- .../composition/register-providers.ts | 3 +- 7 files changed, 9 insertions(+), 103 deletions(-) diff --git a/apps/server/src/features/agents/canonical/canonical-agent-boundary.ts b/apps/server/src/features/agents/canonical/canonical-agent-boundary.ts index 2360c92d9..72acd8995 100644 --- a/apps/server/src/features/agents/canonical/canonical-agent-boundary.ts +++ b/apps/server/src/features/agents/canonical/canonical-agent-boundary.ts @@ -492,11 +492,6 @@ export class CanonicalAgentBoundary implements ParentTurnDurability, CodexCollab : this.eventStore.commit(this.toEventStoreCommitInput(input)); } - /** Retain diagnostics on the server after a provider writer acknowledges its durable events. */ - recordProviderCommitDiagnostics(events: readonly CanonicalAgentEventEnvelope[]): void { - this.recordCanonicalDiagnostics(events); - } - private commitInsideTransaction(input: CanonicalAgentCommitInput): CanonicalAgentCommitResult { return serverWorkTrace ? serverWorkTrace.measure("canonical-write", input.threadId, input.executionId, diff --git a/apps/server/src/features/agents/observability/__tests__/canonical-agent-diagnostics.test.ts b/apps/server/src/features/agents/observability/__tests__/canonical-agent-diagnostics.test.ts index 7928c580c..282854d8f 100644 --- a/apps/server/src/features/agents/observability/__tests__/canonical-agent-diagnostics.test.ts +++ b/apps/server/src/features/agents/observability/__tests__/canonical-agent-diagnostics.test.ts @@ -90,32 +90,4 @@ describe("CanonicalAgentDiagnostics", () => { confirmRaw: true, }).rawEvents).toEqual([]); }); - - it("records a replayed canonical event once while raw capture retains it", () => { - const diagnostics = new CanonicalAgentDiagnostics(); - diagnostics.startRawCapture({ turnId: "turn-1", consent: true, expiresInMs: 60_000 }); - const event = { eventId: "canonical-1", payload: { type: "item.recorded", content: "answer" } }; - const input = { turnId: "turn-1", executionId: "execution-1", source: "canonical" as const, event }; - diagnostics.record(input); - for (let index = 0; index < CANONICAL_DIAGNOSTIC_RING_CAPACITY; index += 1) { - diagnostics.record({ - turnId: "other-turn", - executionId: "other-execution", - source: "provider", - event: { type: "textDelta", delta: `other-${index}` }, - }); - } - diagnostics.record(input); - diagnostics.record({ - turnId: "turn-2", - executionId: "execution-2", - source: "canonical", - event, - }); - - const exported = diagnostics.exportTurn("turn-1", { includeRaw: true, confirmRaw: true }); - expect(exported.rawEvents).toEqual([event]); - expect(exported.truncation).toEqual({ droppedEntries: 1 }); - expect(diagnostics.exportTurn("turn-2").entries).toHaveLength(1); - }); }); diff --git a/apps/server/src/features/agents/observability/canonical-agent-diagnostics.ts b/apps/server/src/features/agents/observability/canonical-agent-diagnostics.ts index a22cb2760..8aca5e236 100644 --- a/apps/server/src/features/agents/observability/canonical-agent-diagnostics.ts +++ b/apps/server/src/features/agents/observability/canonical-agent-diagnostics.ts @@ -66,15 +66,6 @@ function eventType(event: unknown): string { return typeof event.type === "string" ? event.type : "unknown"; } -function canonicalEventId(event: unknown): string | undefined { - if (!event || typeof event !== "object" || !("eventId" in event)) return undefined; - return typeof event.eventId === "string" ? event.eventId : undefined; -} - -function canonicalEventKey(turnId: string, eventId: string): string { - return JSON.stringify([turnId, eventId]); -} - interface ContentMeasureState { bytes: number; visited: number; @@ -171,8 +162,6 @@ function serializeRawEvent(event: unknown): { event: unknown; bytes: number } | /** Retains bounded redacted diagnostics and consent-scoped raw turn captures. */ export class CanonicalAgentDiagnostics { private readonly entries: CanonicalDiagnosticEntry[] = []; - private readonly canonicalEventIds = new Set(); - private readonly canonicalEventIdByEntry = new WeakMap(); private readonly droppedByTurn = new Map(); private readonly rawCaptures = new Map(); @@ -202,8 +191,6 @@ export class CanonicalAgentDiagnostics { /** Record one event without retaining content unless raw capture is active. */ record(input: CanonicalDiagnosticRecordInput): void { - const eventId = input.source === "canonical" ? canonicalEventId(input.event) : undefined; - if (eventId && this.hasRecordedCanonicalEvent(input.turnId, eventId)) return; const measuredContent = measureContent(input.event); const entry: CanonicalDiagnosticEntry = { turnId: input.turnId, @@ -216,13 +203,13 @@ export class CanonicalAgentDiagnostics { recordedAt: new Date(this.now()).toISOString(), }; this.entries.push(entry); - this.rememberCanonicalEvent(entry, eventId); - this.evictOldestEntryIfNeeded(); - - this.recordRawCapture(input); - } + if (this.entries.length > CANONICAL_DIAGNOSTIC_RING_CAPACITY) { + const [dropped] = this.entries.splice(0, 1); + if (dropped) { + this.incrementDroppedEntries(dropped.turnId); + } + } - private recordRawCapture(input: CanonicalDiagnosticRecordInput): void { const raw = this.rawCaptures.get(input.turnId); if (!raw) return; if (raw.expiresAt <= this.now()) { @@ -243,28 +230,6 @@ export class CanonicalAgentDiagnostics { if (input.terminal) raw.active = false; } - private hasRecordedCanonicalEvent(turnId: string, eventId: string): boolean { - if (this.canonicalEventIds.has(canonicalEventKey(turnId, eventId))) return true; - const raw = this.rawCaptures.get(turnId); - return raw?.events.some((event) => canonicalEventId(event) === eventId) ?? false; - } - - private rememberCanonicalEvent(entry: CanonicalDiagnosticEntry, eventId: string | undefined): void { - if (!eventId) return; - const key = canonicalEventKey(entry.turnId, eventId); - this.canonicalEventIds.add(key); - this.canonicalEventIdByEntry.set(entry, key); - } - - private evictOldestEntryIfNeeded(): void { - if (this.entries.length <= CANONICAL_DIAGNOSTIC_RING_CAPACITY) return; - const dropped = this.entries.shift(); - if (!dropped) return; - const eventId = this.canonicalEventIdByEntry.get(dropped); - if (eventId) this.canonicalEventIds.delete(eventId); - this.incrementDroppedEntries(dropped.turnId); - } - /** Export redacted diagnostics and, after separate confirmation, raw content. */ exportTurn( turnId: string, diff --git a/apps/server/src/features/providers/composition/__tests__/provider-composition-container.test.ts b/apps/server/src/features/providers/composition/__tests__/provider-composition-container.test.ts index d41c1ecd6..405c47486 100644 --- a/apps/server/src/features/providers/composition/__tests__/provider-composition-container.test.ts +++ b/apps/server/src/features/providers/composition/__tests__/provider-composition-container.test.ts @@ -130,16 +130,6 @@ describe("provider composition container", () => { providerIdentities: [], projectUserMessage: () => messages.create("thread-1", "user", "Start", 1), }); - const baselineDiagnostics = canonical.exportTurnDiagnostics("turn-1").entries.length; - canonical.startRawTurnCapture({ turnId: "turn-1", consent: true, expiresInMs: 60_000 }); - const writer = container.resolve(CanonicalAgentWriterClient); - const commit = writer.commit.bind(writer); - let firstReceipt: Awaited> | undefined; - vi.spyOn(writer, "commit").mockImplementation(async (operationId, input) => { - if (firstReceipt) return firstReceipt; - firstReceipt = await commit(operationId, input); - return firstReceipt; - }); await expect(host.events.submit(runtimeBatch())).resolves.toMatchObject({ commit: { outcome: "committed", eventCount: 1 }, @@ -153,12 +143,6 @@ describe("provider composition container", () => { canonicalReceipt: expect.objectContaining({ eventId: "cursor:runtime-event-1" }), })]); }); - await host.events.submit(runtimeBatch()); - const diagnostics = canonical.exportTurnDiagnostics("turn-1", { includeRaw: true, confirmRaw: true }); - expect(diagnostics.entries).toHaveLength(baselineDiagnostics + 1); - expect(diagnostics.rawEvents).toEqual([ - expect.objectContaining({ eventId: "cursor:runtime-event-1" }), - ]); }); it("waits for provider cleanup even when another provider shutdown fails", async () => { diff --git a/apps/server/src/features/providers/composition/__tests__/provider-host-ports.test.ts b/apps/server/src/features/providers/composition/__tests__/provider-host-ports.test.ts index d98c03422..ab7af0d2c 100644 --- a/apps/server/src/features/providers/composition/__tests__/provider-host-ports.test.ts +++ b/apps/server/src/features/providers/composition/__tests__/provider-host-ports.test.ts @@ -54,7 +54,6 @@ describe("createProviderHostPorts", () => { threadControl: {}, grants: {}, events: {}, - diagnostics: {}, publishCanonicalEvents: vi.fn(), ingress: {}, } as never); @@ -83,7 +82,6 @@ describe("createProviderHostPorts", () => { events, }; }); - const recordProviderCommitDiagnostics = vi.fn(() => deliveryOrder.push("diagnostics")); const publishCanonicalEvents = vi.fn(() => deliveryOrder.push("publication")); const acceptCommitted = vi.fn(() => deliveryOrder.push("ingress")); const acknowledgeOperation = vi.fn(async () => { deliveryOrder.push("acknowledge"); }); @@ -95,7 +93,6 @@ describe("createProviderHostPorts", () => { threadControl: {}, grants: {}, events: { commit, acknowledgeOperation }, - diagnostics: { recordProviderCommitDiagnostics }, publishCanonicalEvents, ingress: { acceptCommitted }, } as never); @@ -113,11 +110,10 @@ describe("createProviderHostPorts", () => { delivery: { ingress: "queued" }, }); expect(commit).toHaveBeenCalledWith(expect.any(String), { ...batch, nativeCursor: undefined }); - expect(recordProviderCommitDiagnostics).toHaveBeenCalledWith(events); expect(publishCanonicalEvents).toHaveBeenCalledWith(events); expect(acceptCommitted).toHaveBeenCalledWith(events); expect(acknowledgeOperation).toHaveBeenCalledWith(EXECUTION_ID, expect.any(String)); - expect(deliveryOrder).toEqual(["commit", "diagnostics", "publication", "ingress", "acknowledge"]); + expect(deliveryOrder).toEqual(["commit", "publication", "ingress", "acknowledge"]); }); it("does not hand duplicate or failed commits to ingress", async () => { @@ -142,7 +138,6 @@ describe("createProviderHostPorts", () => { threadControl: {}, grants: {}, events: { commit, acknowledgeOperation: vi.fn() }, - diagnostics: { recordProviderCommitDiagnostics: vi.fn() }, publishCanonicalEvents: vi.fn(), ingress: { acceptCommitted }, } as never); @@ -173,7 +168,6 @@ describe("createProviderHostPorts", () => { threadControl: {}, grants: {}, events: { commit, acknowledgeOperation: vi.fn() }, - diagnostics: { recordProviderCommitDiagnostics: vi.fn() }, publishCanonicalEvents: vi.fn(), ingress: { acceptCommitted: vi.fn() }, } as never); @@ -209,7 +203,6 @@ describe("createProviderHostPorts", () => { })), acknowledgeOperation, }, - diagnostics: { recordProviderCommitDiagnostics: vi.fn() }, publishCanonicalEvents: () => { throw new Error("push unavailable"); }, ingress: { acceptCommitted }, } as never); diff --git a/apps/server/src/features/providers/composition/provider-host-ports.ts b/apps/server/src/features/providers/composition/provider-host-ports.ts index 6ea42d79b..dc37d7b7e 100644 --- a/apps/server/src/features/providers/composition/provider-host-ports.ts +++ b/apps/server/src/features/providers/composition/provider-host-ports.ts @@ -5,7 +5,7 @@ import { logger } from "@mcode/shared"; import type { JobObject } from "../../../runtime/process/containment/job-object.js"; import type { EnvService } from "../../../runtime/environment/env-service.js"; import type { ScopedPreGrantService } from "../../agents/permissions/scoped-pre-grant.js"; -import type { CanonicalAgentBoundary, CanonicalAgentEventPublisher } from "../../agents/canonical/canonical-agent-boundary.js"; +import type { CanonicalAgentEventPublisher } from "../../agents/canonical/canonical-agent-boundary.js"; import type { CanonicalAgentWriterClient } from "../../agents/canonical/canonical-agent-writer-client.js"; import type { BrowserAutomationSessionLease } from "../../browser-automation/index.js"; import type { InternalThreadControlMcpRuntime } from "../../thread-control/index.js"; @@ -21,7 +21,6 @@ export interface ProviderHostPortDependencies { threadControl: InternalThreadControlMcpRuntime; grants: ScopedPreGrantService; events: Pick; - diagnostics: Pick; publishCanonicalEvents: CanonicalAgentEventPublisher; ingress: ProviderEventIngress; } @@ -106,7 +105,6 @@ export function createProviderHostPorts( }); let published = true; if (result.outcome === "committed" && result.events.length > 0) { - dependencies.diagnostics.recordProviderCommitDiagnostics(result.events); try { dependencies.publishCanonicalEvents(result.events); } catch { diff --git a/apps/server/src/features/providers/composition/register-providers.ts b/apps/server/src/features/providers/composition/register-providers.ts index afb5d236c..7ad724a34 100644 --- a/apps/server/src/features/providers/composition/register-providers.ts +++ b/apps/server/src/features/providers/composition/register-providers.ts @@ -9,7 +9,7 @@ import { ProviderRegistry } from "./provider-registry.js"; import { createProviderHostPorts } from "./provider-host-ports.js"; import { BrowserAutomationSessionLease } from "../../browser-automation/index.js"; import { InternalThreadControlMcpRuntime } from "../../thread-control/index.js"; -import { CanonicalAgentBoundary, publishCanonicalAgentEvents } from "../../agents/canonical/canonical-agent-boundary.js"; +import { publishCanonicalAgentEvents } from "../../agents/canonical/canonical-agent-boundary.js"; import { CanonicalAgentWriterClient } from "../../agents/canonical/canonical-agent-writer-client.js"; import { ScopedPreGrantService } from "../../agents/permissions/scoped-pre-grant.js"; import { EnvService } from "../../../runtime/environment/env-service.js"; @@ -94,7 +94,6 @@ export function registerProviderAdapters(container: DependencyContainer): void { threadControl: c.resolve(InternalThreadControlMcpRuntime), grants: c.resolve(ScopedPreGrantService), events: c.resolve(CanonicalAgentWriterClient), - diagnostics: c.resolve(CanonicalAgentBoundary), publishCanonicalEvents: publishCanonicalAgentEvents, ingress: c.resolve(ProviderEventIngress), }), From 783bff63c762e1ea03df8531245b6a9b4d278314 Mon Sep 17 00:00:00 2001 From: "(CJ) Chukwudi Nwobodo" <142016413+chuks-qua@users.noreply.github.com> Date: Thu, 24 Sep 2026 20:27:39 +0100 Subject: [PATCH 033/136] Revert "feat(server): checkpoint parent narration through writer receipts" This reverts commit aea91851041e64a2654f86a0524c41470d2c9b9e. --- .../agents/composition/register-agents.ts | 8 - .../agent-service-narrative-persist.test.ts | 156 +++------------ .../__tests__/agent-service-test-harness.ts | 31 --- ...ent-narrative-recovery-coordinator.test.ts | 90 ++------- .../parent-narrative-recovery-coordinator.ts | 131 ++++-------- .../turns/provider-turn-event-application.ts | 189 +++++------------- 6 files changed, 137 insertions(+), 468 deletions(-) diff --git a/apps/server/src/features/agents/composition/register-agents.ts b/apps/server/src/features/agents/composition/register-agents.ts index dece995d3..1b7dca5e8 100644 --- a/apps/server/src/features/agents/composition/register-agents.ts +++ b/apps/server/src/features/agents/composition/register-agents.ts @@ -75,11 +75,6 @@ import { ThreadCreationCoordinator } from "../turns/thread-creation-coordinator. import { ThreadStartupService } from "../../thread-startup/thread-startup-service.js"; import { ProviderSessionCursorPersistence } from "../turns/provider-session-cursor-persistence.js"; import { ProviderTurnEventApplication } from "../turns/provider-turn-event-application.js"; -import { CanonicalAgentWriterClient } from "../canonical/canonical-agent-writer-client.js"; -import { - PARENT_NARRATIVE_RECOVERY_WRITER, - type ParentNarrativeRecoveryWriter, -} from "../turns/parent-narrative-recovery-coordinator.js"; import { TURN_RUNTIME_EVENT_CONTROL, type TurnRuntimeEventControl, @@ -247,9 +242,6 @@ export function registerAgentServices(container: DependencyContainer): void { container.register(TURN_RUNTIME_EVENT_CONTROL, { useFactory: (c) => c.resolve(TurnRuntimeController), }); - container.register(PARENT_NARRATIVE_RECOVERY_WRITER, { - useFactory: (c) => c.resolve(CanonicalAgentWriterClient), - }); container.register( ProviderTurnEventApplication, { useClass: ProviderTurnEventApplication }, diff --git a/apps/server/src/features/agents/orchestration/__tests__/agent-service-narrative-persist.test.ts b/apps/server/src/features/agents/orchestration/__tests__/agent-service-narrative-persist.test.ts index 0179e8e25..1557663c3 100644 --- a/apps/server/src/features/agents/orchestration/__tests__/agent-service-narrative-persist.test.ts +++ b/apps/server/src/features/agents/orchestration/__tests__/agent-service-narrative-persist.test.ts @@ -79,7 +79,6 @@ function providerRuntimeEventForNarrativeTest(eventName: string, event: unknown) import type { PlanQuestionAnswersRepo } from "../../planning/persistence/plan-question-answers-repo.js"; import { CodexCollaborationEventAdapter } from "../../collaboration/adapters/codex-collaboration-event-adapter.js"; import { ProviderEventIngress } from "../../../providers/composition/provider-event-ingress.js"; -import type { ParentNarrativeRecoveryWriter } from "../../turns/parent-narrative-recovery-coordinator.js"; vi.mock("../../../../application/transport/push.js", () => ({ broadcast: vi.fn() })); vi.mock("fs", async (importOriginal) => { @@ -165,7 +164,6 @@ function build(options: { messageRepo?: MessageRepo; parentAssistantTextCheckpoints?: ParentAssistantTextCheckpointService; onProviderEvent?: (event: AgentEvent) => void; - narrativeWriter?: ParentNarrativeRecoveryWriter; } = {}): Built { const thread = makeThread(); const providerEmitter = Object.assign(new NodeEvents.EventEmitter(), { @@ -318,10 +316,6 @@ function build(options: { undefined, undefined, new TaskPersistenceService(taskRepo, narrativeStore), - undefined, - undefined, - undefined, - options.narrativeWriter, ); startAgentServiceIngressForTest(service, options.onProviderEvent); // Provider adapters always stamp turn-scoped events with the active execution @@ -364,105 +358,11 @@ function build(options: { }; } -function buildCommittedNarrativeTurn( - writerFor: (sink: CanonicalAgentEventSink) => ParentNarrativeRecoveryWriter, - onProviderEvent: (event: AgentEvent) => void, -): Built { - const db = openMemoryDatabase(); - const now = "2026-08-24T10:00:00.000Z"; - db.prepare( - "INSERT INTO workspaces (id, name, path, created_at, updated_at) VALUES (?, ?, ?, ?, ?)", - ).run("ws-1", "Workspace", "/workspace", now, now); - db.prepare( - "INSERT INTO threads (id, workspace_id, title, branch, provider, status, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", - ).run(THREAD_ID, "ws-1", "Parent", "main", "claude", "active", now, now); - const canonicalSink = new CanonicalAgentEventSink(db, vi.fn()); - const built = build({ db, canonicalSink, onProviderEvent, narrativeWriter: writerFor(canonicalSink) }); - canonicalSink.startParentTurn({ - thread: { id: THREAD_ID, workspaceId: "ws-1", providerId: "claude", createdAt: now }, - turnId: "turn-async-narrative", - executionId: narrativeExecutionId(built.service), - permissionMode: "supervised", - providerIdentities: [], - projectUserMessage: () => new SqliteMessageRepo(db).create(THREAD_ID, "user", "start", 1), - }); - return built; -} - describe("AgentService narrative persistence", () => { beforeEach(() => { vi.clearAllMocks(); }); - it("waits for a narrative writer acknowledgement before publishing and finalizing Stop", async () => { - let releaseWrite!: () => void; - const writeGate = new Promise((resolve) => { releaseWrite = resolve; }); - const published: AgentEvent[] = []; - const { db, service, providerEmitter, canonicalSink } = buildCommittedNarrativeTurn((sink) => ({ - async recordParentNarrativeRecovery(_operationId, input) { - await writeGate; - return { recorded: sink.recordParentNarrativeRecovery(input) }; - }, - async classifyParentNarrativeRecovery() { throw new Error("Unexpected classification"); }, - async acknowledgeOperation() {}, - }), (event) => published.push(event)); - const finalize = vi.spyOn(finalizerForAgentServiceTest(service), "finalize"); - try { - providerEmitter.emit("event", { - type: AgentEventType.TextDelta, - threadId: THREAD_ID, - turnExecutionId: narrativeExecutionId(service), - isFinalResponse: false, - delta: "awaited thought", - }); - await waitForAgentServiceIngressForTest(service, THREAD_ID); - await service.stopSession(THREAD_ID); - - expect(published).toEqual([]); - expect(finalize).not.toHaveBeenCalled(); - releaseWrite(); - await vi.waitFor(() => expect(finalize).toHaveBeenCalledOnce()); - expect(published.filter((event) => event.type === AgentEventType.TextDelta)).toHaveLength(1); - expect(canonicalSink.loadParentNarrativeRecovery("turn-async-narrative")).toHaveLength(1); - } finally { - db.close(); - } - }); - - it("does not finalize a stopped execution after its narrative write fails", async () => { - let failWrite!: (error: Error) => void; - const writeGate = new Promise((_resolve, reject) => { failWrite = reject; }); - const published: AgentEvent[] = []; - const { db, service, providerEmitter } = buildCommittedNarrativeTurn(() => ({ - async recordParentNarrativeRecovery() { - await writeGate; - return { recorded: true }; - }, - async classifyParentNarrativeRecovery() { throw new Error("Unexpected classification"); }, - async acknowledgeOperation() {}, - }), (event) => published.push(event)); - const finalize = vi.spyOn(finalizerForAgentServiceTest(service), "finalize"); - try { - providerEmitter.emit("event", { - type: AgentEventType.TextDelta, - threadId: THREAD_ID, - turnExecutionId: narrativeExecutionId(service), - isFinalResponse: false, - delta: "uncommitted thought", - }); - await waitForAgentServiceIngressForTest(service, THREAD_ID); - failWrite(new Error("injected writer failure")); - await vi.waitFor(() => expect(providerEmitter.stopSession).toHaveBeenCalled()); - await service.stopSession(THREAD_ID); - await Promise.resolve(); - - expect(published).toEqual([]); - expect(finalize).not.toHaveBeenCalled(); - } finally { - db.close(); - } - }); - it("moves unclassified text to narration and clears its assistant checkpoint at a false boundary", async () => { const db = openMemoryDatabase(); const now = "2026-08-24T10:00:00.000Z"; @@ -518,17 +418,15 @@ describe("AgentService narrative persistence", () => { }); await waitForAgentServiceIngressForTest(service, THREAD_ID); - await vi.waitFor(() => { - expect(published - .filter((event) => event.type === AgentEventType.TextDelta) - .map((event) => event.delta)) - .toEqual(["durable ", "text"]); - expect(db.prepare(` - SELECT first_sequence, last_sequence, text - FROM parent_assistant_text_checkpoint_chunks - WHERE execution_id = ? - `).all(executionId)).toEqual([]); - }); + expect(published + .filter((event) => event.type === AgentEventType.TextDelta) + .map((event) => event.delta)) + .toEqual(["durable ", "text"]); + expect(db.prepare(` + SELECT first_sequence, last_sequence, text + FROM parent_assistant_text_checkpoint_chunks + WHERE execution_id = ? + `).all(executionId)).toEqual([]); } finally { vi.useRealTimers(); db.close(); @@ -583,14 +481,12 @@ describe("AgentService narrative persistence", () => { } await waitForAgentServiceIngressForTest(service, THREAD_ID); - await vi.waitFor(() => { - expect(published - .filter((event) => event.type === AgentEventType.TextDelta) - .map((event) => event.delta.length)) - .toEqual([...firstSegment, ...secondSegment].map((delta) => delta.length)); - expect(published.filter((event) => event.type === AgentEventType.AssistantMessageBoundary)) - .toHaveLength(2); - }); + expect(published + .filter((event) => event.type === AgentEventType.TextDelta) + .map((event) => event.delta.length)) + .toEqual([...firstSegment, ...secondSegment].map((delta) => delta.length)); + expect(published.filter((event) => event.type === AgentEventType.AssistantMessageBoundary)) + .toHaveLength(2); } finally { db.close(); } @@ -996,12 +892,12 @@ describe("AgentService narrative persistence", () => { expect(published).toEqual([]); continueAgentTurnWithoutSavingForTest(service, executionId); - await vi.waitFor(() => { - expect(published.map((event) => event.type)).toEqual([ - AgentEventType.TextDelta, - AgentEventType.AssistantMessageBoundary, - ]); - }); + await Promise.resolve(); + + expect(published.map((event) => event.type)).toEqual([ + AgentEventType.TextDelta, + AgentEventType.AssistantMessageBoundary, + ]); expect(narrativeStore.recoverySnapshot(THREAD_ID)).toEqual([ expect.objectContaining({ kind: "narrationSegment", @@ -1052,7 +948,7 @@ describe("AgentService narrative persistence", () => { }); await waitForAgentServiceIngressForTest(service, THREAD_ID); vi.advanceTimersByTime(250); - await vi.waitFor(() => expect(published).toHaveLength(1)); + expect(published).toHaveLength(1); published.length = 0; db.exec(` CREATE TRIGGER reject_narration_recovery @@ -1134,11 +1030,9 @@ describe("AgentService narrative persistence", () => { }); await waitForAgentServiceIngressForTest(service, THREAD_ID); - await vi.waitFor(() => { - expect(observed.at(-1)).toMatchObject({ - type: AgentEventType.AssistantMessageBoundary, - persisted: expect.stringContaining("I will inspect the child."), - }); + expect(observed.at(-1)).toMatchObject({ + type: AgentEventType.AssistantMessageBoundary, + persisted: expect.stringContaining("I will inspect the child."), }); expect(observed.at(-1)?.persisted).not.toContain("turnExecutionId"); db.close(); diff --git a/apps/server/src/features/agents/orchestration/__tests__/agent-service-test-harness.ts b/apps/server/src/features/agents/orchestration/__tests__/agent-service-test-harness.ts index 91a05a6cd..b32a5cd6f 100644 --- a/apps/server/src/features/agents/orchestration/__tests__/agent-service-test-harness.ts +++ b/apps/server/src/features/agents/orchestration/__tests__/agent-service-test-harness.ts @@ -58,10 +58,6 @@ import { TurnFeatureEffects } from "../../turns/turn-feature-effects.js"; import { AgentEventPublicationRuntimePort, AgentTurnContinuationPort } from "../agent-runtime-internal-ports.js"; import { TurnRuntimeController } from "../turn-runtime-controller.js"; import { ProviderTurnEventApplication } from "../../turns/provider-turn-event-application.js"; -import { - PARENT_NARRATIVE_RECOVERY_WRITER, - type ParentNarrativeRecoveryWriter, -} from "../../turns/parent-narrative-recovery-coordinator.js"; import { TURN_RUNTIME_EVENT_CONTROL, type TurnRuntimeEventControl } from "../turn-runtime-event-control.js"; const testEventPublications = new WeakMap(); @@ -204,7 +200,6 @@ export function createAgentServiceForTest( threadBranching?: ThreadBranchingService, eventPublication?: AgentEventPublicationRegistry, threadStartups?: ThreadStartupService, - narrativeWriterOverride?: ParentNarrativeRecoveryWriter, ): AgentService { if (!parentDurability) throw new Error("Parent turn durability is required by the test harness"); const turnDiffs = new TurnDiffService(new TurnDiffRepo(db)); @@ -323,9 +318,6 @@ export function createAgentServiceForTest( testContainer.registerInstance(AgentReliabilityPort, reliability); testContainer.registerInstance(AgentEventPublicationRegistry, publication); testContainer.registerInstance(PARENT_TURN_DURABILITY, parentDurability); - testContainer.registerInstance(PARENT_NARRATIVE_RECOVERY_WRITER, narrativeWriterForTest( - narrativeWriterOverride, db, parentDurability, parentAssistantTextCheckpoints, - )); testContainer.registerInstance(NarrativeStore, narrativeStore); testContainer.registerInstance(ParentAssistantTextCheckpointService, parentAssistantTextCheckpoints); testContainer.registerInstance("Database", db); @@ -350,26 +342,3 @@ export function createAgentServiceForTest( testGoalLifecycles.set(service, resolvedGoals); return service; } - -function narrativeWriterForTest( - override: ParentNarrativeRecoveryWriter | undefined, - db: Database, - parentDurability: ParentTurnDurability, - parentAssistantTextCheckpoints: ParentAssistantTextCheckpointService, -): ParentNarrativeRecoveryWriter { - return override ?? { - async recordParentNarrativeRecovery(_operationId, input) { - return { recorded: parentDurability.recordParentNarrativeRecovery(input) }; - }, - async classifyParentNarrativeRecovery(_operationId, input) { - return db.transaction(() => { - const recorded = parentDurability.recordParentNarrativeRecovery(input); - if (!recorded) throw new Error("Canonical parent turn was not found"); - const reset = parentAssistantTextCheckpoints.resetInTransaction(input.executionId); - if (!reset) throw new Error("Provisional assistant text checkpoint was not reset"); - return { recorded, reset }; - })(); - }, - async acknowledgeOperation() {}, - }; -} diff --git a/apps/server/src/features/agents/turns/__tests__/parent-narrative-recovery-coordinator.test.ts b/apps/server/src/features/agents/turns/__tests__/parent-narrative-recovery-coordinator.test.ts index cc3180f20..2b05aadc5 100644 --- a/apps/server/src/features/agents/turns/__tests__/parent-narrative-recovery-coordinator.test.ts +++ b/apps/server/src/features/agents/turns/__tests__/parent-narrative-recovery-coordinator.test.ts @@ -4,10 +4,7 @@ import { type AgentEvent, type ParentNarrativeRecoveryItem, } from "@mcode/contracts"; -import { - ParentNarrativeRecoveryCoordinator, - type ParentNarrativeRecoveryWriter, -} from "../parent-narrative-recovery-coordinator.js"; +import { ParentNarrativeRecoveryCoordinator } from "../parent-narrative-recovery-coordinator.js"; import type { NarrativeStore } from "../../conversation/narrative/narrative-store.js"; import type { ParentTurnDurability } from "../parent-turn-durability.js"; @@ -26,84 +23,41 @@ const recoveryItems = [ }, }, ] satisfies ParentNarrativeRecoveryItem[]; -const event: AgentEvent = { - type: AgentEventType.TextDelta, - threadId: THREAD_ID, - turnExecutionId: EXECUTION_ID, - delta: "Reasoning before the answer.", -}; - -function coordinatorFor(writer: ParentNarrativeRecoveryWriter): ParentNarrativeRecoveryCoordinator { - const narrativeStore = { - recoverySnapshot: vi.fn(() => recoveryItems), - } as unknown as NarrativeStore; - const durability = { - loadTurnByExecution: vi.fn(() => ({ id: "turn-1" })), - } as unknown as ParentTurnDurability; - return new ParentNarrativeRecoveryCoordinator(writer, narrativeStore, durability); -} describe("ParentNarrativeRecoveryCoordinator", () => { - it("retries an uncommitted recovery snapshot with the same operation identity", async () => { + it("retries an uncommitted parent recovery snapshot before deduplicating it", () => { const recordParentNarrativeRecovery = vi.fn() - .mockResolvedValueOnce({ recorded: false }) - .mockResolvedValue({ recorded: true }); - const writer: ParentNarrativeRecoveryWriter = { + .mockReturnValueOnce(false) + .mockReturnValue(true); + const durability = { + loadTurnByExecution: vi.fn(() => ({ id: "turn-1" })), recordParentNarrativeRecovery, - classifyParentNarrativeRecovery: vi.fn(), - acknowledgeOperation: vi.fn(async () => undefined), + } as unknown as ParentTurnDurability; + const narrativeStore = { + recoverySnapshot: vi.fn(() => recoveryItems), + } as unknown as NarrativeStore; + const coordinator = new ParentNarrativeRecoveryCoordinator(durability, narrativeStore); + const event: AgentEvent = { + type: AgentEventType.TextDelta, + threadId: THREAD_ID, + turnExecutionId: EXECUTION_ID, + delta: "Reasoning before the answer.", }; - const coordinator = coordinatorFor(writer); const expectedCommit = { executionId: EXECUTION_ID, items: recoveryItems, discardedItemIds: [], }; - await expect(coordinator.checkpoint(event)).rejects.toThrow( + expect(() => coordinator.checkpoint(event)).toThrow( `Canonical parent turn was not found: ${EXECUTION_ID}`, ); - await coordinator.checkpoint(event); - await coordinator.checkpoint(event); - expect(recordParentNarrativeRecovery).toHaveBeenCalledTimes(2); - const operationId = recordParentNarrativeRecovery.mock.calls[0]?.[0]; - expect(operationId).toMatch(/^narrative:/); - expect(recordParentNarrativeRecovery).toHaveBeenNthCalledWith(1, operationId, expectedCommit); - expect(recordParentNarrativeRecovery).toHaveBeenNthCalledWith(2, operationId, expectedCommit); - expect(writer.acknowledgeOperation).not.toHaveBeenCalled(); - coordinator.acknowledgeHandled(event); - await vi.waitFor(() => expect(writer.acknowledgeOperation).toHaveBeenCalledWith(EXECUTION_ID, operationId)); - }); + coordinator.checkpoint(event); + coordinator.checkpoint(event); - it("retries the atomic narration classification without changing its input", async () => { - const classifyParentNarrativeRecovery = vi.fn() - .mockRejectedValueOnce(new Error("writer unavailable")) - .mockResolvedValue({ recorded: true, reset: true }); - const writer: ParentNarrativeRecoveryWriter = { - recordParentNarrativeRecovery: vi.fn(), - classifyParentNarrativeRecovery, - acknowledgeOperation: vi.fn(async () => undefined), - }; - const coordinator = coordinatorFor(writer); - const boundary: AgentEvent = { - type: AgentEventType.AssistantMessageBoundary, - threadId: THREAD_ID, - turnExecutionId: EXECUTION_ID, - isFinalResponse: false, - }; - - await expect(coordinator.classify(boundary, recoveryItems)).rejects.toThrow("writer unavailable"); - await coordinator.classify(boundary, recoveryItems); - await coordinator.checkpoint(boundary); - - expect(classifyParentNarrativeRecovery).toHaveBeenCalledTimes(2); - expect(classifyParentNarrativeRecovery.mock.calls[1]).toEqual(classifyParentNarrativeRecovery.mock.calls[0]); - expect(writer.acknowledgeOperation).not.toHaveBeenCalled(); - coordinator.acknowledgeHandled(boundary); - await vi.waitFor(() => expect(writer.acknowledgeOperation).toHaveBeenCalledWith( - EXECUTION_ID, - classifyParentNarrativeRecovery.mock.calls[0]?.[0], - )); + expect(recordParentNarrativeRecovery).toHaveBeenCalledTimes(2); + expect(recordParentNarrativeRecovery).toHaveBeenNthCalledWith(1, expectedCommit); + expect(recordParentNarrativeRecovery).toHaveBeenNthCalledWith(2, expectedCommit); }); }); diff --git a/apps/server/src/features/agents/turns/parent-narrative-recovery-coordinator.ts b/apps/server/src/features/agents/turns/parent-narrative-recovery-coordinator.ts index e08a4d9c8..2b0c2b6bf 100644 --- a/apps/server/src/features/agents/turns/parent-narrative-recovery-coordinator.ts +++ b/apps/server/src/features/agents/turns/parent-narrative-recovery-coordinator.ts @@ -1,137 +1,85 @@ -import * as NodeCrypto from "node:crypto"; -import { logger } from "@mcode/shared"; import { AgentEventType, type AgentEvent, type ParentNarrativeRecoveryItem, } from "@mcode/contracts"; -import type { NarrativeStore } from "../conversation/narrative/narrative-store.js"; -import type { CanonicalAgentWriterClient } from "../canonical/canonical-agent-writer-client.js"; -import type { ParentNarrativeRecoveryCommit } from "./parent-turn-durability.js"; import type { ParentTurnDurability } from "./parent-turn-durability.js"; +import type { NarrativeStore } from "../conversation/narrative/narrative-store.js"; import { serverWorkTrace } from "../diagnostics/server-work-trace.js"; -/** The single acknowledged writer used for recovery and text classification. */ -export const PARENT_NARRATIVE_RECOVERY_WRITER = Symbol("ParentNarrativeRecoveryWriter"); -export type ParentNarrativeRecoveryWriter = Pick; - interface ParentNarrativeRecoveryCheckpoint { - operationId: string; - input: ParentNarrativeRecoveryCommit; - committed: boolean; + persist(): void; confirm(): void; } /** Commits changed structured narrative records before AgentService publishes their source event. */ export class ParentNarrativeRecoveryCoordinator { private readonly fingerprintsByExecution = new Map>(); - private readonly preparedByEvent = new WeakMap(); - private readonly existingExecutions = new Set(); constructor( - private readonly writer: ParentNarrativeRecoveryWriter, - private readonly narrativeStore: NarrativeStore, private readonly canonicalSink: ParentTurnDurability, + private readonly narrativeStore: NarrativeStore, ) {} /** Commit only the semantic records changed by this accepted provider event. */ - checkpoint(event: AgentEvent): Promise | void { - const checkpoint = this.prepareCheckpoint(event); - if (!checkpoint || checkpoint.committed) return; - const persist = () => this.writer.recordParentNarrativeRecovery(checkpoint.operationId, checkpoint.input); - const pending = serverWorkTrace - ? serverWorkTrace.measure("narrative-persist", event.threadId, event.turnExecutionId, persist) - : persist(); - return pending.then((receipt) => { - if (!receipt.recorded) throw new Error(`Canonical parent turn was not found: ${event.turnExecutionId}`); - if (serverWorkTrace) { - serverWorkTrace.measure("narrative-confirm", event.threadId, event.turnExecutionId, checkpoint.confirm); - } else checkpoint.confirm(); - checkpoint.committed = true; - }); - } - - /** Commit staged narration and reset provisional assistant text in one writer transaction. */ - async classify( - event: AgentEvent, - snapshot: readonly ParentNarrativeRecoveryItem[], - ): Promise { - const checkpoint = this.prepareCheckpoint(event, snapshot, true); - if (!checkpoint) throw new Error("Narrative classification requires a turn execution"); - await this.writer.classifyParentNarrativeRecovery(checkpoint.operationId, checkpoint.input); - checkpoint.confirm(); - checkpoint.committed = true; - } - - /** Release a receipt only after its source event was actually published or its recovery journal retired. */ - acknowledgeHandled(event: AgentEvent): void { - const checkpoint = this.preparedByEvent.get(event as object); - if (!checkpoint?.committed) return; - this.preparedByEvent.delete(event as object); - const { executionId } = checkpoint.input; - void this.writer.acknowledgeOperation(executionId, checkpoint.operationId).catch(() => { - logger.warn("Canonical narrative receipt acknowledgement failed", { - executionId, - operationId: checkpoint.operationId, - }); - }); + checkpoint(event: AgentEvent): void { + if (!serverWorkTrace) { + const checkpoint = this.prepareCheckpoint(event); + if (!checkpoint) return; + checkpoint.persist(); + checkpoint.confirm(); + return; + } + const checkpoint = serverWorkTrace.measure("narrative-prepare", event.threadId, + event.turnExecutionId, () => this.prepareCheckpoint(event)); + if (!checkpoint) return; + serverWorkTrace.measure("narrative-persist", event.threadId, + event.turnExecutionId, () => checkpoint.persist()); + serverWorkTrace.measure("narrative-confirm", event.threadId, + event.turnExecutionId, () => checkpoint.confirm()); } - private prepareCheckpoint( + /** Prepare a recovery commit whose dedupe state advances only after its transaction commits. */ + prepareCheckpoint( event: AgentEvent, snapshot?: readonly ParentNarrativeRecoveryItem[], - force = false, ): ParentNarrativeRecoveryCheckpoint | null { if (!this.requiresStructuredRecovery(event) || !event.turnExecutionId) return null; - if (!force && !this.existingExecutions.has(event.turnExecutionId)) { - if (!this.canonicalSink.loadTurnByExecution(event.turnExecutionId)) return null; - this.existingExecutions.add(event.turnExecutionId); - } - const existing = this.preparedByEvent.get(event as object); - if (existing) return existing; - const executionId = event.turnExecutionId; - const prepare = () => this.buildCheckpoint(event, executionId, snapshot, force); - return serverWorkTrace - ? serverWorkTrace.measure("narrative-prepare", event.threadId, event.turnExecutionId, prepare) - : prepare(); - } - - private buildCheckpoint( - event: AgentEvent, - executionId: string, - snapshot: readonly ParentNarrativeRecoveryItem[] | undefined, - force: boolean, - ): ParentNarrativeRecoveryCheckpoint | null { + if (!this.canonicalSink.loadTurnByExecution(event.turnExecutionId)) return null; const snapshots = snapshot ?? this.narrativeStore.recoverySnapshot(event.threadId); - const fingerprints = this.fingerprintsByExecution.get(executionId) ?? new Map(); + + const fingerprints = this.fingerprintsByExecution.get(event.turnExecutionId) ?? new Map(); const nextFingerprints = new Map(); const currentKeys = new Set(snapshots.map((item) => `${item.kind}:${item.record.id}`)); const changed = snapshots.filter((item) => { const key = `${item.kind}:${item.record.id}`; const fingerprint = JSON.stringify(item); nextFingerprints.set(key, fingerprint); - return fingerprints.get(key) !== fingerprint; + if (fingerprints.get(key) === fingerprint) return false; + return true; }); const discardedItemIds = [...fingerprints.keys()] .filter((key) => !currentKeys.has(key)) .map((key) => this.canonicalItemId(key)); - if (!force && changed.length === 0 && discardedItemIds.length === 0) return null; - const prepared = { - operationId: `narrative:${NodeCrypto.randomUUID()}`, - input: { executionId, items: changed, discardedItemIds }, - committed: false, - confirm: () => this.fingerprintsByExecution.set(executionId, nextFingerprints), + if (changed.length === 0 && discardedItemIds.length === 0) return null; + return { + persist: () => { + const committed = this.canonicalSink.recordParentNarrativeRecovery({ + executionId: event.turnExecutionId!, + items: changed, + discardedItemIds, + }); + if (!committed) { + throw new Error(`Canonical parent turn was not found: ${event.turnExecutionId}`); + } + }, + confirm: () => this.fingerprintsByExecution.set(event.turnExecutionId!, nextFingerprints), }; - this.preparedByEvent.set(event as object, prepared); - return prepared; } /** Forget volatile dedupe state after a terminal turn releases its buffers. */ clear(executionId: string | undefined): void { - if (!executionId) return; - this.fingerprintsByExecution.delete(executionId); - this.existingExecutions.delete(executionId); + if (executionId) this.fingerprintsByExecution.delete(executionId); } private requiresStructuredRecovery(event: AgentEvent): boolean { @@ -150,4 +98,5 @@ export class ParentNarrativeRecoveryCoordinator { if (!kind || !id) throw new Error(`Invalid narrative recovery identity: ${key}`); return kind === "toolCall" ? `toolCall:${id}` : `${kind}:${id}`; } + } diff --git a/apps/server/src/features/agents/turns/provider-turn-event-application.ts b/apps/server/src/features/agents/turns/provider-turn-event-application.ts index a96548302..844120ca8 100644 --- a/apps/server/src/features/agents/turns/provider-turn-event-application.ts +++ b/apps/server/src/features/agents/turns/provider-turn-event-application.ts @@ -1,3 +1,4 @@ +import type { Database } from "bun:sqlite"; import { inject, injectable } from "tsyringe"; import { AgentEventType, @@ -16,11 +17,7 @@ import { TURN_FINALIZER, TurnFinalizer } from "./turn-finalizer.js"; import { TURN_FILE_EFFECTS, TurnFileEffects } from "./turn-file-effects.js"; import { ParentAssistantTextCheckpointService } from "./parent-assistant-text-checkpoint-service.js"; import { ParentAssistantTextCoordinator } from "./parent-assistant-text-coordinator.js"; -import { - PARENT_NARRATIVE_RECOVERY_WRITER, - ParentNarrativeRecoveryCoordinator, - type ParentNarrativeRecoveryWriter, -} from "./parent-narrative-recovery-coordinator.js"; +import { ParentNarrativeRecoveryCoordinator } from "./parent-narrative-recovery-coordinator.js"; import { PARENT_TURN_DURABILITY, type ParentTurnDurability } from "./parent-turn-durability.js"; import { TurnConversationProjectionService } from "./turn-conversation-projection-service.js"; import { PostTerminalHookCompletionEffect } from "./post-terminal-hook-completion-effect.js"; @@ -58,11 +55,6 @@ export class ProviderTurnEventApplication implements TurnEventApplication { private readonly lastContextByThread = new Map(); private readonly lastContextWindowByThread = new Map(); private readonly terminalProjectionByThread = new Map>(); - private readonly narrativeApplicationsByThread = new Map; - }>(); - private readonly failedNarrativeExecutionByThread = new Map(); constructor( @inject(TURN_FINALIZER) private readonly finalizer: TurnFinalizer, @@ -80,7 +72,7 @@ export class ProviderTurnEventApplication implements TurnEventApplication { @inject(NarrativeStore) private readonly narrative: NarrativeStore, @inject(ParentAssistantTextCheckpointService) checkpoints: ParentAssistantTextCheckpointService, @inject(TURN_FEATURE_EFFECTS) private readonly featureEffects: TurnFeatureEffects, - @inject(PARENT_NARRATIVE_RECOVERY_WRITER) narrativeWriter: ParentNarrativeRecoveryWriter, + @inject("Database") private readonly db: Database, @inject(TURN_RUNTIME_EVENT_CONTROL) private readonly runtime: TurnRuntimeEventControl, @inject(AgentEventPublicationRegistry) private readonly publication: AgentEventPublicationRegistry, @@ -95,7 +87,7 @@ export class ProviderTurnEventApplication implements TurnEventApplication { } }, ); - this.parentNarrativeRecovery = new ParentNarrativeRecoveryCoordinator(narrativeWriter, narrative, parentDurability); + this.parentNarrativeRecovery = new ParentNarrativeRecoveryCoordinator(parentDurability, narrative); this.browserNarrativeEventSanitizer = new BrowserNarrativeEventSanitizer( (threadId, toolCallId) => this.narrative.getBufferedToolCalls(threadId) .find((toolCall) => toolCall.toolCallId === toolCallId) @@ -114,7 +106,7 @@ export class ProviderTurnEventApplication implements TurnEventApplication { } /** Apply one queue-admitted provider event and publish it only after its durable prerequisites complete. */ - apply(input: ProviderEventIngressEvent, event: AgentEvent, publish: boolean): boolean | Promise { + apply(input: ProviderEventIngressEvent, event: AgentEvent, publish: boolean): boolean { const queued = this.queueVisibleAssistantText(input, event, publish); if (queued !== undefined) return queued; return this.applyPreparedEvent(input, event, publish); @@ -155,25 +147,13 @@ export class ProviderTurnEventApplication implements TurnEventApplication { if (this.terminalFinalizedThreads.has(command.threadId)) return null; this.terminalFinalizedThreads.add(command.threadId); const executionId = this.runtime.snapshot(command.threadId)?.turnExecutionId; - const finalizeEffects = () => this.fileEffects.finalize( + const finalization = this.fileEffects.finalize( command.threadId, command.outcome, executionId ?? undefined, command.source, ); - const narrative = this.narrativeApplicationsByThread.get(command.threadId); - const failedExecutionId = this.failedNarrativeExecutionByThread.get(command.threadId); - const failed = this.failedNarrativeExecutionByThread.has(command.threadId) - && (failedExecutionId === undefined || failedExecutionId === executionId); - const finalization = failed - ? Promise.resolve(false) - : narrative && narrative.executionId === executionId - ? narrative.pending.then((ready) => ready ? finalizeEffects() : false) - : finalizeEffects(); - void finalization.then( - () => this.clearFinalizedEventState(command.threadId, executionId), - () => this.clearFinalizedEventState(command.threadId, executionId), - ); + void finalization.finally(() => this.clearFinalizedEventState(command.threadId, executionId)); return finalization; } @@ -269,55 +249,15 @@ export class ProviderTurnEventApplication implements TurnEventApplication { event: AgentEvent, publish: boolean, textIsDurable = false, - ): boolean | Promise { + ): boolean { const terminal = isTerminal(event); const preparation = this.prepareEventForApplication(event, publish, textIsDurable, terminal); - if (preparation instanceof Promise) { - return preparation.then((ready) => ready === undefined - ? this.applyReadyEvent(input, event, publish, terminal) - : ready); - } if (preparation !== undefined) return preparation; - return this.applyReadyEvent(input, event, publish, terminal); - } - - private applyReadyEvent( - input: ProviderEventIngressEvent, - event: AgentEvent, - publish: boolean, - terminal: boolean, - ): boolean | Promise { this.recordDiagnostic(input, event); const accepted = this.applyEvent(input.providerId, event, publish); - if (accepted instanceof Promise) { - return accepted.then((result) => this.finishAppliedEvent(event, publish, terminal, result)); - } - return this.finishAppliedEvent(event, publish, terminal, accepted); - } - - private finishAppliedEvent( - event: AgentEvent, - publish: boolean, - terminal: boolean, - accepted: EventApplicationResult, - ): boolean | Promise { if (accepted === false) return true; if (terminal && accepted !== true) return false; - const durable = this.checkpointNarrative(event, publish); - if (durable instanceof Promise) { - return durable.then((ready) => this.publishAppliedEvent(event, publish, terminal, accepted, ready)); - } - return this.publishAppliedEvent(event, publish, terminal, accepted, durable); - } - - private publishAppliedEvent( - event: AgentEvent, - publish: boolean, - terminal: boolean, - accepted: EventApplicationResult, - durable: boolean, - ): boolean { - if (!durable) return false; + if (!this.checkpointNarrative(event, publish)) return false; if (publish && accepted === OWNED_LATE_HOOK_COMPLETION) return true; if (publish) this.publishAfterDurability(event, terminal); return true; @@ -331,34 +271,16 @@ export class ProviderTurnEventApplication implements TurnEventApplication { if (!publish || !this.publication.isBound() || event.type !== AgentEventType.TextDelta) return undefined; if (event.isFinalResponse === false || !event.turnExecutionId) return undefined; const queued = this.queueParentAssistantText(event, () => { - this.applyQueuedVisibleText(input, event); + void this.applyPreparedEvent(input, event, true, true); }); return queued === "blocked" ? false : queued; } - private applyQueuedVisibleText(input: ProviderEventIngressEvent, event: AgentEvent): void { - const executionId = event.turnExecutionId; - if (!executionId) return; - const prior = this.narrativeApplicationsByThread.get(event.threadId)?.pending ?? Promise.resolve(true); - const pending = prior.then((ready) => ready ? this.applyPreparedEvent(input, event, true, true) : false) - .catch(() => this.failNarrativeCheckpoint(event)); - this.narrativeApplicationsByThread.set(event.threadId, { executionId, pending }); - void pending.then((ready) => { - if (ready && this.narrativeApplicationsByThread.get(event.threadId)?.pending === pending) { - this.narrativeApplicationsByThread.delete(event.threadId); - } - }); - } - - private applyEvent( - providerId: ProviderId, - event: AgentEvent, - publish: boolean, - ): EventApplicationResult | Promise { + private applyEvent(providerId: ProviderId, event: AgentEvent, publish: boolean): EventApplicationResult { return this.applyNarrativeEvent(providerId, event) ?? this.applyLifecycleEvent(providerId, event, publish); } - private applyNarrativeEvent(providerId: ProviderId, event: AgentEvent): EventApplicationResult | Promise { + private applyNarrativeEvent(providerId: ProviderId, event: AgentEvent): EventApplicationResult { switch (event.type) { case AgentEventType.TextDelta: return this.applyTextDelta(event); case AgentEventType.GeneratedAttachment: return this.applyGeneratedAttachment(event); @@ -393,14 +315,12 @@ export class ProviderTurnEventApplication implements TurnEventApplication { publish: boolean, textIsDurable: boolean, terminal: boolean, - ): boolean | Promise | undefined { + ): boolean | undefined { if (!this.prepareTerminalText(event, publish, terminal)) return false; - if (!publish || textIsDurable) return undefined; - if (!this.parentAssistantText.prepareSemanticBoundary(event.threadId)) { - return this.parentAssistantText.hasThreadStoppedForStorageFailure(event.threadId); + if (!publish || textIsDurable || this.parentAssistantText.prepareSemanticBoundary(event.threadId)) { + return undefined; } - const narrative = this.narrativeApplicationsByThread.get(event.threadId); - return narrative ? narrative.pending.then((ready) => ready ? undefined : false) : undefined; + return this.parentAssistantText.hasThreadStoppedForStorageFailure(event.threadId); } private applyTextDelta(event: Extract): boolean { @@ -440,7 +360,7 @@ export class ProviderTurnEventApplication implements TurnEventApplication { return true; } - private applyAssistantMessageBoundary(event: Extract): boolean | Promise { + private applyAssistantMessageBoundary(event: Extract): boolean { if (event.isFinalResponse === true) { const finalText = this.narrative.takeOpenThought(event.threadId); if (finalText) this.finalizer.appendStreamingText(event.threadId, finalText); @@ -666,29 +586,20 @@ export class ProviderTurnEventApplication implements TurnEventApplication { return false; } - private checkpointNarrative(event: AgentEvent, publish: boolean): boolean | Promise { + private checkpointNarrative(event: AgentEvent, publish: boolean): boolean { if (!publish || this.isUnsavedNarrationBoundary(event)) return true; try { - const checkpoint = () => this.parentNarrativeRecovery.checkpoint(event); - const pending = serverWorkTrace - ? serverWorkTrace.measure("narrative-checkpoint", event.threadId, event.turnExecutionId, checkpoint) - : checkpoint(); - if (!pending) return true; - return pending.then( - () => true, - () => this.failNarrativeCheckpoint(event), - ); + if (serverWorkTrace) { + serverWorkTrace.measure("narrative-checkpoint", event.threadId, event.turnExecutionId, + () => this.parentNarrativeRecovery.checkpoint(event)); + } else this.parentNarrativeRecovery.checkpoint(event); + return true; } catch { - return this.failNarrativeCheckpoint(event); + this.runtime.stopForEventApplicationFailure(event, "Parent narrative recovery checkpoint failed"); + return false; } } - private failNarrativeCheckpoint(event: AgentEvent): false { - this.failedNarrativeExecutionByThread.set(event.threadId, event.turnExecutionId); - this.runtime.stopForEventApplicationFailure(event, "Parent narrative recovery checkpoint failed"); - return false; - } - private publishAfterDurability(event: AgentEvent, terminal: boolean): void { if (!terminal && !this.isLateHook(event)) { this.publish(event); @@ -710,7 +621,6 @@ export class ProviderTurnEventApplication implements TurnEventApplication { ? { ...event, outcome: "interrupted" } : event, ); - this.parentNarrativeRecovery.acknowledgeHandled(event); } /** Retain terminal durability from its scheduling point until every dependent publication observes it. */ @@ -792,7 +702,7 @@ export class ProviderTurnEventApplication implements TurnEventApplication { private classifyUnclassifiedAssistantTextAsNarration( event: Extract, - ): boolean | Promise { + ): boolean { const executionId = event.turnExecutionId; if (!executionId) return this.clearUnclassifiedAssistantText(event.threadId); const firstSequence = this.unclassifiedAssistantTextStartByExecution.get(executionId); @@ -810,21 +720,30 @@ export class ProviderTurnEventApplication implements TurnEventApplication { .map((chunk) => chunk.text) .join(""); const staged = this.narrative.stageNarrationSegment(event.threadId, text); - const snapshot = staged - ? this.narrative.recoverySnapshotWithStagedNarration(event.threadId, staged) - : this.narrative.recoverySnapshot(event.threadId); - return this.parentNarrativeRecovery.classify(event, snapshot).then( - () => { - this.parentAssistantText.checkpoints.discardRecoveryJournal(executionId); - this.parentNarrativeRecovery.acknowledgeHandled(event); - this.parentAssistantText.discard(executionId); - if (staged) this.narrative.applyStagedNarrationSegment(event.threadId, staged); - this.unclassifiedAssistantTextStartByExecution.delete(executionId); - this.finalizer.resetStreamingText(event.threadId); - return true; - }, - () => this.failNarrativeCheckpoint(event), - ); + let confirm: (() => void) | undefined; + try { + this.db.transaction(() => { + const checkpoint = this.parentNarrativeRecovery.prepareCheckpoint( + event, + staged ? this.narrative.recoverySnapshotWithStagedNarration(event.threadId, staged) : undefined, + ); + checkpoint?.persist(); + if (!this.parentAssistantText.checkpoints.resetInTransaction(executionId)) { + throw new Error(`Unclassified assistant text checkpoint was not reset: ${executionId}`); + } + confirm = checkpoint?.confirm; + })(); + } catch { + this.runtime.stopForEventApplicationFailure(event, "Parent narrative recovery checkpoint failed"); + return false; + } + this.parentAssistantText.checkpoints.discardRecoveryJournal(executionId); + this.parentAssistantText.discard(executionId); + confirm?.(); + if (staged) this.narrative.applyStagedNarrationSegment(event.threadId, staged); + this.unclassifiedAssistantTextStartByExecution.delete(executionId); + this.finalizer.resetStreamingText(event.threadId); + return true; } private clearUnclassifiedAssistantText(threadId: string): true { @@ -849,14 +768,6 @@ export class ProviderTurnEventApplication implements TurnEventApplication { } private clearFinalizedEventState(threadId: string, executionId: string | null | undefined): void { - const failedExecutionId = this.failedNarrativeExecutionByThread.get(threadId); - if (this.failedNarrativeExecutionByThread.has(threadId) - && (failedExecutionId === undefined || failedExecutionId === executionId)) { - this.failedNarrativeExecutionByThread.delete(threadId); - } - if (this.narrativeApplicationsByThread.get(threadId)?.executionId === executionId) { - this.narrativeApplicationsByThread.delete(threadId); - } if (executionId) { this.parentAssistantText.discard(executionId); this.unclassifiedAssistantTextStartByExecution.delete(executionId); From 1cdcdefbe20774b6866653f4d69ebe59f1e315df Mon Sep 17 00:00:00 2001 From: "(CJ) Chukwudi Nwobodo" <142016413+chuks-qua@users.noreply.github.com> Date: Thu, 24 Sep 2026 20:28:22 +0100 Subject: [PATCH 034/136] Revert "feat(server): route canonical provider commits through SQLite writer" This reverts commit cd79780a362ceaa044c796d5bdf6896c52689aa1. --- .../application/bootstrap/server-bootstrap.ts | 4 - .../provider-composition-container.test.ts | 2 - .../__tests__/provider-host-ports.test.ts | 81 ++----------------- .../composition/provider-host-ports.ts | 38 +-------- .../composition/register-providers.ts | 10 +-- 5 files changed, 10 insertions(+), 125 deletions(-) diff --git a/apps/server/src/application/bootstrap/server-bootstrap.ts b/apps/server/src/application/bootstrap/server-bootstrap.ts index 9b5a1681a..5975addb5 100644 --- a/apps/server/src/application/bootstrap/server-bootstrap.ts +++ b/apps/server/src/application/bootstrap/server-bootstrap.ts @@ -70,7 +70,6 @@ import { startAgentOrchestration, } from "../../features/agents"; import { AgentEventPublicationRegistry } from "../../features/agents/orchestration/agent-event-publication-registry.js"; -import { CanonicalAgentWriterClient } from "../../features/agents/canonical/canonical-agent-writer-client.js"; import { AgentEventPublicationRuntimePort, AgentReliabilityPort, @@ -343,7 +342,6 @@ const messageRepo = container.resolve(MessageRepo); const threadRepo = container.resolve(ThreadRepo); const providerRegistry = container.resolve(ProviderRegistry); const providerEventIngress = container.resolve(ProviderEventIngress); -const canonicalWriter = container.resolve(CanonicalAgentWriterClient); const cursorProvider = container.resolve("CursorProvider"); const providerAvailability = container.resolve(ProviderAvailabilityService); const toolCallRecordRepo = container.resolve(ToolCallRecordRepo); @@ -847,7 +845,6 @@ async function bootstrapServer(): Promise { }); } await canonicalSink.materializeConversationDisplay(); - await canonicalWriter.whenReady(); recordStartupCheckpoint("conversation display materialization completed"); interruptThreadStartupsAtStartup(); @@ -954,7 +951,6 @@ async function shutdown(): Promise { await captureCleanupFailure(() => providerRegistry.shutdown()); shutdownCoordinator.setPhase("shutdown provider event workers"); providerEventIngress.shutdown(); - await captureCleanupFailure(() => canonicalWriter.close()); browserAutomationBroker.shutdown(); browserAutomationSessionLease.shutdown(); diff --git a/apps/server/src/features/providers/composition/__tests__/provider-composition-container.test.ts b/apps/server/src/features/providers/composition/__tests__/provider-composition-container.test.ts index 405c47486..05f96508c 100644 --- a/apps/server/src/features/providers/composition/__tests__/provider-composition-container.test.ts +++ b/apps/server/src/features/providers/composition/__tests__/provider-composition-container.test.ts @@ -13,7 +13,6 @@ import type { ProviderHostPorts } from "@mcode/providers"; import { setupContainer } from "../../../../application/composition/container.js"; import { CanonicalAgentBoundary } from "../../../agents/canonical/canonical-agent-boundary.js"; -import { CanonicalAgentWriterClient } from "../../../agents/canonical/canonical-agent-writer-client.js"; import { MessageRepo } from "../../../agents/conversation/persistence/message-repo.js"; import { ProviderRegistry } from "../provider-registry.js"; import { SettingsService } from "../../../settings/settings-service.js"; @@ -92,7 +91,6 @@ describe("provider composition container", () => { vi.restoreAllMocks(); await container.resolve(ProviderRegistry).shutdown(); container.resolve(ProviderEventIngress).shutdown(); - await container.resolve(CanonicalAgentWriterClient).close(); container.resolve(SettingsService).dispose(); database?.close(true); database = undefined; diff --git a/apps/server/src/features/providers/composition/__tests__/provider-host-ports.test.ts b/apps/server/src/features/providers/composition/__tests__/provider-host-ports.test.ts index ab7af0d2c..4e94cef32 100644 --- a/apps/server/src/features/providers/composition/__tests__/provider-host-ports.test.ts +++ b/apps/server/src/features/providers/composition/__tests__/provider-host-ports.test.ts @@ -54,7 +54,6 @@ describe("createProviderHostPorts", () => { threadControl: {}, grants: {}, events: {}, - publishCanonicalEvents: vi.fn(), ingress: {}, } as never); @@ -71,7 +70,7 @@ describe("createProviderHostPorts", () => { it("hands a committed canonical batch directly to ingress after durable acceptance", async () => { const events = [committedRuntimeEnvelope()]; const deliveryOrder: string[] = []; - const commit = vi.fn(async () => { + const commit = vi.fn(() => { deliveryOrder.push("commit"); return { outcome: "committed" as const, @@ -82,9 +81,7 @@ describe("createProviderHostPorts", () => { events, }; }); - const publishCanonicalEvents = vi.fn(() => deliveryOrder.push("publication")); const acceptCommitted = vi.fn(() => deliveryOrder.push("ingress")); - const acknowledgeOperation = vi.fn(async () => { deliveryOrder.push("acknowledge"); }); const ports = createProviderHostPorts({ runtime: { platform: "linux", architecture: "x64", nodeAbi: "127" }, envService: { getEnv: () => ({ PATH: "test" }) }, @@ -92,8 +89,7 @@ describe("createProviderHostPorts", () => { browser: {}, threadControl: {}, grants: {}, - events: { commit, acknowledgeOperation }, - publishCanonicalEvents, + events: { commit }, ingress: { acceptCommitted }, } as never); const batch = { threadId: "thread-1", turnId: "turn-1", executionId: EXECUTION_ID, phase: "streaming", events: [] }; @@ -109,11 +105,9 @@ describe("createProviderHostPorts", () => { }, delivery: { ingress: "queued" }, }); - expect(commit).toHaveBeenCalledWith(expect.any(String), { ...batch, nativeCursor: undefined }); - expect(publishCanonicalEvents).toHaveBeenCalledWith(events); + expect(commit).toHaveBeenCalledWith({ ...batch, nativeCursor: undefined }); expect(acceptCommitted).toHaveBeenCalledWith(events); - expect(acknowledgeOperation).toHaveBeenCalledWith(EXECUTION_ID, expect.any(String)); - expect(deliveryOrder).toEqual(["commit", "publication", "ingress", "acknowledge"]); + expect(deliveryOrder).toEqual(["commit", "ingress"]); }); it("does not hand duplicate or failed commits to ingress", async () => { @@ -137,8 +131,7 @@ describe("createProviderHostPorts", () => { browser: {}, threadControl: {}, grants: {}, - events: { commit, acknowledgeOperation: vi.fn() }, - publishCanonicalEvents: vi.fn(), + events: { commit }, ingress: { acceptCommitted }, } as never); const batch = { threadId: "thread-1", turnId: "turn-1", executionId: EXECUTION_ID, phase: "streaming", events: [] }; @@ -150,68 +143,4 @@ describe("createProviderHostPorts", () => { await expect(ports.events.submit(batch)).rejects.toThrow("commit failed"); expect(acceptCommitted).not.toHaveBeenCalled(); }); - - it("uses one operation identity for a retry and distinct identities for other phases", async () => { - const commit = vi.fn(async () => ({ - outcome: "duplicate" as const, - conversationRevision: 1, - rosterRevision: 0, - acceptedThrough: 1, - durableThrough: 1, - events: [], - })); - const ports = createProviderHostPorts({ - runtime: { platform: "linux", architecture: "x64", nodeAbi: "127" }, - envService: { getEnv: () => ({}) }, - jobObject: { isWindowsJob: false }, - browser: {}, - threadControl: {}, - grants: {}, - events: { commit, acknowledgeOperation: vi.fn() }, - publishCanonicalEvents: vi.fn(), - ingress: { acceptCommitted: vi.fn() }, - } as never); - const batch = { threadId: "thread-1", turnId: "turn-1", executionId: EXECUTION_ID, phase: "streaming", events: [] }; - - await ports.events.submit(batch); - await ports.events.submit(batch); - await ports.events.submit({ ...batch, phase: "completed" }); - const operationIds = commit.mock.calls.map(([operationId]) => operationId); - expect(operationIds[0]).toBe(operationIds[1]); - expect(operationIds[2]).not.toBe(operationIds[0]); - }); - - it("retains the receipt when publication fails after a durable commit", async () => { - const events = [committedRuntimeEnvelope()]; - const acknowledgeOperation = vi.fn(); - const acceptCommitted = vi.fn(); - const ports = createProviderHostPorts({ - runtime: { platform: "linux", architecture: "x64", nodeAbi: "127" }, - envService: { getEnv: () => ({}) }, - jobObject: { isWindowsJob: false }, - browser: {}, - threadControl: {}, - grants: {}, - events: { - commit: vi.fn(async () => ({ - outcome: "committed", - conversationRevision: 1, - rosterRevision: 0, - acceptedThrough: 1, - durableThrough: 1, - events, - })), - acknowledgeOperation, - }, - publishCanonicalEvents: () => { throw new Error("push unavailable"); }, - ingress: { acceptCommitted }, - } as never); - - await expect(ports.events.submit({ - threadId: "thread-1", turnId: "turn-1", executionId: EXECUTION_ID, - phase: "streaming", events: [], - })).resolves.toMatchObject({ commit: { outcome: "committed" } }); - expect(acceptCommitted).toHaveBeenCalledWith(events); - expect(acknowledgeOperation).not.toHaveBeenCalled(); - }); }); diff --git a/apps/server/src/features/providers/composition/provider-host-ports.ts b/apps/server/src/features/providers/composition/provider-host-ports.ts index dc37d7b7e..890f578d9 100644 --- a/apps/server/src/features/providers/composition/provider-host-ports.ts +++ b/apps/server/src/features/providers/composition/provider-host-ports.ts @@ -1,12 +1,9 @@ import type { ProviderHostPorts } from "@mcode/providers"; -import * as NodeCrypto from "node:crypto"; import type { HostRuntime } from "@mcode/shared/node/host-runtime"; -import { logger } from "@mcode/shared"; import type { JobObject } from "../../../runtime/process/containment/job-object.js"; import type { EnvService } from "../../../runtime/environment/env-service.js"; import type { ScopedPreGrantService } from "../../agents/permissions/scoped-pre-grant.js"; -import type { CanonicalAgentEventPublisher } from "../../agents/canonical/canonical-agent-boundary.js"; -import type { CanonicalAgentWriterClient } from "../../agents/canonical/canonical-agent-writer-client.js"; +import type { CanonicalAgentBoundary } from "../../agents/index.js"; import type { BrowserAutomationSessionLease } from "../../browser-automation/index.js"; import type { InternalThreadControlMcpRuntime } from "../../thread-control/index.js"; import { killProcessTree } from "../../../runtime/process/containment/process-kill.js"; @@ -20,8 +17,7 @@ export interface ProviderHostPortDependencies { browser: BrowserAutomationSessionLease; threadControl: InternalThreadControlMcpRuntime; grants: ScopedPreGrantService; - events: Pick; - publishCanonicalEvents: CanonicalAgentEventPublisher; + events: CanonicalAgentBoundary; ingress: ProviderEventIngress; } @@ -87,15 +83,7 @@ export function createProviderHostPorts( }, events: { submit: async (batch) => { - const operationId = NodeCrypto.createHash("sha256") - .update(JSON.stringify([ - batch.executionId, - batch.phase, - batch.nativeCursor ?? null, - batch.events.map((event) => event.eventId), - ])) - .digest("hex"); - const result = await dependencies.events.commit(operationId, { + const result = dependencies.events.commit({ threadId: batch.threadId, turnId: batch.turnId, executionId: batch.executionId, @@ -103,29 +91,9 @@ export function createProviderHostPorts( nativeCursor: batch.nativeCursor, events: batch.events, }); - let published = true; if (result.outcome === "committed" && result.events.length > 0) { - try { - dependencies.publishCanonicalEvents(result.events); - } catch { - published = false; - logger.warn("Canonical provider publication deferred after durable commit", { - threadId: batch.threadId, - executionId: batch.executionId, - }); - } dependencies.ingress.acceptCommitted(result.events); } - if (published) { - try { - await dependencies.events.acknowledgeOperation(batch.executionId, operationId); - } catch { - logger.warn("Canonical provider receipt acknowledgement deferred", { - threadId: batch.threadId, - executionId: batch.executionId, - }); - } - } return { commit: { outcome: providerCommitOutcome(result.outcome), diff --git a/apps/server/src/features/providers/composition/register-providers.ts b/apps/server/src/features/providers/composition/register-providers.ts index 7ad724a34..6f7011140 100644 --- a/apps/server/src/features/providers/composition/register-providers.ts +++ b/apps/server/src/features/providers/composition/register-providers.ts @@ -1,5 +1,4 @@ import { instanceCachingFactory, Lifecycle, type DependencyContainer } from "tsyringe"; -import type { Database } from "bun:sqlite"; import { hostRuntime } from "@mcode/shared/node/host-runtime"; import { ClaudeProvider } from "../adapters/claude/claude-provider.js"; @@ -9,8 +8,7 @@ import { ProviderRegistry } from "./provider-registry.js"; import { createProviderHostPorts } from "./provider-host-ports.js"; import { BrowserAutomationSessionLease } from "../../browser-automation/index.js"; import { InternalThreadControlMcpRuntime } from "../../thread-control/index.js"; -import { publishCanonicalAgentEvents } from "../../agents/canonical/canonical-agent-boundary.js"; -import { CanonicalAgentWriterClient } from "../../agents/canonical/canonical-agent-writer-client.js"; +import { CanonicalAgentBoundary } from "../../agents/index.js"; import { ScopedPreGrantService } from "../../agents/permissions/scoped-pre-grant.js"; import { EnvService } from "../../../runtime/environment/env-service.js"; import type { JobObject } from "../../../runtime/process/containment/job-object.js"; @@ -30,9 +28,6 @@ import { /** Register provider adapters, the provider registry, and provider host ports. */ export function registerProviderAdapters(container: DependencyContainer): void { - container.register(CanonicalAgentWriterClient, { - useFactory: instanceCachingFactory((c) => new CanonicalAgentWriterClient(c.resolve("Database").filename)), - }); container.register(PROVIDER_EVENT_WORKER_POOL, { useFactory: instanceCachingFactory(() => new ThreadEventWorkerPool()), }); @@ -93,8 +88,7 @@ export function registerProviderAdapters(container: DependencyContainer): void { browser: c.resolve(BrowserAutomationSessionLease), threadControl: c.resolve(InternalThreadControlMcpRuntime), grants: c.resolve(ScopedPreGrantService), - events: c.resolve(CanonicalAgentWriterClient), - publishCanonicalEvents: publishCanonicalAgentEvents, + events: c.resolve(CanonicalAgentBoundary), ingress: c.resolve(ProviderEventIngress), }), }); From a8aa3bb1017b835e52c5f9b789464f0ea96e5b44 Mon Sep 17 00:00:00 2001 From: "(CJ) Chukwudi Nwobodo" <142016413+chuks-qua@users.noreply.github.com> Date: Thu, 24 Sep 2026 20:09:43 +0100 Subject: [PATCH 035/136] feat(server): persist supported execution semantics in canonical writer --- ...anonical-execution-semantic-writer.test.ts | 188 +++++++++++++ .../canonical/canonical-agent-boundary.ts | 10 +- .../canonical-execution-semantic-writer.ts | 258 ++++++++++++++++++ .../canonical/canonical-parent-turn-write.ts | 7 +- 4 files changed, 458 insertions(+), 5 deletions(-) create mode 100644 apps/server/src/features/agents/canonical/__tests__/canonical-execution-semantic-writer.test.ts create mode 100644 apps/server/src/features/agents/canonical/canonical-execution-semantic-writer.ts diff --git a/apps/server/src/features/agents/canonical/__tests__/canonical-execution-semantic-writer.test.ts b/apps/server/src/features/agents/canonical/__tests__/canonical-execution-semantic-writer.test.ts new file mode 100644 index 000000000..6d435e4a5 --- /dev/null +++ b/apps/server/src/features/agents/canonical/__tests__/canonical-execution-semantic-writer.test.ts @@ -0,0 +1,188 @@ +import "reflect-metadata"; +import * as NodeFS from "node:fs"; +import * as NodeOS from "node:os"; +import * as NodePath from "node:path"; +import type { Database } from "bun:sqlite"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; + +import { openDatabase } from "../../../../runtime/persistence/sqlite/database.js"; +import { MessageRepo } from "../../conversation/persistence/message-repo.js"; +import type { ExecutionSemanticOperation } from "../../execution/execution-worker-handler.js"; +import { ExecutionWorkerHandler } from "../../execution/execution-worker-handler.js"; +import { CanonicalExecutionSemanticWriter } from "../canonical-execution-semantic-writer.js"; +import type { DataOnlyParentTurnStartInput } from "../canonical-parent-turn-write.js"; + +const THREAD_ID = "thread-1"; +const TURN_ID = "turn-1"; +const EXECUTION_ID = "00000000-0000-4000-8000-000000000001"; +const NOW = "2026-09-24T10:00:00.000Z"; +const execution = { threadId: THREAD_ID, turnId: TURN_ID, executionId: EXECUTION_ID } as const; +const lease = { ownerEpoch: 1, workerIndex: 0, workerGeneration: 1, leaseId: "lease-1" } as const; + +function seedThread(db: Database): void { + db.prepare("INSERT INTO workspaces (id, name, path, created_at, updated_at) VALUES (?, ?, ?, ?, ?)") + .run("workspace-1", "Workspace", "C:/fixture", NOW, NOW); + db.prepare("INSERT INTO threads (id, workspace_id, title, branch, provider, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?)") + .run(THREAD_ID, "workspace-1", "Thread", "main", "codex", NOW, NOW); +} + +function startInput(): DataOnlyParentTurnStartInput { + return { + thread: { id: THREAD_ID, workspaceId: "workspace-1", providerId: "codex", createdAt: NOW }, + turnId: TURN_ID, + executionId: EXECUTION_ID, + permissionMode: "supervised", + providerIdentities: [], + userMessage: { kind: "create", messageId: "user-1", content: "Question", sequence: 1 }, + }; +} + +function event() { + return { + eventId: `${EXECUTION_ID}:item-1`, + routing: { ...execution, itemId: "item-1" }, + sourceProviderId: "codex", + sourceIdentities: [], + payload: { + type: "item.recorded" as const, + item: { + id: "item-1", + threadId: THREAD_ID, + turnId: TURN_ID, + kind: "message" as const, + providerIdentities: [], + payload: { projection: "message", content: "Event" }, + createdAt: NOW, + updatedAt: NOW, + }, + }, + }; +} + +function operation(ordinal: number, mutation: ExecutionSemanticOperation["mutation"]): ExecutionSemanticOperation { + return { operationId: `${lease.leaseId}:${ordinal}`, execution, lease, ordinal, mutation }; +} + +describe("CanonicalExecutionSemanticWriter through ExecutionWorkerHandler", () => { + let directory: string; + let path: string; + let db: Database; + let published: string[]; + let writer: CanonicalExecutionSemanticWriter; + let handler: ExecutionWorkerHandler; + + beforeEach(() => { + directory = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "mcode-semantic-write-")); + path = NodePath.join(directory, "mcode.db"); + db = openDatabase({ dbPath: path }); + seedThread(db); + published = []; + writer = new CanonicalExecutionSemanticWriter(db, (events) => { + published.push(...events.map((item) => item.eventId)); + }); + handler = new ExecutionWorkerHandler(writer); + }); + + afterEach(() => { + db.close(true); + NodeFS.rmSync(directory, { recursive: true, force: true }); + }); + + async function send(ordinal: number, command: Parameters[0]["command"]) { + return (await handler.handle({ requestId: ordinal, execution, lease, ordinal, command })).result; + } + + it("commits start, event, and finalization with durable receipts across a database reload", async () => { + const begin = operation(1, { kind: "begin", providerId: "codex", input: startInput() }); + expect((await send(1, { kind: "start", providerId: "codex", input: startInput() })).kind).toBe("committed"); + expect((await send(2, { kind: "event", events: [event()] })).kind).toBe("committed"); + const staged = new MessageRepo(db).create(THREAD_ID, "assistant", "Answer", 2, undefined, undefined, undefined, "model", true); + const finish = { + threadId: THREAD_ID, + turnId: TURN_ID, + executionId: EXECUTION_ID, + providerId: "codex", + providerIdentities: [], + outcome: "completed" as const, + projection: { message: staged, narrative: [] }, + }; + const terminal = await send(3, { kind: "finalize", outcome: "completed", input: finish }); + expect(terminal.kind).toBe("committed"); + expect(published).toContain(`${EXECUTION_ID}:turn.completed`); + + db.close(true); + db = openDatabase({ dbPath: path }); + writer = new CanonicalExecutionSemanticWriter(db, () => {}); + expect(await writer.transact(begin)).toMatchObject({ kind: "committed", operationId: "lease-1:1" }); + expect(await writer.transact(operation(1, { + kind: "begin", + providerId: "codex", + input: { ...startInput(), permissionMode: "full" }, + }))).toEqual({ kind: "conflict", operationId: "lease-1:1" }); + expect(await writer.transact(operation(3, { kind: "finish", outcome: "completed", input: finish }))) + .toEqual(terminal); + expect(db.prepare("SELECT terminal_outcome FROM canonical_agent_ingest_checkpoints WHERE execution_id = ?").get(EXECUTION_ID)) + .toEqual({ terminal_outcome: "completed" }); + expect(db.prepare("SELECT kind, receipt_json FROM canonical_writer_operation_receipts WHERE execution_id = ? AND operation_id = ?") + .get(EXECUTION_ID, "semantic:head")).toMatchObject({ kind: "semantic-head" }); + expect(db.prepare("SELECT COUNT(*) AS count FROM canonical_writer_operation_receipts WHERE execution_id = ?").get(EXECUTION_ID)) + .toEqual({ count: 4 }); + }); + + it("rejects stale leases, skipped ordinals and unsupported commands before canonical mutation", async () => { + expect((await send(1, { kind: "start", providerId: "codex", input: startInput() })).kind).toBe("committed"); + const eventsBefore = db.prepare("SELECT COUNT(*) AS count FROM canonical_agent_events WHERE execution_id = ?").get(EXECUTION_ID); + const append = operation(2, { kind: "append-events", events: [event()] }); + expect(await writer.transact({ ...append, lease: { ...lease, ownerEpoch: 2, leaseId: "lease-2" }, operationId: "lease-2:2" })) + .toEqual({ kind: "conflict", operationId: "lease-2:2" }); + expect(await writer.transact({ ...append, ordinal: 3, operationId: "lease-1:3" })) + .toEqual({ kind: "conflict", operationId: "lease-1:3" }); + expect(await send(2, { kind: "checkpoint", phase: "running", nativeCursor: null })) + .toEqual({ kind: "rejected", reason: "writer-conflict" }); + expect(db.prepare("SELECT COUNT(*) AS count FROM canonical_agent_events WHERE execution_id = ?").get(EXECUTION_ID)) + .toEqual(eventsBefore); + expect(db.prepare("SELECT receipt_json FROM canonical_writer_operation_receipts WHERE execution_id = ? AND operation_id = ?") + .get(EXECUTION_ID, "semantic:head")).toMatchObject({ receipt_json: expect.stringContaining('"ordinal":1') }); + }); + + it("rolls back terminal checkpoint and semantic receipt together when receipt storage fails", async () => { + expect((await send(1, { kind: "start", providerId: "codex", input: startInput() })).kind).toBe("committed"); + const staged = new MessageRepo(db).create(THREAD_ID, "assistant", "Answer", 2, undefined, undefined, undefined, "model", true); + const finish = { + threadId: THREAD_ID, + turnId: TURN_ID, + executionId: EXECUTION_ID, + providerId: "codex", + providerIdentities: [], + outcome: "completed" as const, + projection: { message: staged, narrative: [] }, + }; + db.run("CREATE TRIGGER fail_semantic_receipt BEFORE INSERT ON canonical_writer_operation_receipts WHEN NEW.kind = 'semantic:finish' BEGIN SELECT RAISE(ABORT, 'receipt unavailable'); END"); + await expect(send(2, { kind: "finalize", outcome: "completed", input: finish })).rejects.toThrow("receipt unavailable"); + expect(db.prepare("SELECT terminal_outcome FROM canonical_agent_ingest_checkpoints WHERE execution_id = ?").get(EXECUTION_ID)) + .toEqual({ terminal_outcome: null }); + expect(db.prepare("SELECT operation_id FROM canonical_writer_operation_receipts WHERE execution_id = ? AND operation_id = ?") + .get(EXECUTION_ID, "lease-1:2")).toBeNull(); + expect(new MessageRepo(db).findByIdInThread(THREAD_ID, staged.id)).toBeNull(); + expect(published).not.toContain(`${EXECUTION_ID}:turn.completed`); + + db.run("DROP TRIGGER fail_semantic_receipt"); + expect((await send(2, { kind: "finalize", outcome: "completed", input: finish })).kind).toBe("committed"); + }); + + it("rolls back an event and its checkpoint when its semantic receipt cannot be stored", async () => { + expect((await send(1, { kind: "start", providerId: "codex", input: startInput() })).kind).toBe("committed"); + const before = db.prepare("SELECT last_durable_sequence FROM canonical_agent_ingest_checkpoints WHERE execution_id = ?") + .get(EXECUTION_ID); + db.run("CREATE TRIGGER fail_semantic_event_receipt BEFORE INSERT ON canonical_writer_operation_receipts WHEN NEW.kind = 'semantic:append-events' BEGIN SELECT RAISE(ABORT, 'event receipt unavailable'); END"); + await expect(send(2, { kind: "event", events: [event()] })).rejects.toThrow("event receipt unavailable"); + expect(db.prepare("SELECT last_durable_sequence FROM canonical_agent_ingest_checkpoints WHERE execution_id = ?") + .get(EXECUTION_ID)).toEqual(before); + expect(db.prepare("SELECT id FROM canonical_agent_items WHERE id = ?").get("item-1")).toBeNull(); + expect(published).not.toContain(event().eventId); + + db.run("DROP TRIGGER fail_semantic_event_receipt"); + expect((await send(2, { kind: "event", events: [event()] })).kind).toBe("committed"); + expect(published).toContain(event().eventId); + }); +}); diff --git a/apps/server/src/features/agents/canonical/canonical-agent-boundary.ts b/apps/server/src/features/agents/canonical/canonical-agent-boundary.ts index 72acd8995..d56f774da 100644 --- a/apps/server/src/features/agents/canonical/canonical-agent-boundary.ts +++ b/apps/server/src/features/agents/canonical/canonical-agent-boundary.ts @@ -2026,15 +2026,16 @@ export class CanonicalAgentBoundary implements ParentTurnDurability, CodexCollab }); } - /** Persist a terminal parent turn in bounded transactions and confirm it only in the final batch. */ + /** Persist a terminal parent turn in bounded transactions; the writer-local hook joins its final transaction. */ async finishParentTurnBatched( input: CanonicalParentTurnFinishInput, + onTerminalCommit?: (durableSequence: number) => void, ): Promise { const checkpoint = this.loadCheckpoint(input.executionId); if (checkpoint?.terminalOutcome) { return this.confirmedParentTerminalBatch(checkpoint, input.threadId); } - return this.writeParentTerminalBatches(input, checkpoint); + return this.writeParentTerminalBatches(input, checkpoint, onTerminalCommit); } private confirmedParentTerminalBatch( @@ -2057,6 +2058,7 @@ export class CanonicalAgentBoundary implements ParentTurnDurability, CodexCollab private async writeParentTerminalBatches( input: CanonicalParentTurnFinishInput, checkpoint: CanonicalAgentCheckpoint | null, + onTerminalCommit?: (durableSequence: number) => void, ): Promise { const projection = input.projectTurn(); const endedAt = new Date().toISOString(); @@ -2084,7 +2086,7 @@ export class CanonicalAgentBoundary implements ParentTurnDurability, CodexCollab terminalRevision, terminalEventId, ), - onBatchFinishing: () => this.finishParentTerminalBatch(state, input, terminalRevision), + onBatchFinishing: () => this.finishParentTerminalBatch(state, input, terminalRevision, onTerminalCommit), onBatchCommitted: () => this.publishParentTerminalBatch(state), }); return this.parentTerminalBatchResult(state, writeBatches); @@ -2254,6 +2256,7 @@ export class CanonicalAgentBoundary implements ParentTurnDurability, CodexCollab state: ParentTerminalBatchState, input: CanonicalParentTurnFinishInput, terminalRevision: number, + onTerminalCommit?: (durableSequence: number) => void, ): void { const modelState = this.parentTerminalBatchModelState(state); if (!state.wrote) { @@ -2265,6 +2268,7 @@ export class CanonicalAgentBoundary implements ParentTurnDurability, CodexCollab if (state.terminal) { this.displayMaterializer.materializeItems(this.parentAssistantItemIds(input.turnId)); this.retireParentNarrativeRecovery(input.turnId); + onTerminalCommit?.(state.acceptedSequence); } state.latest = this.committedParentTerminalBatchResult(state, input.threadId, terminalRevision); } diff --git a/apps/server/src/features/agents/canonical/canonical-execution-semantic-writer.ts b/apps/server/src/features/agents/canonical/canonical-execution-semantic-writer.ts new file mode 100644 index 000000000..fc75a4030 --- /dev/null +++ b/apps/server/src/features/agents/canonical/canonical-execution-semantic-writer.ts @@ -0,0 +1,258 @@ +import type { Database } from "bun:sqlite"; +import * as NodeCrypto from "node:crypto"; +import { z } from "zod"; + +import type { ExecutionIdentity, ExecutionLease } from "../execution/execution-mailbox-protocol.js"; +import type { + ExecutionSemanticOperation, + ExecutionSemanticWriter, + ExecutionWriteReceipt, +} from "../execution/execution-worker-handler.js"; +import type { CanonicalAgentEventPublisher } from "./canonical-agent-boundary.js"; +import { CanonicalParentTurnWrite } from "./canonical-parent-turn-write.js"; + +const HEAD_ID = "semantic:head"; +const HEAD_KIND = "semantic-head"; +const storedHeadSchema = z.object({ + execution: z.object({ threadId: z.string(), turnId: z.string(), executionId: z.string() }), + providerId: z.string(), + lease: z.object({ + ownerEpoch: z.number().int(), + workerIndex: z.number().int(), + workerGeneration: z.number().int(), + leaseId: z.string(), + }), + ordinal: z.number().int(), + durableRevision: z.number().int(), + terminal: z.boolean(), +}); +const storedReceiptSchema = z.object({ + kind: z.literal("committed"), + operationId: z.string(), + durableRevision: z.number().int(), +}); + +interface SemanticHead { + readonly execution: ExecutionIdentity; + readonly providerId: string; + readonly lease: ExecutionLease; + readonly ordinal: number; + readonly durableRevision: number; + readonly terminal: boolean; +} + +interface StoredOperation { + readonly kind: string; + readonly input_hash: string; + readonly receipt_json: string; +} + +class SemanticConflict extends Error {} + +/** Adapts the supported execution mutations to one writer-local canonical SQLite connection. */ +export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter { + private readonly turns: CanonicalParentTurnWrite; + private readonly findOperation: ReturnType; + private readonly insertOperation: ReturnType; + private readonly updateHead: ReturnType; + private bufferedPublication: Parameters[0][] | null = null; + + constructor(private readonly db: Database, private readonly publish: CanonicalAgentEventPublisher) { + this.turns = new CanonicalParentTurnWrite(db, (events) => { + if (this.bufferedPublication) this.bufferedPublication.push(events); + else this.publish(events); + }); + this.findOperation = db.prepare("SELECT kind, input_hash, receipt_json FROM canonical_writer_operation_receipts WHERE execution_id = ? AND operation_id = ?"); + this.insertOperation = db.prepare("INSERT INTO canonical_writer_operation_receipts (execution_id, operation_id, kind, input_hash, receipt_json) VALUES (?, ?, ?, ?, ?)"); + this.updateHead = db.prepare("UPDATE canonical_writer_operation_receipts SET receipt_json = ? WHERE execution_id = ? AND operation_id = ? AND kind = ?"); + } + + /** Commit a supported operation with a durable receipt, or reject it without changing canonical state. */ + async transact(operation: ExecutionSemanticOperation): Promise { + if (!validOperation(operation)) return conflict(operation); + const hash = fingerprint(operation); + const replay = this.findOperation.get(operation.execution.executionId, operation.operationId) as StoredOperation | null; + if (replay) return this.replay(operation, hash, replay); + try { + if (operation.mutation.kind === "begin") return this.begin(operation, hash); + if (operation.mutation.kind === "append-events") return this.append(operation, hash); + if (operation.mutation.kind === "finish") return await this.finish(operation, hash); + return conflict(operation); + } catch (error) { + if (error instanceof SemanticConflict) return conflict(operation); + throw error; + } + } + + private begin(operation: ExecutionSemanticOperation, hash: string): ExecutionWriteReceipt { + const mutation = operation.mutation; + if (mutation.kind !== "begin" || operation.ordinal !== 1 || this.loadHead(operation.execution.executionId)) { + return conflict(operation); + } + if (mutation.providerId !== mutation.input.thread.providerId + || mutation.input.thread.id !== operation.execution.threadId + || mutation.input.turnId !== operation.execution.turnId + || mutation.input.executionId !== operation.execution.executionId) return conflict(operation); + return this.withBufferedPublication(() => this.db.transaction(() => { + if (this.loadHead(operation.execution.executionId)) throw new SemanticConflict(); + const result = this.turns.start(mutation.input); + if (result.outcome !== "committed") throw new SemanticConflict(); + const receipt = committed(operation, result.durableThrough); + const head: SemanticHead = { + execution: operation.execution, + providerId: mutation.providerId, + lease: operation.lease, + ordinal: operation.ordinal, + durableRevision: receipt.durableRevision, + terminal: false, + }; + this.insertOperation.run(operation.execution.executionId, HEAD_ID, HEAD_KIND, fingerprint(head.lease), JSON.stringify(head)); + this.storeReceipt(operation, hash, receipt); + return receipt; + })()); + } + + private append(operation: ExecutionSemanticOperation, hash: string): ExecutionWriteReceipt { + const mutation = operation.mutation; + if (mutation.kind !== "append-events" || mutation.events.length === 0 + || mutation.events.some((event) => event.routing.threadId !== operation.execution.threadId + || event.routing.turnId !== operation.execution.turnId + || event.routing.executionId !== operation.execution.executionId)) return conflict(operation); + return this.withBufferedPublication(() => this.db.transaction(() => { + const head = this.requireNextHead(operation); + const result = this.turns.append({ + ...operation.execution, + phase: "running", + events: mutation.events, + }); + if (result.outcome !== "committed") throw new SemanticConflict(); + const receipt = committed(operation, result.durableThrough); + this.storeHead({ ...head, ordinal: operation.ordinal, durableRevision: receipt.durableRevision }); + this.storeReceipt(operation, hash, receipt); + return receipt; + })()); + } + + private async finish(operation: ExecutionSemanticOperation, hash: string): Promise { + const mutation = operation.mutation; + if (mutation.kind !== "finish" || !validFinish(operation, mutation)) return conflict(operation); + const head = this.loadHead(operation.execution.executionId); + if (!head || !nextHead(head, operation) || head.providerId !== mutation.input.providerId) return conflict(operation); + const result = await this.turns.finish(mutation.input, (durableSequence) => { + const current = this.requireNextHead(operation); + if (current.providerId !== mutation.input.providerId) throw new SemanticConflict(); + const receipt = committed(operation, durableSequence); + this.storeHead({ ...current, ordinal: operation.ordinal, durableRevision: receipt.durableRevision, terminal: true }); + this.storeReceipt(operation, hash, receipt); + }); + if (result.outcome !== "committed") return conflict(operation); + const stored = this.findOperation.get(operation.execution.executionId, operation.operationId) as StoredOperation | null; + return stored ? this.replay(operation, hash, stored) : conflict(operation); + } + + private requireNextHead(operation: ExecutionSemanticOperation): SemanticHead { + const head = this.loadHead(operation.execution.executionId); + if (!head || !nextHead(head, operation)) throw new SemanticConflict(); + return head; + } + + private loadHead(executionId: string): SemanticHead | null { + const row = this.findOperation.get(executionId, HEAD_ID) as StoredOperation | null; + if (!row) return null; + if (row.kind !== HEAD_KIND) throw new SemanticConflict(); + return storedHeadSchema.parse(JSON.parse(row.receipt_json)); + } + + private storeHead(head: SemanticHead): void { + this.updateHead.run(JSON.stringify(head), head.execution.executionId, HEAD_ID, HEAD_KIND); + } + + private storeReceipt(operation: ExecutionSemanticOperation, hash: string, receipt: ExecutionWriteReceipt): void { + this.insertOperation.run( + operation.execution.executionId, + operation.operationId, + `semantic:${operation.mutation.kind}`, + hash, + JSON.stringify(receipt), + ); + } + + private replay(operation: ExecutionSemanticOperation, hash: string, stored: StoredOperation): ExecutionWriteReceipt { + if (stored.kind !== `semantic:${operation.mutation.kind}` || stored.input_hash !== hash) return conflict(operation); + const receipt = storedReceiptSchema.parse(JSON.parse(stored.receipt_json)); + return receipt.operationId === operation.operationId && Number.isSafeInteger(receipt.durableRevision) + ? receipt : conflict(operation); + } + + private withBufferedPublication(write: () => T): T { + const pending: Parameters[0][] = []; + this.bufferedPublication = pending; + try { + const result = write(); + this.bufferedPublication = null; + for (const events of pending) this.publish(events); + return result; + } finally { + this.bufferedPublication = null; + } + } +} + +function validOperation(operation: ExecutionSemanticOperation): boolean { + return operation.operationId === `${operation.lease.leaseId}:${operation.ordinal}` + && operation.operationId !== HEAD_ID && operation.operationId.length <= 256 + && Number.isSafeInteger(operation.ordinal) && operation.ordinal > 0 + && validIdentity(operation.execution) && validLease(operation.lease); +} + +function validIdentity(identity: ExecutionIdentity): boolean { + return Boolean(identity.threadId && identity.turnId && identity.executionId); +} + +function validLease(lease: ExecutionLease): boolean { + return Boolean(lease.leaseId) && Number.isSafeInteger(lease.ownerEpoch) && lease.ownerEpoch >= 0 + && Number.isSafeInteger(lease.workerIndex) && lease.workerIndex >= 0 + && Number.isSafeInteger(lease.workerGeneration) && lease.workerGeneration >= 0; +} + +function validFinish( + operation: ExecutionSemanticOperation, + mutation: Extract, +): boolean { + return mutation.outcome === mutation.input.outcome + && mutation.input.threadId === operation.execution.threadId + && mutation.input.turnId === operation.execution.turnId + && mutation.input.executionId === operation.execution.executionId; +} + +function nextHead(head: SemanticHead, operation: ExecutionSemanticOperation): boolean { + const lease = head.lease; + const candidate = operation.lease; + return !head.terminal && head.execution.threadId === operation.execution.threadId + && head.execution.turnId === operation.execution.turnId + && head.ordinal + 1 === operation.ordinal + && lease.ownerEpoch === candidate.ownerEpoch && lease.workerIndex === candidate.workerIndex + && lease.workerGeneration === candidate.workerGeneration && lease.leaseId === candidate.leaseId; +} + +function committed( + operation: ExecutionSemanticOperation, + durableRevision: number, +): Extract { + return { kind: "committed", operationId: operation.operationId, durableRevision }; +} + +function conflict(operation: ExecutionSemanticOperation): ExecutionWriteReceipt { + return { kind: "conflict", operationId: operation.operationId }; +} + +function fingerprint(value: unknown): string { + return NodeCrypto.createHash("sha256").update(JSON.stringify(sortJson(value))).digest("hex"); +} + +function sortJson(value: unknown): unknown { + if (Array.isArray(value)) return value.map(sortJson); + if (value === null || typeof value !== "object") return value; + return Object.fromEntries(Object.entries(value).sort(([a], [b]) => a.localeCompare(b)) + .map(([key, entry]) => [key, sortJson(entry)])); +} diff --git a/apps/server/src/features/agents/canonical/canonical-parent-turn-write.ts b/apps/server/src/features/agents/canonical/canonical-parent-turn-write.ts index 7ae75a3c0..35ffb3686 100644 --- a/apps/server/src/features/agents/canonical/canonical-parent-turn-write.ts +++ b/apps/server/src/features/agents/canonical/canonical-parent-turn-write.ts @@ -92,7 +92,10 @@ export class CanonicalParentTurnWrite { } /** Confirm the terminal checkpoint and publish the staged assistant in one transaction. */ - finish(input: DataOnlyParentTurnFinishInput): Promise { + finish( + input: DataOnlyParentTurnFinishInput, + onTerminalCommit?: (durableSequence: number) => void, + ): Promise { this.assertStagedAssistant(input); const projection: ParentTurnProjection = { message: input.projection.message @@ -113,7 +116,7 @@ export class CanonicalParentTurnWrite { this.messages.setAssistantOutcome(projection.message.id, input.outcome, input.executionId); this.messages.publishAssistant(projection.message.id); }, - }); + }, onTerminalCommit); } private projectUserMessage(input: DataOnlyParentTurnStartInput) { From 247cca94d04f31e419c1989d1fcaff74db9a8303 Mon Sep 17 00:00:00 2001 From: "(CJ) Chukwudi Nwobodo" <142016413+chuks-qua@users.noreply.github.com> Date: Thu, 24 Sep 2026 20:10:13 +0100 Subject: [PATCH 036/136] test(server): prove semantic begin receipt rollback --- .../canonical-execution-semantic-writer.test.ts | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/apps/server/src/features/agents/canonical/__tests__/canonical-execution-semantic-writer.test.ts b/apps/server/src/features/agents/canonical/__tests__/canonical-execution-semantic-writer.test.ts index 6d435e4a5..353ee1c0a 100644 --- a/apps/server/src/features/agents/canonical/__tests__/canonical-execution-semantic-writer.test.ts +++ b/apps/server/src/features/agents/canonical/__tests__/canonical-execution-semantic-writer.test.ts @@ -185,4 +185,18 @@ describe("CanonicalExecutionSemanticWriter through ExecutionWorkerHandler", () = expect((await send(2, { kind: "event", events: [event()] })).kind).toBe("committed"); expect(published).toContain(event().eventId); }); + + it("rolls back the canonical start and user message when the begin receipt fails", async () => { + db.run("CREATE TRIGGER fail_semantic_begin_receipt BEFORE INSERT ON canonical_writer_operation_receipts WHEN NEW.kind = 'semantic:begin' BEGIN SELECT RAISE(ABORT, 'begin receipt unavailable'); END"); + await expect(send(1, { kind: "start", providerId: "codex", input: startInput() })) + .rejects.toThrow("begin receipt unavailable"); + expect(db.prepare("SELECT execution_id FROM canonical_agent_turns WHERE execution_id = ?").get(EXECUTION_ID)).toBeNull(); + expect(new MessageRepo(db).findByIdInThread(THREAD_ID, "user-1")).toBeNull(); + expect(db.prepare("SELECT operation_id FROM canonical_writer_operation_receipts WHERE execution_id = ?") + .all(EXECUTION_ID)).toEqual([]); + expect(published).toEqual([]); + + db.run("DROP TRIGGER fail_semantic_begin_receipt"); + expect((await send(1, { kind: "start", providerId: "codex", input: startInput() })).kind).toBe("committed"); + }); }); From 618497c51c3d629320352e9064d0cbe0c466709f Mon Sep 17 00:00:00 2001 From: "(CJ) Chukwudi Nwobodo" <142016413+chuks-qua@users.noreply.github.com> Date: Thu, 24 Sep 2026 20:17:40 +0100 Subject: [PATCH 037/136] fix(server): validate semantic writer rows and bound publication scope --- ...anonical-execution-semantic-writer.test.ts | 75 +++++++++++++++++++ .../canonical-execution-semantic-writer.ts | 25 ++++--- .../canonical/canonical-parent-turn-write.ts | 4 +- 3 files changed, 94 insertions(+), 10 deletions(-) diff --git a/apps/server/src/features/agents/canonical/__tests__/canonical-execution-semantic-writer.test.ts b/apps/server/src/features/agents/canonical/__tests__/canonical-execution-semantic-writer.test.ts index 353ee1c0a..e0456e843 100644 --- a/apps/server/src/features/agents/canonical/__tests__/canonical-execution-semantic-writer.test.ts +++ b/apps/server/src/features/agents/canonical/__tests__/canonical-execution-semantic-writer.test.ts @@ -199,4 +199,79 @@ describe("CanonicalExecutionSemanticWriter through ExecutionWorkerHandler", () = db.run("DROP TRIGGER fail_semantic_begin_receipt"); expect((await send(1, { kind: "start", providerId: "codex", input: startInput() })).kind).toBe("committed"); }); + + it("keeps publication separate while a batched finish starts another execution", async () => { + const otherExecution = { + threadId: "thread-2", + turnId: "turn-2", + executionId: "00000000-0000-4000-8000-000000000002", + }; + const otherLease = { ...lease, workerIndex: 1, leaseId: "lease-2" }; + db.prepare("INSERT INTO threads (id, workspace_id, title, branch, provider, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?)") + .run(otherExecution.threadId, "workspace-1", "Second", "main", "codex", NOW, NOW); + let finishing = false; + let secondStartRequested = false; + let secondStart: Promise | null = null; + writer = new CanonicalExecutionSemanticWriter(db, (events) => { + published.push(...events.map((item) => item.eventId)); + if (!finishing || secondStartRequested) return; + secondStartRequested = true; + secondStart = handler.handle({ + requestId: 1, + execution: otherExecution, + lease: otherLease, + ordinal: 1, + command: { + kind: "start", + providerId: "codex", + input: { + ...startInput(), + thread: { ...startInput().thread, id: otherExecution.threadId }, + turnId: otherExecution.turnId, + executionId: otherExecution.executionId, + userMessage: { kind: "create", messageId: "user-2", content: "Second question", sequence: 1 }, + }, + }, + }).then((reply) => reply.result); + }); + handler = new ExecutionWorkerHandler(writer); + expect((await send(1, { kind: "start", providerId: "codex", input: startInput() })).kind).toBe("committed"); + const staged = new MessageRepo(db).create(THREAD_ID, "assistant", "Answer", 2, undefined, undefined, undefined, "model", true); + const narrative = Array.from({ length: 100 }, (_, index) => ({ + kind: "toolCall" as const, + sequence: 2, + sortOrder: index, + record: { + id: `tool-${index}`, + message_id: staged.id, + parent_tool_call_id: null, + tool_name: "Read", + input_summary: `file-${index}`, + output_summary: "ok", + status: "completed" as const, + started_at: NOW, + completed_at: NOW, + sort_order: index, + }, + })); + finishing = true; + expect((await send(2, { + kind: "finalize", + outcome: "completed", + input: { + threadId: THREAD_ID, + turnId: TURN_ID, + executionId: EXECUTION_ID, + providerId: "codex", + providerIdentities: [], + outcome: "completed", + projection: { message: staged, narrative }, + }, + })).kind).toBe("committed"); + expect(await secondStart).toMatchObject({ kind: "committed", operationId: "lease-2:1" }); + expect(published).toContain(`${EXECUTION_ID}:turn.completed`); + expect(published.some((eventId) => eventId.startsWith(otherExecution.executionId))).toBe(true); + expect(db.prepare("SELECT COUNT(*) AS count FROM canonical_writer_operation_receipts WHERE kind = 'semantic-head'").get()) + .toEqual({ count: 2 }); + }); }); diff --git a/apps/server/src/features/agents/canonical/canonical-execution-semantic-writer.ts b/apps/server/src/features/agents/canonical/canonical-execution-semantic-writer.ts index fc75a4030..678ddd0d3 100644 --- a/apps/server/src/features/agents/canonical/canonical-execution-semantic-writer.ts +++ b/apps/server/src/features/agents/canonical/canonical-execution-semantic-writer.ts @@ -31,6 +31,11 @@ const storedReceiptSchema = z.object({ operationId: z.string(), durableRevision: z.number().int(), }); +const storedOperationSchema = z.object({ + kind: z.string(), + input_hash: z.string(), + receipt_json: z.string(), +}); interface SemanticHead { readonly execution: ExecutionIdentity; @@ -41,11 +46,7 @@ interface SemanticHead { readonly terminal: boolean; } -interface StoredOperation { - readonly kind: string; - readonly input_hash: string; - readonly receipt_json: string; -} +type StoredOperation = z.infer; class SemanticConflict extends Error {} @@ -71,7 +72,7 @@ export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter async transact(operation: ExecutionSemanticOperation): Promise { if (!validOperation(operation)) return conflict(operation); const hash = fingerprint(operation); - const replay = this.findOperation.get(operation.execution.executionId, operation.operationId) as StoredOperation | null; + const replay = this.loadOperation(operation.execution.executionId, operation.operationId); if (replay) return this.replay(operation, hash, replay); try { if (operation.mutation.kind === "begin") return this.begin(operation, hash); @@ -146,7 +147,7 @@ export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter this.storeReceipt(operation, hash, receipt); }); if (result.outcome !== "committed") return conflict(operation); - const stored = this.findOperation.get(operation.execution.executionId, operation.operationId) as StoredOperation | null; + const stored = this.loadOperation(operation.execution.executionId, operation.operationId); return stored ? this.replay(operation, hash, stored) : conflict(operation); } @@ -157,12 +158,16 @@ export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter } private loadHead(executionId: string): SemanticHead | null { - const row = this.findOperation.get(executionId, HEAD_ID) as StoredOperation | null; + const row = this.loadOperation(executionId, HEAD_ID); if (!row) return null; if (row.kind !== HEAD_KIND) throw new SemanticConflict(); return storedHeadSchema.parse(JSON.parse(row.receipt_json)); } + private loadOperation(executionId: string, operationId: string): StoredOperation | null { + return storedOperationSchema.nullable().parse(this.findOperation.get(executionId, operationId)); + } + private storeHead(head: SemanticHead): void { this.updateHead.run(JSON.stringify(head), head.execution.executionId, HEAD_ID, HEAD_KIND); } @@ -184,7 +189,9 @@ export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter ? receipt : conflict(operation); } - private withBufferedPublication(write: () => T): T { + // Only the synchronous begin and append transactions use this buffer. Finish publishes after each committed batch. + private withBufferedPublication(write: () => ExecutionWriteReceipt): ExecutionWriteReceipt { + if (this.bufferedPublication) throw new Error("Nested semantic publication buffer"); const pending: Parameters[0][] = []; this.bufferedPublication = pending; try { diff --git a/apps/server/src/features/agents/canonical/canonical-parent-turn-write.ts b/apps/server/src/features/agents/canonical/canonical-parent-turn-write.ts index 35ffb3686..3c932bebb 100644 --- a/apps/server/src/features/agents/canonical/canonical-parent-turn-write.ts +++ b/apps/server/src/features/agents/canonical/canonical-parent-turn-write.ts @@ -1,4 +1,5 @@ import type { Database } from "bun:sqlite"; +import { z } from "zod"; import { MessageRepo } from "../conversation/persistence/message-repo.js"; import { PlanQuestionAnswersRepo } from "../planning/persistence/plan-question-answers-repo.js"; @@ -18,6 +19,7 @@ import { } from "./canonical-agent-boundary.js"; type CreateMessageArgument = Parameters; +const stagedAssistantSchema = z.object({ role: z.string(), content: z.string() }); /** User-message data the writer can project inside the canonical start transaction. */ export type ParentUserMessageWrite = @@ -149,7 +151,7 @@ export class CanonicalParentTurnWrite { private assertStagedAssistant(input: DataOnlyParentTurnFinishInput): void { const projected = input.projection.message; if (!projected) return; - const staged = this.stagedAssistant.get(projected.id, input.threadId) as { role: string; content: string } | null; + const staged = stagedAssistantSchema.nullable().parse(this.stagedAssistant.get(projected.id, input.threadId)); if (!staged || staged.role !== "assistant" || staged.content !== projected.content) { throw new Error(`Staged assistant projection not found: ${projected.id}`); } From 763ea5b22e0cfd89dbd5c211a9a300a5e24815da Mon Sep 17 00:00:00 2001 From: "(CJ) Chukwudi Nwobodo" <142016413+chuks-qua@users.noreply.github.com> Date: Thu, 24 Sep 2026 20:27:13 +0100 Subject: [PATCH 038/136] fix(server): replay canonical events from semantic receipts --- ...anonical-execution-semantic-writer.test.ts | 172 +++++++++++++++--- .../canonical/canonical-agent-boundary.ts | 25 ++- .../canonical-execution-semantic-writer.ts | 117 ++++++++++-- .../canonical/canonical-parent-turn-write.ts | 5 +- 4 files changed, 278 insertions(+), 41 deletions(-) diff --git a/apps/server/src/features/agents/canonical/__tests__/canonical-execution-semantic-writer.test.ts b/apps/server/src/features/agents/canonical/__tests__/canonical-execution-semantic-writer.test.ts index e0456e843..c0df75dc1 100644 --- a/apps/server/src/features/agents/canonical/__tests__/canonical-execution-semantic-writer.test.ts +++ b/apps/server/src/features/agents/canonical/__tests__/canonical-execution-semantic-writer.test.ts @@ -59,6 +59,26 @@ function event() { }; } +function toolNarrative(messageId: string, count: number) { + return Array.from({ length: count }, (_, index) => ({ + kind: "toolCall" as const, + sequence: 2, + sortOrder: index, + record: { + id: `tool-${index}`, + message_id: messageId, + parent_tool_call_id: null, + tool_name: "Read", + input_summary: `file-${index}`, + output_summary: "ok", + status: "completed" as const, + started_at: NOW, + completed_at: NOW, + sort_order: index, + }, + })); +} + function operation(ordinal: number, mutation: ExecutionSemanticOperation["mutation"]): ExecutionSemanticOperation { return { operationId: `${lease.leaseId}:${ordinal}`, execution, lease, ordinal, mutation }; } @@ -125,7 +145,7 @@ describe("CanonicalExecutionSemanticWriter through ExecutionWorkerHandler", () = .toEqual({ terminal_outcome: "completed" }); expect(db.prepare("SELECT kind, receipt_json FROM canonical_writer_operation_receipts WHERE execution_id = ? AND operation_id = ?") .get(EXECUTION_ID, "semantic:head")).toMatchObject({ kind: "semantic-head" }); - expect(db.prepare("SELECT COUNT(*) AS count FROM canonical_writer_operation_receipts WHERE execution_id = ?").get(EXECUTION_ID)) + expect(db.prepare("SELECT COUNT(*) AS count FROM canonical_writer_operation_receipts WHERE execution_id = ? AND kind != 'semantic-publication'").get(EXECUTION_ID)) .toEqual({ count: 4 }); }); @@ -157,12 +177,12 @@ describe("CanonicalExecutionSemanticWriter through ExecutionWorkerHandler", () = outcome: "completed" as const, projection: { message: staged, narrative: [] }, }; - db.run("CREATE TRIGGER fail_semantic_receipt BEFORE INSERT ON canonical_writer_operation_receipts WHEN NEW.kind = 'semantic:finish' BEGIN SELECT RAISE(ABORT, 'receipt unavailable'); END"); + db.run("CREATE TRIGGER fail_semantic_receipt BEFORE UPDATE OF kind ON canonical_writer_operation_receipts WHEN NEW.kind = 'semantic:finish' BEGIN SELECT RAISE(ABORT, 'receipt unavailable'); END"); await expect(send(2, { kind: "finalize", outcome: "completed", input: finish })).rejects.toThrow("receipt unavailable"); expect(db.prepare("SELECT terminal_outcome FROM canonical_agent_ingest_checkpoints WHERE execution_id = ?").get(EXECUTION_ID)) .toEqual({ terminal_outcome: null }); - expect(db.prepare("SELECT operation_id FROM canonical_writer_operation_receipts WHERE execution_id = ? AND operation_id = ?") - .get(EXECUTION_ID, "lease-1:2")).toBeNull(); + expect(db.prepare("SELECT kind FROM canonical_writer_operation_receipts WHERE execution_id = ? AND operation_id = ?") + .get(EXECUTION_ID, "lease-1:2")).toEqual({ kind: "semantic:finish-pending" }); expect(new MessageRepo(db).findByIdInThread(THREAD_ID, staged.id)).toBeNull(); expect(published).not.toContain(`${EXECUTION_ID}:turn.completed`); @@ -186,6 +206,123 @@ describe("CanonicalExecutionSemanticWriter through ExecutionWorkerHandler", () = expect(published).toContain(event().eventId); }); + it("replays committed event publication after a failed publisher and database reopen", async () => { + let failPublication = false; + writer = new CanonicalExecutionSemanticWriter(db, (events) => { + if (failPublication) throw new Error("publisher unavailable"); + published.push(...events.map((item) => item.eventId)); + }); + handler = new ExecutionWorkerHandler(writer); + expect((await send(1, { kind: "start", providerId: "codex", input: startInput() })).kind).toBe("committed"); + failPublication = true; + await expect(send(2, { kind: "event", events: [event()] })).rejects.toThrow("publisher unavailable"); + expect(db.prepare("SELECT event_id FROM canonical_agent_events WHERE event_id = ?").get(event().eventId)) + .toEqual({ event_id: event().eventId }); + const receipt = db.prepare("SELECT receipt_json FROM canonical_writer_operation_receipts WHERE execution_id = ? AND operation_id = ?") + .get(EXECUTION_ID, "lease-1:2") as { receipt_json: string }; + expect(receipt.receipt_json.length).toBeLessThan(256); + + db.close(true); + db = openDatabase({ dbPath: path }); + published = []; + writer = new CanonicalExecutionSemanticWriter(db, (events) => { + published.push(...events.map((item) => item.eventId)); + }); + expect(await writer.transact(operation(2, { kind: "append-events", events: [event()] }))) + .toMatchObject({ kind: "committed", operationId: "lease-1:2" }); + expect(published).toContain(event().eventId); + }); + + it("replays terminal publication after its checkpoint committed and the publisher failed", async () => { + writer = new CanonicalExecutionSemanticWriter(db, (events) => { + if (events.some((item) => item.eventId === `${EXECUTION_ID}:turn.completed`)) { + throw new Error("terminal publisher unavailable"); + } + published.push(...events.map((item) => item.eventId)); + }); + handler = new ExecutionWorkerHandler(writer); + expect((await send(1, { kind: "start", providerId: "codex", input: startInput() })).kind).toBe("committed"); + const staged = new MessageRepo(db).create(THREAD_ID, "assistant", "Answer", 2, undefined, undefined, undefined, "model", true); + const finish = { + threadId: THREAD_ID, + turnId: TURN_ID, + executionId: EXECUTION_ID, + providerId: "codex", + providerIdentities: [], + outcome: "completed" as const, + projection: { message: staged, narrative: [] }, + }; + await expect(send(2, { kind: "finalize", outcome: "completed", input: finish })) + .rejects.toThrow("terminal publisher unavailable"); + expect(db.prepare("SELECT terminal_outcome FROM canonical_agent_ingest_checkpoints WHERE execution_id = ?").get(EXECUTION_ID)) + .toEqual({ terminal_outcome: "completed" }); + expect(db.prepare("SELECT kind FROM canonical_writer_operation_receipts WHERE execution_id = ? AND operation_id = ?") + .get(EXECUTION_ID, "lease-1:2")).toEqual({ kind: "semantic:finish" }); + + db.close(true); + db = openDatabase({ dbPath: path }); + published = []; + writer = new CanonicalExecutionSemanticWriter(db, (events) => { + published.push(...events.map((item) => item.eventId)); + }); + expect(await writer.transact(operation(2, { kind: "finish", outcome: "completed", input: finish }))) + .toMatchObject({ kind: "committed", operationId: "lease-1:2" }); + expect(published).toContain(`${EXECUTION_ID}:turn.completed`); + }); + + it("replays earlier terminal batches after publication failed before finalization", async () => { + let failTerminalBatch = false; + writer = new CanonicalExecutionSemanticWriter(db, (events) => { + if (failTerminalBatch) { + failTerminalBatch = false; + throw new Error("first terminal batch unavailable"); + } + published.push(...events.map((item) => item.eventId)); + }); + handler = new ExecutionWorkerHandler(writer); + const start = await send(1, { kind: "start", providerId: "codex", input: startInput() }); + expect(start.kind).toBe("committed"); + if (start.kind !== "committed") throw new Error("Canonical start did not commit"); + const staged = new MessageRepo(db).create(THREAD_ID, "assistant", "Answer", 2, undefined, undefined, undefined, "model", true); + const finish = { + threadId: THREAD_ID, + turnId: TURN_ID, + executionId: EXECUTION_ID, + providerId: "codex", + providerIdentities: [], + outcome: "completed" as const, + projection: { message: staged, narrative: toolNarrative(staged.id, 100) }, + }; + failTerminalBatch = true; + await expect(send(2, { kind: "finalize", outcome: "completed", input: finish })) + .rejects.toThrow("first terminal batch unavailable"); + expect(db.prepare("SELECT terminal_outcome FROM canonical_agent_ingest_checkpoints WHERE execution_id = ?").get(EXECUTION_ID)) + .toEqual({ terminal_outcome: null }); + expect(db.prepare("SELECT kind FROM canonical_writer_operation_receipts WHERE execution_id = ? AND operation_id = ?") + .get(EXECUTION_ID, "lease-1:2")).toEqual({ kind: "semantic:finish-pending" }); + const priorChunks = db.prepare("SELECT COUNT(*) AS count FROM canonical_writer_operation_receipts WHERE execution_id = ? AND kind = 'semantic-publication'") + .get(EXECUTION_ID) as { count: number }; + expect(priorChunks.count).toBeGreaterThan(0); + expect(await writer.transact(operation(2, { + kind: "finish", + outcome: "completed", + input: { ...finish, error: "different input" }, + }))).toEqual({ kind: "conflict", operationId: "lease-1:2" }); + + db.close(true); + db = openDatabase({ dbPath: path }); + published = []; + writer = new CanonicalExecutionSemanticWriter(db, (events) => { + published.push(...events.map((item) => item.eventId)); + }); + expect(await writer.transact(operation(2, { kind: "finish", outcome: "completed", input: finish }))) + .toMatchObject({ kind: "committed", operationId: "lease-1:2" }); + const terminalEventCount = db.prepare("SELECT COUNT(*) AS count FROM canonical_agent_events WHERE execution_id = ? AND accepted_sequence > ?") + .get(EXECUTION_ID, start.durableRevision) as { count: number }; + expect(published).toContain(`${EXECUTION_ID}:turn.completed`); + expect(published).toHaveLength(terminalEventCount.count); + }); + it("rolls back the canonical start and user message when the begin receipt fails", async () => { db.run("CREATE TRIGGER fail_semantic_begin_receipt BEFORE INSERT ON canonical_writer_operation_receipts WHEN NEW.kind = 'semantic:begin' BEGIN SELECT RAISE(ABORT, 'begin receipt unavailable'); END"); await expect(send(1, { kind: "start", providerId: "codex", input: startInput() })) @@ -237,23 +374,7 @@ describe("CanonicalExecutionSemanticWriter through ExecutionWorkerHandler", () = handler = new ExecutionWorkerHandler(writer); expect((await send(1, { kind: "start", providerId: "codex", input: startInput() })).kind).toBe("committed"); const staged = new MessageRepo(db).create(THREAD_ID, "assistant", "Answer", 2, undefined, undefined, undefined, "model", true); - const narrative = Array.from({ length: 100 }, (_, index) => ({ - kind: "toolCall" as const, - sequence: 2, - sortOrder: index, - record: { - id: `tool-${index}`, - message_id: staged.id, - parent_tool_call_id: null, - tool_name: "Read", - input_summary: `file-${index}`, - output_summary: "ok", - status: "completed" as const, - started_at: NOW, - completed_at: NOW, - sort_order: index, - }, - })); + const narrative = toolNarrative(staged.id, 100); finishing = true; expect((await send(2, { kind: "finalize", @@ -271,6 +392,15 @@ describe("CanonicalExecutionSemanticWriter through ExecutionWorkerHandler", () = expect(await secondStart).toMatchObject({ kind: "committed", operationId: "lease-2:1" }); expect(published).toContain(`${EXECUTION_ID}:turn.completed`); expect(published.some((eventId) => eventId.startsWith(otherExecution.executionId))).toBe(true); + const terminalReceipt = db.prepare("SELECT receipt_json FROM canonical_writer_operation_receipts WHERE execution_id = ? AND operation_id = ?") + .get(EXECUTION_ID, "lease-1:2") as { receipt_json: string }; + expect(terminalReceipt.receipt_json.length).toBeLessThan(256); + const chunks = db.prepare("SELECT COUNT(*) AS count FROM canonical_writer_operation_receipts WHERE execution_id = ? AND kind = 'semantic-publication'") + .get(EXECUTION_ID) as { count: number }; + expect(chunks.count).toBeGreaterThan(1); + const chunkSize = db.prepare("SELECT MAX(json_array_length(receipt_json)) AS size FROM canonical_writer_operation_receipts WHERE execution_id = ? AND kind = 'semantic-publication'") + .get(EXECUTION_ID) as { size: number }; + expect(chunkSize.size).toBeLessThanOrEqual(64); expect(db.prepare("SELECT COUNT(*) AS count FROM canonical_writer_operation_receipts WHERE kind = 'semantic-head'").get()) .toEqual({ count: 2 }); }); diff --git a/apps/server/src/features/agents/canonical/canonical-agent-boundary.ts b/apps/server/src/features/agents/canonical/canonical-agent-boundary.ts index d56f774da..3e94eedfd 100644 --- a/apps/server/src/features/agents/canonical/canonical-agent-boundary.ts +++ b/apps/server/src/features/agents/canonical/canonical-agent-boundary.ts @@ -190,6 +190,13 @@ export type CanonicalAgentBatchedCommitResult = Omit void, + onBatchWrite?: (batch: CanonicalTerminalBatchWrite) => void, ): Promise { const checkpoint = this.loadCheckpoint(input.executionId); if (checkpoint?.terminalOutcome) { return this.confirmedParentTerminalBatch(checkpoint, input.threadId); } - return this.writeParentTerminalBatches(input, checkpoint, onTerminalCommit); + return this.writeParentTerminalBatches(input, checkpoint, onBatchWrite); } private confirmedParentTerminalBatch( @@ -2058,7 +2065,7 @@ export class CanonicalAgentBoundary implements ParentTurnDurability, CodexCollab private async writeParentTerminalBatches( input: CanonicalParentTurnFinishInput, checkpoint: CanonicalAgentCheckpoint | null, - onTerminalCommit?: (durableSequence: number) => void, + onBatchWrite?: (batch: CanonicalTerminalBatchWrite) => void, ): Promise { const projection = input.projectTurn(); const endedAt = new Date().toISOString(); @@ -2086,7 +2093,7 @@ export class CanonicalAgentBoundary implements ParentTurnDurability, CodexCollab terminalRevision, terminalEventId, ), - onBatchFinishing: () => this.finishParentTerminalBatch(state, input, terminalRevision, onTerminalCommit), + onBatchFinishing: () => this.finishParentTerminalBatch(state, input, terminalRevision, onBatchWrite), onBatchCommitted: () => this.publishParentTerminalBatch(state), }); return this.parentTerminalBatchResult(state, writeBatches); @@ -2256,7 +2263,7 @@ export class CanonicalAgentBoundary implements ParentTurnDurability, CodexCollab state: ParentTerminalBatchState, input: CanonicalParentTurnFinishInput, terminalRevision: number, - onTerminalCommit?: (durableSequence: number) => void, + onBatchWrite?: (batch: CanonicalTerminalBatchWrite) => void, ): void { const modelState = this.parentTerminalBatchModelState(state); if (!state.wrote) { @@ -2268,8 +2275,12 @@ export class CanonicalAgentBoundary implements ParentTurnDurability, CodexCollab if (state.terminal) { this.displayMaterializer.materializeItems(this.parentAssistantItemIds(input.turnId)); this.retireParentNarrativeRecovery(input.turnId); - onTerminalCommit?.(state.acceptedSequence); } + onBatchWrite?.({ + durableSequence: state.acceptedSequence, + publishedSequences: state.pendingPublication.map((event) => event.acceptedSequence), + terminal: state.terminal, + }); state.latest = this.committedParentTerminalBatchResult(state, input.threadId, terminalRevision); } diff --git a/apps/server/src/features/agents/canonical/canonical-execution-semantic-writer.ts b/apps/server/src/features/agents/canonical/canonical-execution-semantic-writer.ts index 678ddd0d3..a8604d0cb 100644 --- a/apps/server/src/features/agents/canonical/canonical-execution-semantic-writer.ts +++ b/apps/server/src/features/agents/canonical/canonical-execution-semantic-writer.ts @@ -1,5 +1,6 @@ import type { Database } from "bun:sqlite"; import * as NodeCrypto from "node:crypto"; +import { CanonicalAgentEventEnvelopeSchema } from "@mcode/contracts"; import { z } from "zod"; import type { ExecutionIdentity, ExecutionLease } from "../execution/execution-mailbox-protocol.js"; @@ -13,6 +14,11 @@ import { CanonicalParentTurnWrite } from "./canonical-parent-turn-write.js"; const HEAD_ID = "semantic:head"; const HEAD_KIND = "semantic-head"; +const PUBLICATION_KIND = "semantic-publication"; +const PENDING_FINISH_KIND = "semantic:finish-pending"; +const PUBLICATION_CHUNK_SIZE = 64; +const storedPublicationSequencesSchema = z.array(z.number().int().positive().max(Number.MAX_SAFE_INTEGER)) + .min(1).max(PUBLICATION_CHUNK_SIZE); const storedHeadSchema = z.object({ execution: z.object({ threadId: z.string(), turnId: z.string(), executionId: z.string() }), providerId: z.string(), @@ -30,12 +36,15 @@ const storedReceiptSchema = z.object({ kind: z.literal("committed"), operationId: z.string(), durableRevision: z.number().int(), + publicationVersion: z.literal(1), }); const storedOperationSchema = z.object({ kind: z.string(), input_hash: z.string(), receipt_json: z.string(), }); +const storedOperationListSchema = z.array(storedOperationSchema); +const storedEnvelopeRowSchema = z.object({ envelope_json: z.string() }); interface SemanticHead { readonly execution: ExecutionIdentity; @@ -54,7 +63,10 @@ class SemanticConflict extends Error {} export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter { private readonly turns: CanonicalParentTurnWrite; private readonly findOperation: ReturnType; + private readonly listPublicationChunks: ReturnType; + private readonly findPublishedEvent: ReturnType; private readonly insertOperation: ReturnType; + private readonly commitPendingFinish: ReturnType; private readonly updateHead: ReturnType; private bufferedPublication: Parameters[0][] | null = null; @@ -64,7 +76,10 @@ export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter else this.publish(events); }); this.findOperation = db.prepare("SELECT kind, input_hash, receipt_json FROM canonical_writer_operation_receipts WHERE execution_id = ? AND operation_id = ?"); + this.listPublicationChunks = db.prepare("SELECT kind, input_hash, receipt_json FROM canonical_writer_operation_receipts WHERE execution_id = ? AND operation_id >= ? AND operation_id < ? ORDER BY operation_id"); + this.findPublishedEvent = db.prepare("SELECT envelope_json FROM canonical_agent_events WHERE execution_id = ? AND accepted_sequence = ?"); this.insertOperation = db.prepare("INSERT INTO canonical_writer_operation_receipts (execution_id, operation_id, kind, input_hash, receipt_json) VALUES (?, ?, ?, ?, ?)"); + this.commitPendingFinish = db.prepare("UPDATE canonical_writer_operation_receipts SET kind = ?, receipt_json = ? WHERE execution_id = ? AND operation_id = ? AND kind = ? AND input_hash = ?"); this.updateHead = db.prepare("UPDATE canonical_writer_operation_receipts SET receipt_json = ? WHERE execution_id = ? AND operation_id = ? AND kind = ?"); } @@ -72,8 +87,8 @@ export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter async transact(operation: ExecutionSemanticOperation): Promise { if (!validOperation(operation)) return conflict(operation); const hash = fingerprint(operation); - const replay = this.loadOperation(operation.execution.executionId, operation.operationId); - if (replay) return this.replay(operation, hash, replay); + const existing = this.existingReceipt(operation, hash); + if (existing) return existing; try { if (operation.mutation.kind === "begin") return this.begin(operation, hash); if (operation.mutation.kind === "append-events") return this.append(operation, hash); @@ -85,6 +100,13 @@ export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter } } + private existingReceipt(operation: ExecutionSemanticOperation, hash: string): ExecutionWriteReceipt | null { + const row = this.loadOperation(operation.execution.executionId, operation.operationId); + if (!row) return null; + if (row.kind !== PENDING_FINISH_KIND) return this.replay(operation, hash, row); + return operation.mutation.kind === "finish" && row.input_hash === hash ? null : conflict(operation); + } + private begin(operation: ExecutionSemanticOperation, hash: string): ExecutionWriteReceipt { const mutation = operation.mutation; if (mutation.kind !== "begin" || operation.ordinal !== 1 || this.loadHead(operation.execution.executionId)) { @@ -108,6 +130,7 @@ export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter terminal: false, }; this.insertOperation.run(operation.execution.executionId, HEAD_ID, HEAD_KIND, fingerprint(head.lease), JSON.stringify(head)); + this.storePublicationChunks(operation.execution.executionId, hash, result.events.map((event) => event.acceptedSequence)); this.storeReceipt(operation, hash, receipt); return receipt; })()); @@ -129,6 +152,7 @@ export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter if (result.outcome !== "committed") throw new SemanticConflict(); const receipt = committed(operation, result.durableThrough); this.storeHead({ ...head, ordinal: operation.ordinal, durableRevision: receipt.durableRevision }); + this.storePublicationChunks(operation.execution.executionId, hash, result.events.map((event) => event.acceptedSequence)); this.storeReceipt(operation, hash, receipt); return receipt; })()); @@ -139,16 +163,21 @@ export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter if (mutation.kind !== "finish" || !validFinish(operation, mutation)) return conflict(operation); const head = this.loadHead(operation.execution.executionId); if (!head || !nextHead(head, operation) || head.providerId !== mutation.input.providerId) return conflict(operation); - const result = await this.turns.finish(mutation.input, (durableSequence) => { - const current = this.requireNextHead(operation); - if (current.providerId !== mutation.input.providerId) throw new SemanticConflict(); - const receipt = committed(operation, durableSequence); - this.storeHead({ ...current, ordinal: operation.ordinal, durableRevision: receipt.durableRevision, terminal: true }); - this.storeReceipt(operation, hash, receipt); + this.reserveFinish(operation, hash); + this.publishStoredEvents(operation.execution.executionId, hash); + const result = await this.turns.finish(mutation.input, (batch) => { + this.storePublicationChunks(operation.execution.executionId, hash, batch.publishedSequences); + if (batch.terminal) { + const current = this.requireNextHead(operation); + if (current.providerId !== mutation.input.providerId) throw new SemanticConflict(); + const receipt = committed(operation, batch.durableSequence); + this.storeHead({ ...current, ordinal: operation.ordinal, durableRevision: receipt.durableRevision, terminal: true }); + this.storeReceipt(operation, hash, receipt); + } }); if (result.outcome !== "committed") return conflict(operation); const stored = this.loadOperation(operation.execution.executionId, operation.operationId); - return stored ? this.replay(operation, hash, stored) : conflict(operation); + return stored ? this.readReceipt(operation, hash, stored) : conflict(operation); } private requireNextHead(operation: ExecutionSemanticOperation): SemanticHead { @@ -172,21 +201,79 @@ export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter this.updateHead.run(JSON.stringify(head), head.execution.executionId, HEAD_ID, HEAD_KIND); } + private reserveFinish(operation: ExecutionSemanticOperation, hash: string): void { + const existing = this.loadOperation(operation.execution.executionId, operation.operationId); + if (existing) { + if (existing.kind !== PENDING_FINISH_KIND || existing.input_hash !== hash) throw new SemanticConflict(); + return; + } + this.insertOperation.run(operation.execution.executionId, operation.operationId, PENDING_FINISH_KIND, hash, "{}"); + } + + private storePublicationChunks(executionId: string, hash: string, sequences: readonly number[]): void { + for (let offset = 0; offset < sequences.length; offset += PUBLICATION_CHUNK_SIZE) { + const chunk = sequences.slice(offset, offset + PUBLICATION_CHUNK_SIZE); + const first = chunk[0]; + if (first === undefined) continue; + const id = publicationChunkId(hash, first); + const json = JSON.stringify(chunk); + const existing = this.loadOperation(executionId, id); + if (existing) { + if (existing.kind !== PUBLICATION_KIND || existing.input_hash !== hash || existing.receipt_json !== json) { + throw new SemanticConflict(); + } + continue; + } + this.insertOperation.run(executionId, id, PUBLICATION_KIND, hash, json); + } + } + private storeReceipt(operation: ExecutionSemanticOperation, hash: string, receipt: ExecutionWriteReceipt): void { + const receiptJson = JSON.stringify({ ...receipt, publicationVersion: 1 }); + if (operation.mutation.kind === "finish") { + const updated = this.commitPendingFinish.run( + "semantic:finish", receiptJson, operation.execution.executionId, operation.operationId, PENDING_FINISH_KIND, hash, + ); + if (updated.changes !== 1) throw new SemanticConflict(); + return; + } this.insertOperation.run( operation.execution.executionId, operation.operationId, `semantic:${operation.mutation.kind}`, hash, - JSON.stringify(receipt), + receiptJson, ); } private replay(operation: ExecutionSemanticOperation, hash: string, stored: StoredOperation): ExecutionWriteReceipt { + const receipt = this.readReceipt(operation, hash, stored); + if (receipt.kind === "committed") this.publishStoredEvents(operation.execution.executionId, hash); + return receipt; + } + + private readReceipt(operation: ExecutionSemanticOperation, hash: string, stored: StoredOperation): ExecutionWriteReceipt { if (stored.kind !== `semantic:${operation.mutation.kind}` || stored.input_hash !== hash) return conflict(operation); const receipt = storedReceiptSchema.parse(JSON.parse(stored.receipt_json)); return receipt.operationId === operation.operationId && Number.isSafeInteger(receipt.durableRevision) - ? receipt : conflict(operation); + ? committed(operation, receipt.durableRevision) : conflict(operation); + } + + private publishStoredEvents(executionId: string, hash: string): void { + const prefix = publicationChunkPrefix(hash); + const chunks = storedOperationListSchema.parse(this.listPublicationChunks.all(executionId, prefix, `${prefix}~`)); + for (const chunk of chunks) { + if (chunk.kind !== PUBLICATION_KIND || chunk.input_hash !== hash) throw new Error("Semantic publication chunk mismatch"); + const sequences = storedPublicationSequencesSchema.parse(JSON.parse(chunk.receipt_json)); + const events = sequences.map((sequence) => this.loadPublishedEvent(executionId, sequence)); + this.publish(events); + } + } + + private loadPublishedEvent(executionId: string, sequence: number) { + const row = storedEnvelopeRowSchema.nullable().parse(this.findPublishedEvent.get(executionId, sequence)); + if (!row) throw new Error(`Semantic publication event missing at ${executionId}:${sequence}`); + return CanonicalAgentEventEnvelopeSchema.parse(JSON.parse(row.envelope_json)); } // Only the synchronous begin and append transactions use this buffer. Finish publishes after each committed batch. @@ -257,6 +344,14 @@ function fingerprint(value: unknown): string { return NodeCrypto.createHash("sha256").update(JSON.stringify(sortJson(value))).digest("hex"); } +function publicationChunkPrefix(hash: string): string { + return `semantic:publication:${hash}:`; +} + +function publicationChunkId(hash: string, firstSequence: number): string { + return `${publicationChunkPrefix(hash)}${firstSequence.toString().padStart(16, "0")}`; +} + function sortJson(value: unknown): unknown { if (Array.isArray(value)) return value.map(sortJson); if (value === null || typeof value !== "object") return value; diff --git a/apps/server/src/features/agents/canonical/canonical-parent-turn-write.ts b/apps/server/src/features/agents/canonical/canonical-parent-turn-write.ts index 3c932bebb..66ff50d5e 100644 --- a/apps/server/src/features/agents/canonical/canonical-parent-turn-write.ts +++ b/apps/server/src/features/agents/canonical/canonical-parent-turn-write.ts @@ -16,6 +16,7 @@ import { type CanonicalAgentCommitResult, type CanonicalAgentEventDraft, type CanonicalAgentEventPublisher, + type CanonicalTerminalBatchWrite, } from "./canonical-agent-boundary.js"; type CreateMessageArgument = Parameters; @@ -96,7 +97,7 @@ export class CanonicalParentTurnWrite { /** Confirm the terminal checkpoint and publish the staged assistant in one transaction. */ finish( input: DataOnlyParentTurnFinishInput, - onTerminalCommit?: (durableSequence: number) => void, + onBatchWrite?: (batch: CanonicalTerminalBatchWrite) => void, ): Promise { this.assertStagedAssistant(input); const projection: ParentTurnProjection = { @@ -118,7 +119,7 @@ export class CanonicalParentTurnWrite { this.messages.setAssistantOutcome(projection.message.id, input.outcome, input.executionId); this.messages.publishAssistant(projection.message.id); }, - }, onTerminalCommit); + }, onBatchWrite); } private projectUserMessage(input: DataOnlyParentTurnStartInput) { From c109826f377892312604868629280b9390de145b Mon Sep 17 00:00:00 2001 From: "(CJ) Chukwudi Nwobodo" <142016413+chuks-qua@users.noreply.github.com> Date: Thu, 24 Sep 2026 20:35:38 +0100 Subject: [PATCH 039/136] feat(server): route semantic execution writes through SQLite worker --- .../canonical-execution-writer-port.test.ts | 74 +++++++++++++++++++ .../canonical-agent-writer-client.ts | 15 ++++ .../canonical-agent-writer-protocol.ts | 9 +++ .../canonical-agent-writer.worker.ts | 34 +++++++++ .../canonical-execution-writer-port.ts | 22 ++++++ 5 files changed, 154 insertions(+) create mode 100644 apps/server/src/features/agents/canonical/__tests__/canonical-execution-writer-port.test.ts create mode 100644 apps/server/src/features/agents/canonical/canonical-execution-writer-port.ts diff --git a/apps/server/src/features/agents/canonical/__tests__/canonical-execution-writer-port.test.ts b/apps/server/src/features/agents/canonical/__tests__/canonical-execution-writer-port.test.ts new file mode 100644 index 000000000..ed5255317 --- /dev/null +++ b/apps/server/src/features/agents/canonical/__tests__/canonical-execution-writer-port.test.ts @@ -0,0 +1,74 @@ +import "reflect-metadata"; +import * as NodeFS from "node:fs"; +import * as NodeOS from "node:os"; +import * as NodePath from "node:path"; +import type { Database } from "bun:sqlite"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; + +import { openDatabase } from "../../../../runtime/persistence/sqlite/database.js"; +import type { ExecutionSemanticOperation } from "../../execution/execution-worker-handler.js"; +import { CanonicalAgentWriterClient } from "../canonical-agent-writer-client.js"; +import { CanonicalExecutionWriterPort } from "../canonical-execution-writer-port.js"; + +const NOW = "2026-09-24T10:00:00.000Z"; +const THREAD_ID = "semantic-worker-thread"; +const TURN_ID = "semantic-worker-turn"; +const EXECUTION_ID = "00000000-0000-4000-8000-000000000177"; +const execution = { threadId: THREAD_ID, turnId: TURN_ID, executionId: EXECUTION_ID } as const; +const lease = { ownerEpoch: 1, workerIndex: 0, workerGeneration: 1, leaseId: "semantic-worker-lease" } as const; + +function beginOperation(): ExecutionSemanticOperation { + return { + operationId: `${lease.leaseId}:1`, execution, lease, ordinal: 1, + mutation: { + kind: "begin", providerId: "codex", + input: { + thread: { id: THREAD_ID, workspaceId: "semantic-worker-workspace", providerId: "codex", createdAt: NOW }, + turnId: TURN_ID, executionId: EXECUTION_ID, permissionMode: "supervised", providerIdentities: [], + userMessage: { kind: "create", messageId: "semantic-worker-user", content: "Question", sequence: 1 }, + }, + }, + }; +} + +describe("execution semantic writer transport", () => { + let directory: string; + let db: Database; + let writer: CanonicalAgentWriterClient | undefined; + + beforeEach(() => { + directory = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "mcode-execution-writer-")); + db = openDatabase({ dbPath: NodePath.join(directory, "app.sqlite") }); + db.prepare("INSERT INTO workspaces (id, name, path, created_at, updated_at) VALUES (?, ?, ?, ?, ?)") + .run("semantic-worker-workspace", "Workspace", directory, NOW, NOW); + db.prepare("INSERT INTO threads (id, workspace_id, title, branch, provider, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?)") + .run(THREAD_ID, "semantic-worker-workspace", "Thread", "main", "codex", NOW, NOW); + }); + + afterEach(async () => { + await writer?.close(); + db.close(true); + NodeFS.rmSync(directory, { recursive: true, force: true }); + }); + + it("commits and replays a semantic start through the dedicated SQLite worker", async () => { + writer = new CanonicalAgentWriterClient(NodePath.join(directory, "app.sqlite")); + const published: string[] = []; + const port = new CanonicalExecutionWriterPort(writer, (events) => { + published.push(...events.map((event) => event.eventId)); + }); + const operation = beginOperation(); + + const first = await port.transact(operation); + expect(first).toMatchObject({ kind: "committed", operationId: operation.operationId }); + expect(db.prepare("SELECT content FROM messages WHERE id = ?").get("semantic-worker-user")) + .toEqual({ content: "Question" }); + expect(published.length).toBeGreaterThan(0); + + published.length = 0; + expect(await port.transact(operation)).toEqual(first); + expect(published.length).toBeGreaterThan(0); + expect(db.prepare("SELECT COUNT(*) AS count FROM messages WHERE id = ?").get("semantic-worker-user")) + .toEqual({ count: 1 }); + }); +}); diff --git a/apps/server/src/features/agents/canonical/canonical-agent-writer-client.ts b/apps/server/src/features/agents/canonical/canonical-agent-writer-client.ts index 1c512c211..b1ac13d87 100644 --- a/apps/server/src/features/agents/canonical/canonical-agent-writer-client.ts +++ b/apps/server/src/features/agents/canonical/canonical-agent-writer-client.ts @@ -5,10 +5,12 @@ import type { CanonicalParentNarrativeRecoveryReceipt, CanonicalProviderWriteInput, CanonicalProviderWriteReceipt, + CanonicalSemanticWriteResult, CanonicalWriterRequest, CanonicalWriterResponse, } from "./canonical-agent-writer-protocol.js"; import type { ParentNarrativeRecoveryCommitInput } from "./canonical-agent-boundary.js"; +import type { ExecutionSemanticOperation } from "../execution/execution-worker-handler.js"; const MAX_PENDING_WRITES = 64; const MAX_WORKER_ATTEMPTS = 3; @@ -66,6 +68,19 @@ export class CanonicalAgentWriterClient { return response.receipt; } + /** Commits one execution semantic operation on the writer worker and returns envelopes for publication. */ + async transactSemantic(operation: ExecutionSemanticOperation): Promise { + if (!operation.operationId || !operation.execution.executionId) { + throw new Error("Semantic writer operation and execution IDs are required"); + } + const response = await this.sendWithRetry({ + kind: "semantic-transact", requestId: NodeCrypto.randomUUID(), + operationId: operation.operationId, executionId: operation.execution.executionId, operation, + }); + if (response.kind !== "semantic-transacted") throw new Error("Canonical writer returned an unexpected response"); + return response.result; + } + /** Resolves after recovery writes finish; a lost reply replays the durable receipt. */ async recordParentNarrativeRecovery( operationId: string, diff --git a/apps/server/src/features/agents/canonical/canonical-agent-writer-protocol.ts b/apps/server/src/features/agents/canonical/canonical-agent-writer-protocol.ts index ab7a53c3a..a53986271 100644 --- a/apps/server/src/features/agents/canonical/canonical-agent-writer-protocol.ts +++ b/apps/server/src/features/agents/canonical/canonical-agent-writer-protocol.ts @@ -1,4 +1,5 @@ import type { CanonicalAgentEventEnvelope } from "@mcode/contracts"; +import type { ExecutionSemanticOperation, ExecutionWriteReceipt } from "../execution/execution-worker-handler.js"; import type { CanonicalAgentCommitInput, CanonicalAgentCommitResult, @@ -33,6 +34,12 @@ export interface CanonicalParentNarrativeClassificationReceipt { reset: true; } +/** A semantic transaction and the committed envelopes awaiting publication. */ +export interface CanonicalSemanticWriteResult { + receipt: ExecutionWriteReceipt; + events: readonly CanonicalAgentEventEnvelope[]; +} + interface Correlation { requestId: string; operationId: string; @@ -42,6 +49,7 @@ interface Correlation { export type CanonicalWriterRequest = | (Correlation & { kind: "open"; dbPath: string }) | (Correlation & { kind: "commit"; input: CanonicalProviderWriteInput }) + | (Correlation & { kind: "semantic-transact"; operation: ExecutionSemanticOperation }) | (Correlation & { kind: "record-parent-narrative-recovery"; input: ParentNarrativeRecoveryCommitInput }) | (Correlation & { kind: "classify-parent-narrative-recovery"; input: ParentNarrativeRecoveryCommitInput }) | (Correlation & { kind: "ack-operation" }) @@ -50,6 +58,7 @@ export type CanonicalWriterRequest = export type CanonicalWriterResponse = | (Correlation & { kind: "opened" }) | (Correlation & { kind: "committed"; receipt: CanonicalProviderWriteReceipt }) + | (Correlation & { kind: "semantic-transacted"; result: CanonicalSemanticWriteResult }) | (Correlation & { kind: "parent-narrative-recovery-recorded"; receipt: CanonicalParentNarrativeRecoveryReceipt }) | (Correlation & { kind: "parent-narrative-recovery-classified"; receipt: CanonicalParentNarrativeClassificationReceipt }) | (Correlation & { kind: "operation-acknowledged" }) diff --git a/apps/server/src/features/agents/canonical/canonical-agent-writer.worker.ts b/apps/server/src/features/agents/canonical/canonical-agent-writer.worker.ts index 31abbed39..55c0f3f9f 100644 --- a/apps/server/src/features/agents/canonical/canonical-agent-writer.worker.ts +++ b/apps/server/src/features/agents/canonical/canonical-agent-writer.worker.ts @@ -2,8 +2,10 @@ import "reflect-metadata"; import { Database } from "bun:sqlite"; import * as NodeFS from "node:fs"; import * as NodePath from "node:path"; +import type { CanonicalAgentEventEnvelope } from "@mcode/contracts"; import { applySQLiteConnectionPolicy } from "../../../runtime/persistence/sqlite/sqlite-connection-policy.js"; import { CanonicalAgentBoundary } from "./canonical-agent-boundary.js"; +import { CanonicalExecutionSemanticWriter } from "./canonical-execution-semantic-writer.js"; import { ParentAssistantTextCheckpointService } from "../turns/parent-assistant-text-checkpoint-service.js"; import { CanonicalAgentWriterReceipts, CanonicalWriterOperationConflict, CanonicalWriterReceiptCapacity } from "./canonical-agent-writer-receipts.js"; import type { @@ -18,6 +20,8 @@ let db: Database | undefined; let boundary: CanonicalAgentBoundary | undefined; let assistantTextCheckpoints: ParentAssistantTextCheckpointService | undefined; let receipts: CanonicalAgentWriterReceipts | undefined; +let semanticWriter: CanonicalExecutionSemanticWriter | undefined; +let semanticPublication: CanonicalAgentEventEnvelope[] | undefined; function openDatabase(dbPath: string): void { if (boundary || !NodePath.isAbsolute(dbPath) || !NodeFS.existsSync(dbPath)) { @@ -29,11 +33,16 @@ function openDatabase(dbPath: string): void { boundary = new CanonicalAgentBoundary(connection, () => {}); assistantTextCheckpoints = new ParentAssistantTextCheckpointService(connection); receipts = new CanonicalAgentWriterReceipts(connection); + semanticWriter = new CanonicalExecutionSemanticWriter(connection, (events) => { + if (!semanticPublication) throw new Error("Semantic publication has no active request"); + semanticPublication.push(...events); + }); db = connection; } catch (error) { boundary = undefined; assistantTextCheckpoints = undefined; receipts = undefined; + semanticWriter = undefined; connection.close(true); throw error; } @@ -93,6 +102,7 @@ async function handle(request: CanonicalWriterRequest): Promise, + correlation: Pick, +): Promise { + try { + if (!semanticWriter || semanticPublication) throw new Error("Semantic writer is not ready"); + if (request.operation.operationId !== request.operationId + || request.operation.execution.executionId !== request.executionId) { + throw new Error("Semantic writer request identity mismatch"); + } + semanticPublication = []; + const receipt = await semanticWriter.transact(request.operation); + const events = semanticPublication; + return { ...correlation, kind: "semantic-transacted", result: { receipt, events } }; + } catch { + return { ...correlation, kind: "failed", reason: "write-failed" }; + } finally { + semanticPublication = undefined; + } +} + async function handleWrite( request: Extract, correlation: Pick, diff --git a/apps/server/src/features/agents/canonical/canonical-execution-writer-port.ts b/apps/server/src/features/agents/canonical/canonical-execution-writer-port.ts new file mode 100644 index 000000000..8003f92c0 --- /dev/null +++ b/apps/server/src/features/agents/canonical/canonical-execution-writer-port.ts @@ -0,0 +1,22 @@ +import type { + ExecutionSemanticOperation, + ExecutionSemanticWriter, + ExecutionWriteReceipt, +} from "../execution/execution-worker-handler.js"; +import type { CanonicalAgentEventPublisher } from "./canonical-agent-boundary.js"; +import { CanonicalAgentWriterClient } from "./canonical-agent-writer-client.js"; + +/** Bridges an execution worker to the sole SQLite writer and publishes only committed events. */ +export class CanonicalExecutionWriterPort implements ExecutionSemanticWriter { + constructor( + private readonly writer: CanonicalAgentWriterClient, + private readonly publish: CanonicalAgentEventPublisher, + ) {} + + /** Resolve after the durable writer receipt and its canonical event publication. */ + async transact(operation: ExecutionSemanticOperation): Promise { + const { receipt, events } = await this.writer.transactSemantic(operation); + if (receipt.kind === "committed" && events.length > 0) this.publish(events); + return receipt; + } +} From 0a3756a48f0b7a92d216c18b90196bf13b8015bd Mon Sep 17 00:00:00 2001 From: "(CJ) Chukwudi Nwobodo" <142016413+chuks-qua@users.noreply.github.com> Date: Thu, 24 Sep 2026 20:35:08 +0100 Subject: [PATCH 040/136] feat(server): stage terminal projection on canonical writer --- .../canonical-parent-turn-write.test.ts | 103 +++++++++ .../canonical/canonical-parent-turn-write.ts | 201 ++++++++++++++++++ .../conversation/narrative/narrative-store.ts | 7 +- .../persistence/thought-segment-repo.ts | 26 +-- .../events/persistence/hook-execution-repo.ts | 34 +-- .../persistence/tool-call-record-repo.ts | 10 +- 6 files changed, 343 insertions(+), 38 deletions(-) diff --git a/apps/server/src/features/agents/canonical/__tests__/canonical-parent-turn-write.test.ts b/apps/server/src/features/agents/canonical/__tests__/canonical-parent-turn-write.test.ts index 9fa4c22e9..55e419691 100644 --- a/apps/server/src/features/agents/canonical/__tests__/canonical-parent-turn-write.test.ts +++ b/apps/server/src/features/agents/canonical/__tests__/canonical-parent-turn-write.test.ts @@ -3,12 +3,15 @@ import * as NodeFS from "node:fs"; import * as NodeOS from "node:os"; import * as NodePath from "node:path"; import type { Database } from "bun:sqlite"; +import type { ParentNarrativeRecoveryItem } from "@mcode/contracts"; import { afterEach, beforeEach, describe, expect, it } from "vitest"; import { openDatabase } from "../../../../runtime/persistence/sqlite/database.js"; import { MessageRepo } from "../../conversation/persistence/message-repo.js"; +import { ToolCallRecordRepo } from "../../tools/persistence/tool-call-record-repo.js"; import { CanonicalParentTurnWrite, + type DataOnlyParentTerminalProjectionInput, type DataOnlyParentTurnFinishInput, type DataOnlyParentTurnStartInput, } from "../canonical-parent-turn-write.js"; @@ -48,6 +51,47 @@ function finishInput(message: ReturnType): DataOnlyParent }; } +function terminalProjectionInput(): DataOnlyParentTerminalProjectionInput { + const narrative: ParentNarrativeRecoveryItem[] = [ + { + kind: "toolCall", + record: { + id: "tool-1", message_id: "", parent_tool_call_id: null, tool_name: "Read", + display_name: null, provider_agent_key: null, subagent_identity_key: null, + subagent_provider_name: null, subagent_prompt: null, subagent_type: null, + subagent_agent_id: null, subagent_duration_ms: null, model: null, reasoning_effort: null, + input_summary: "file.txt", output_summary: "", output_total_bytes: null, + output_artifact_path: null, exit_code: null, status: "running", + started_at: NOW, completed_at: null, sort_order: 1, + }, + }, + { + kind: "narrationSegment", + record: { id: "thought-1", message_id: "", text: "Answer", started_at: NOW, ended_at: NOW, sort_order: 2 }, + }, + { + kind: "hook", + record: { + id: "hook-1", message_id: "", hook_name: "Stop", tool_name: null, phase: "stop", + payload: "{}", duration_ms: null, did_block: false, started_at: NOW, + ended_at: null, sort_order: 3, + }, + }, + ]; + return { + threadId: THREAD_ID, + executionId: EXECUTION_ID, + outcome: "completed", + endedAt: "2026-09-24T10:00:01.000Z", + assistant: { + content: "Answer", + model: "claude-sonnet-4-6", + attachments: [{ id: "attachment-1", name: "result.txt", mimeType: "text/plain", sizeBytes: 6 }], + }, + narrative, + }; +} + describe("CanonicalParentTurnWrite", () => { let directory: string; let path: string; @@ -143,4 +187,63 @@ describe("CanonicalParentTurnWrite", () => { expect((await writer.finish(finish)).outcome).toBe("committed"); expect(new MessageRepo(db).findByIdInThread(THREAD_ID, staged.id)).toMatchObject({ is_internal: false, outcome: "completed" }); }); + + it("stages cloneable terminal rows once and publishes only after finish", async () => { + writer.start(startInput()); + const input = terminalProjectionInput(); + expect(() => structuredClone(input)).not.toThrow(); + new ToolCallRecordRepo(db).create({ + toolCallId: "tool-1", messageId: "user-1", toolName: "Read", inputSummary: "earlier", + outputSummary: "", status: "running", startedAt: NOW, sortOrder: 1, + }); + + const first = writer.stageTerminalProjection(input); + expect(first.messageId).toMatch(/^[0-9a-f]{64}$/); + expect(first.toolCallCount).toBe(1); + expect(db.prepare("SELECT is_internal, outcome FROM messages WHERE id = ?").get(first.messageId!)) + .toEqual({ is_internal: 1, outcome: null }); + expect(db.prepare("SELECT message_id, status, completed_at FROM tool_call_records WHERE id = 'tool-1'").get()) + .toEqual({ message_id: first.messageId, status: "completed", completed_at: input.endedAt }); + expect(db.prepare("SELECT message_id, is_final_response FROM thought_segments WHERE id = 'thought-1'").get()) + .toEqual({ message_id: first.messageId, is_final_response: 1 }); + expect(db.prepare("SELECT message_id, ended_at FROM hook_executions WHERE id = 'hook-1'").get()) + .toEqual({ message_id: first.messageId, ended_at: input.endedAt }); + expect(published.flat()).not.toContain(`${EXECUTION_ID}:turn.completed`); + + db.close(true); + db = openDatabase({ dbPath: path }); + writer = new CanonicalParentTurnWrite(db, (events) => { + published.push(events.map((event) => event.eventId)); + }); + const repeated = writer.stageTerminalProjection(input); + expect(repeated.messageId).toBe(first.messageId); + expect(repeated.toolCallCount).toBe(1); + expect(() => structuredClone(repeated.projection)).not.toThrow(); + expect(db.prepare("SELECT COUNT(*) AS count FROM tool_call_records WHERE message_id = ?").get(first.messageId!)) + .toEqual({ count: 1 }); + const finish = { ...finishInput(first.projection.message!), projection: first.projection }; + expect((await writer.finish(finish)).outcome).toBe("committed"); + expect(published.flat()).toContain(`${EXECUTION_ID}:turn.completed`); + expect(new MessageRepo(db).findByIdInThread(THREAD_ID, first.messageId!)) + .toMatchObject({ is_internal: false, outcome: "completed", attachments: input.assistant.attachments }); + expect(writer.stageTerminalProjection(input).messageId).toBe(first.messageId); + }); + + it("rolls back all staged rows when a narrative write fails", async () => { + writer.start(startInput()); + const input = terminalProjectionInput(); + db.run("CREATE TRIGGER fail_hook BEFORE INSERT ON hook_executions BEGIN SELECT RAISE(ABORT, 'hook unavailable'); END"); + expect(() => writer.stageTerminalProjection(input)).toThrow("hook unavailable"); + expect(db.prepare("SELECT COUNT(*) AS count FROM messages WHERE role = 'assistant'").get()).toEqual({ count: 0 }); + expect(db.prepare("SELECT COUNT(*) AS count FROM tool_call_records").get()).toEqual({ count: 0 }); + expect(db.prepare("SELECT COUNT(*) AS count FROM thought_segments").get()).toEqual({ count: 0 }); + expect(db.prepare("SELECT terminal_outcome FROM canonical_agent_ingest_checkpoints WHERE execution_id = ?").get(EXECUTION_ID)) + .toEqual({ terminal_outcome: null }); + expect(published.flat()).not.toContain(`${EXECUTION_ID}:turn.completed`); + + db.run("DROP TRIGGER fail_hook"); + const staged = writer.stageTerminalProjection(input); + expect(staged.messageId).not.toBeNull(); + expect(staged.toolCallCount).toBe(1); + }); }); diff --git a/apps/server/src/features/agents/canonical/canonical-parent-turn-write.ts b/apps/server/src/features/agents/canonical/canonical-parent-turn-write.ts index 66ff50d5e..f8808deae 100644 --- a/apps/server/src/features/agents/canonical/canonical-parent-turn-write.ts +++ b/apps/server/src/features/agents/canonical/canonical-parent-turn-write.ts @@ -1,8 +1,19 @@ import type { Database } from "bun:sqlite"; +import { + ParentNarrativeRecoveryItemSchema, + type ParentNarrativeRecoveryItem, + type StoredAttachment, + type TurnOutcome, +} from "@mcode/contracts"; import { z } from "zod"; import { MessageRepo } from "../conversation/persistence/message-repo.js"; +import { NarrativeStore } from "../conversation/narrative/narrative-store.js"; +import { ThoughtSegmentRepo } from "../conversation/narrative/persistence/thought-segment-repo.js"; +import { HookExecutionRepo } from "../events/persistence/hook-execution-repo.js"; import { PlanQuestionAnswersRepo } from "../planning/persistence/plan-question-answers-repo.js"; +import { ToolCallRecordRepo } from "../tools/persistence/tool-call-record-repo.js"; +import { deriveTurnAssistantMessageId } from "../turns/turn-assistant-message-id.js"; import type { ParentTurnFinishInput, ParentTurnProjection, @@ -56,17 +67,44 @@ export interface DataOnlyParentTurnFinishInput extends Omit; constructor(db: Database, publish: CanonicalAgentEventPublisher) { + this.db = db; this.canonical = new CanonicalAgentBoundary(db, publish); this.messages = new MessageRepo(db); + this.narrative = new NarrativeStore( + this.messages, + new ToolCallRecordRepo(db), + new ThoughtSegmentRepo(db), + new HookExecutionRepo(db), + db, + ); this.threads = new ThreadRepo(db); this.planAnswers = new PlanQuestionAnswersRepo(db); this.stagedAssistant = db.prepare("SELECT role, content FROM messages WHERE id = ? AND thread_id = ?"); @@ -94,6 +132,93 @@ export class CanonicalParentTurnWrite { return this.canonical.commit(input); } + /** Stage an internal assistant and its terminal narrative without publishing them. */ + stageTerminalProjection(input: DataOnlyParentTerminalProjectionInput): StagedParentTerminalProjection { + if (!input.threadId || !input.executionId || !Number.isFinite(Date.parse(input.endedAt))) { + throw new Error("Invalid parent terminal projection identity or end time"); + } + const narrative = input.narrative.map((item) => ParentNarrativeRecoveryItemSchema().parse(item)); + return this.db.transaction(() => this.stageTerminalProjectionInTransaction(input, narrative))(); + } + + private stageTerminalProjectionInTransaction( + input: DataOnlyParentTerminalProjectionInput, + snapshot: readonly ParentNarrativeRecoveryItem[], + ): StagedParentTerminalProjection { + const turn = this.terminalProjectionTurn(input); + if (turn.terminal) return this.persistedTerminalProjection(input, turn.turnId); + if (!hasTerminalSubstance(input, snapshot)) { + return { projection: { message: null, narrative: [] }, messageId: null, toolCallCount: 0 }; + } + const message = this.stageAssistant(input); + const settled = settleTerminalNarrative(snapshot, message.id, input.outcome, input.endedAt, input.assistant.content); + this.narrative.persistRecoveredNarrative(message.id, settled, true); + return this.projectionForMessage(message); + } + + private terminalProjectionTurn(input: DataOnlyParentTerminalProjectionInput): { turnId: string; terminal: boolean } { + const turn = this.canonical.loadTurnByExecution(input.executionId); + const checkpoint = this.canonical.loadCheckpoint(input.executionId); + if (!turn || turn.threadId !== input.threadId || checkpoint?.turnId !== turn.id) { + throw new Error(`Canonical parent execution not found: ${input.executionId}`); + } + if (checkpoint.terminalOutcome && checkpoint.terminalOutcome !== input.outcome) { + throw new Error(`Canonical parent execution has a different terminal outcome: ${input.executionId}`); + } + return { turnId: turn.id, terminal: checkpoint.terminalOutcome !== null }; + } + + private persistedTerminalProjection(input: DataOnlyParentTerminalProjectionInput, turnId: string): StagedParentTerminalProjection { + const persisted = this.canonical.loadTerminalProjection(turnId); + if (!persisted.message) { + if (input.assistant.content.trim() || input.assistant.attachments.length > 0 || input.narrative.length > 0) { + throw new Error(`Canonical terminal projection is empty: ${input.executionId}`); + } + return { projection: { message: null, narrative: [] }, messageId: null, toolCallCount: 0 }; + } + this.assertAssistantMatches(persisted.message, input); + return this.projectionForMessage(persisted.message); + } + + private stageAssistant(input: DataOnlyParentTerminalProjectionInput) { + const id = deriveTurnAssistantMessageId(input.threadId, `execution:${input.executionId}`); + const existing = this.messages.findByIdInThread(input.threadId, id); + if (existing) { + this.assertAssistantMatches(existing, input); + return existing; + } + const sequence = this.messages.getLatestSequenceIncludingInternal(input.threadId) + 1; + return this.messages.createAssistantIdempotent({ + id, + threadId: input.threadId, + content: input.assistant.content, + sequence, + model: input.assistant.model, + attachments: [...input.assistant.attachments], + isInternal: true, + }); + } + + private assertAssistantMatches( + message: NonNullable>, + input: DataOnlyParentTerminalProjectionInput, + ): void { + if (message.role !== "assistant" || message.content !== input.assistant.content + || message.model !== input.assistant.model + || JSON.stringify(message.attachments ?? []) !== JSON.stringify(input.assistant.attachments)) { + throw new Error(`Staged assistant projection conflicts with terminal input: ${input.executionId}`); + } + } + + private projectionForMessage(message: NonNullable>): StagedParentTerminalProjection { + const narrative = this.narrative.loadForMessages([message]); + return { + projection: { message, narrative }, + messageId: message.id, + toolCallCount: narrative.filter((entry) => entry.kind === "toolCall").length, + }; + } + /** Confirm the terminal checkpoint and publish the staged assistant in one transaction. */ finish( input: DataOnlyParentTurnFinishInput, @@ -158,3 +283,79 @@ export class CanonicalParentTurnWrite { } } } + +function settleTerminalNarrative( + items: readonly ParentNarrativeRecoveryItem[], + messageId: string, + outcome: TurnOutcome, + endedAt: string, + assistantContent: string, +): ParentNarrativeRecoveryItem[] { + const finalText = assistantContent.trim(); + const lastThoughtOrder = Math.max(...items.filter((item) => item.kind === "narrationSegment") + .map((item) => item.record.sort_order)); + return items.map((item) => { + if (item.kind === "toolCall") return settleToolCall(item, messageId, outcome, endedAt); + if (item.kind === "hook") return settleHook(item, messageId, endedAt); + return settleThought(item, messageId, finalText, lastThoughtOrder); + }); +} + +function hasTerminalSubstance( + input: DataOnlyParentTerminalProjectionInput, + snapshot: readonly ParentNarrativeRecoveryItem[], +): boolean { + return Boolean(input.assistant.content.trim()) || input.assistant.attachments.length > 0 || snapshot.length > 0; +} + +function settleToolCall( + item: Extract, + messageId: string, + outcome: TurnOutcome, + endedAt: string, +): ParentNarrativeRecoveryItem { + const status = item.record.status === "running" ? terminalToolStatus(outcome) : item.record.status; + return { kind: "toolCall", record: { + ...item.record, + message_id: messageId, + status, + completed_at: item.record.status === "running" ? endedAt : item.record.completed_at, + } }; +} + +function settleHook( + item: Extract, + messageId: string, + endedAt: string, +): ParentNarrativeRecoveryItem { + const durationMs = item.record.ended_at + ? item.record.duration_ms + : Math.max(0, Date.parse(endedAt) - Date.parse(item.record.started_at)); + return { kind: "hook", record: { + ...item.record, + message_id: messageId, + duration_ms: durationMs, + ended_at: item.record.ended_at ?? endedAt, + } }; +} + +function settleThought( + item: Extract, + messageId: string, + finalText: string, + lastThoughtOrder: number, +): ParentNarrativeRecoveryItem { + const thoughtText = item.record.text.trim(); + const isFinalResponse = finalText.length > 0 && thoughtText.length > 0 + && (thoughtText === finalText || (item.record.sort_order === lastThoughtOrder && finalText.endsWith(thoughtText))); + return { kind: "narrationSegment", record: { + ...item.record, + message_id: messageId, + is_final_response: isFinalResponse ? 1 : item.record.is_final_response, + } }; +} + +function terminalToolStatus(outcome: TurnOutcome): "completed" | "failed" | "cancelled" { + if (outcome === "errored" || outcome === "interrupted") return "failed"; + return outcome === "cancelled" ? "cancelled" : "completed"; +} diff --git a/apps/server/src/features/agents/conversation/narrative/narrative-store.ts b/apps/server/src/features/agents/conversation/narrative/narrative-store.ts index d950e467c..99a1f1831 100644 --- a/apps/server/src/features/agents/conversation/narrative/narrative-store.ts +++ b/apps/server/src/features/agents/conversation/narrative/narrative-store.ts @@ -1318,6 +1318,7 @@ export class NarrativeStore { persistRecoveredNarrative( messageId: string, items: readonly ParentNarrativeRecoveryItem[], + replaceExisting = false, ): void { const tools: CreateToolCallRecordInput[] = []; const thoughts: CreateThoughtSegmentInput[] = []; @@ -1327,9 +1328,9 @@ export class NarrativeStore { if (item.kind === "narrationSegment") thoughts.push(this.recoveredThought(messageId, item)); if (item.kind === "hook") hooks.push(this.recoveredHook(messageId, item)); } - if (tools.length > 0) this.toolCallRecordRepo.bulkCreate(tools); - if (thoughts.length > 0) this.thoughtSegmentRepo.bulkCreate(thoughts); - if (hooks.length > 0) this.hookExecutionRepo.bulkCreate(hooks); + if (tools.length > 0) this.toolCallRecordRepo.bulkCreate(tools, replaceExisting); + if (thoughts.length > 0) this.thoughtSegmentRepo.bulkCreate(thoughts, replaceExisting); + if (hooks.length > 0) this.hookExecutionRepo.bulkCreate(hooks, replaceExisting); } private assertRecoveryItemFitsWriteBatch(byteLength: number, threadId: string): void { diff --git a/apps/server/src/features/agents/conversation/narrative/persistence/thought-segment-repo.ts b/apps/server/src/features/agents/conversation/narrative/persistence/thought-segment-repo.ts index 040fbc195..2b95ee014 100644 --- a/apps/server/src/features/agents/conversation/narrative/persistence/thought-segment-repo.ts +++ b/apps/server/src/features/agents/conversation/narrative/persistence/thought-segment-repo.ts @@ -83,22 +83,22 @@ export class ThoughtSegmentRepo { } /** Insert multiple thought segment records in a single transaction. */ - bulkCreate(inputs: CreateThoughtSegmentInput[]): void { + bulkCreate(inputs: CreateThoughtSegmentInput[], replaceExisting = false): void { if (inputs.length === 0) return; this.orm.transaction((tx) => { for (const item of inputs) { - tx.insert(thoughtSegments) - .values({ - id: item.id ?? NodeCrypto.randomUUID(), - messageId: item.messageId, - text: item.text, - startedAt: item.startedAt, - endedAt: item.endedAt, - sortOrder: item.sortOrder, - isFinalResponse: item.isFinalResponse ?? 0, - }) - .onConflictDoNothing() - .run(); + const { id, ...rest } = { + id: item.id ?? NodeCrypto.randomUUID(), + messageId: item.messageId, + text: item.text, + startedAt: item.startedAt, + endedAt: item.endedAt, + sortOrder: item.sortOrder, + isFinalResponse: item.isFinalResponse ?? 0, + }; + const write = tx.insert(thoughtSegments).values({ id, ...rest }); + if (replaceExisting) write.onConflictDoUpdate({ target: thoughtSegments.id, set: rest }).run(); + else write.onConflictDoNothing().run(); } }); } diff --git a/apps/server/src/features/agents/events/persistence/hook-execution-repo.ts b/apps/server/src/features/agents/events/persistence/hook-execution-repo.ts index 1f10e39f7..05a90cadf 100644 --- a/apps/server/src/features/agents/events/persistence/hook-execution-repo.ts +++ b/apps/server/src/features/agents/events/persistence/hook-execution-repo.ts @@ -97,26 +97,26 @@ export class HookExecutionRepo { } /** Insert multiple hook execution records in a single transaction. */ - bulkCreate(inputs: CreateHookExecutionInput[]): void { + bulkCreate(inputs: CreateHookExecutionInput[], replaceExisting = false): void { if (inputs.length === 0) return; this.orm.transaction((tx) => { for (const item of inputs) { - tx.insert(hookExecutions) - .values({ - id: item.id ?? NodeCrypto.randomUUID(), - messageId: item.messageId, - hookName: item.hookName, - toolName: item.toolName, - phase: item.phase, - payload: item.payload, - durationMs: item.durationMs, - didBlock: item.didBlock ? 1 : 0, - startedAt: item.startedAt, - endedAt: item.endedAt, - sortOrder: item.sortOrder, - }) - .onConflictDoNothing() - .run(); + const { id, ...rest } = { + id: item.id ?? NodeCrypto.randomUUID(), + messageId: item.messageId, + hookName: item.hookName, + toolName: item.toolName, + phase: item.phase, + payload: item.payload, + durationMs: item.durationMs, + didBlock: item.didBlock ? 1 : 0, + startedAt: item.startedAt, + endedAt: item.endedAt, + sortOrder: item.sortOrder, + }; + const write = tx.insert(hookExecutions).values({ id, ...rest }); + if (replaceExisting) write.onConflictDoUpdate({ target: hookExecutions.id, set: rest }).run(); + else write.onConflictDoNothing().run(); } }); } diff --git a/apps/server/src/features/agents/tools/persistence/tool-call-record-repo.ts b/apps/server/src/features/agents/tools/persistence/tool-call-record-repo.ts index b326c6c37..e40cc97ee 100644 --- a/apps/server/src/features/agents/tools/persistence/tool-call-record-repo.ts +++ b/apps/server/src/features/agents/tools/persistence/tool-call-record-repo.ts @@ -155,15 +155,15 @@ export class ToolCallRecordRepo { } /** Create multiple tool call records in a single transaction. */ - bulkCreate(inputs: CreateToolCallRecordInput[]): void { + bulkCreate(inputs: CreateToolCallRecordInput[], replaceExisting = false): void { this.orm.transaction((tx) => { const now = new Date().toISOString(); for (const item of inputs) { const insert = prepareToolCallRecordInsert(item, now); - tx.insert(toolCallRecords) - .values(this.insertValues(item, insert)) - .onConflictDoNothing() - .run(); + const { id, ...rest } = this.insertValues(item, insert); + const write = tx.insert(toolCallRecords).values({ id, ...rest }); + if (replaceExisting) write.onConflictDoUpdate({ target: toolCallRecords.id, set: rest }).run(); + else write.onConflictDoNothing().run(); } }); } From 407b7b62636b56923305808c83bdb2624d7a9b99 Mon Sep 17 00:00:00 2001 From: "(CJ) Chukwudi Nwobodo" <142016413+chuks-qua@users.noreply.github.com> Date: Thu, 24 Sep 2026 20:39:11 +0100 Subject: [PATCH 041/136] test(server): prove semantic writer failure and lost reply --- .../canonical-execution-writer-port.test.ts | 53 +++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/apps/server/src/features/agents/canonical/__tests__/canonical-execution-writer-port.test.ts b/apps/server/src/features/agents/canonical/__tests__/canonical-execution-writer-port.test.ts index ed5255317..ed571d72f 100644 --- a/apps/server/src/features/agents/canonical/__tests__/canonical-execution-writer-port.test.ts +++ b/apps/server/src/features/agents/canonical/__tests__/canonical-execution-writer-port.test.ts @@ -71,4 +71,57 @@ describe("execution semantic writer transport", () => { expect(db.prepare("SELECT COUNT(*) AS count FROM messages WHERE id = ?").get("semantic-worker-user")) .toEqual({ count: 1 }); }); + + it("does not acknowledge a failed database write and accepts a clean retry", async () => { + writer = new CanonicalAgentWriterClient(NodePath.join(directory, "app.sqlite")); + const port = new CanonicalExecutionWriterPort(writer, () => {}); + db.run(`CREATE TRIGGER reject_semantic_start BEFORE INSERT ON canonical_agent_events + BEGIN SELECT RAISE(FAIL, 'injected write failure'); END`); + + await expect(port.transact(beginOperation())).rejects.toThrow("Canonical writer write-failed"); + expect(db.prepare("SELECT COUNT(*) AS count FROM messages WHERE id = ?").get("semantic-worker-user")) + .toEqual({ count: 0 }); + expect(db.prepare("SELECT COUNT(*) AS count FROM canonical_writer_operation_receipts WHERE execution_id = ?") + .get(EXECUTION_ID)).toEqual({ count: 0 }); + + db.run("DROP TRIGGER reject_semantic_start"); + expect(await port.transact(beginOperation())).toMatchObject({ kind: "committed" }); + }); + + it("replays a durable semantic receipt after the writer loses its reply", async () => { + let dropReply = true; + const createWorker = (): Worker => { + const worker = new Worker(new URL("../canonical-agent-writer.worker.ts", import.meta.url), { type: "module" }); + return new Proxy(worker, { + set(target, property, value) { + if (property !== "onmessage" || typeof value !== "function") return Reflect.set(target, property, value); + target.onmessage = (message) => { + if (dropReply && message.data.kind === "semantic-transacted") { + dropReply = false; + target.terminate(); + return; + } + value(message); + }; + return true; + }, + get(target, property) { + const value: unknown = Reflect.get(target, property, target); + return typeof value === "function" ? value.bind(target) : value; + }, + }); + }; + writer = new CanonicalAgentWriterClient(NodePath.join(directory, "app.sqlite"), createWorker); + const published: string[] = []; + const port = new CanonicalExecutionWriterPort(writer, (events) => { + published.push(...events.map((event) => event.eventId)); + }); + + expect(await port.transact(beginOperation())).toMatchObject({ kind: "committed" }); + expect(dropReply).toBe(false); + expect(published.length).toBeGreaterThan(0); + expect(new Set(published).size).toBe(published.length); + expect(db.prepare("SELECT COUNT(*) AS count FROM messages WHERE id = ?").get("semantic-worker-user")) + .toEqual({ count: 1 }); + }); }); From 4493628b5a1d66c74228010e9a6fe68ffbf45144 Mon Sep 17 00:00:00 2001 From: "(CJ) Chukwudi Nwobodo" <142016413+chuks-qua@users.noreply.github.com> Date: Thu, 24 Sep 2026 20:44:26 +0100 Subject: [PATCH 042/136] feat(server): stage terminal projection under execution lease --- .../canonical-execution-writer-port.test.ts | 64 +++++++++++++++++++ .../canonical-execution-semantic-writer.ts | 21 ++++++ .../canonical/canonical-parent-turn-write.ts | 37 +++++++---- .../conversation/persistence/message-repo.ts | 8 +++ .../execution/execution-worker-handler.ts | 20 ++++-- 5 files changed, 133 insertions(+), 17 deletions(-) diff --git a/apps/server/src/features/agents/canonical/__tests__/canonical-execution-writer-port.test.ts b/apps/server/src/features/agents/canonical/__tests__/canonical-execution-writer-port.test.ts index ed571d72f..0de7e7d11 100644 --- a/apps/server/src/features/agents/canonical/__tests__/canonical-execution-writer-port.test.ts +++ b/apps/server/src/features/agents/canonical/__tests__/canonical-execution-writer-port.test.ts @@ -6,6 +6,8 @@ import type { Database } from "bun:sqlite"; import { afterEach, beforeEach, describe, expect, it } from "vitest"; import { openDatabase } from "../../../../runtime/persistence/sqlite/database.js"; +import { MessageRepo } from "../../conversation/persistence/message-repo.js"; +import { deriveTurnAssistantMessageId } from "../../turns/turn-assistant-message-id.js"; import type { ExecutionSemanticOperation } from "../../execution/execution-worker-handler.js"; import { CanonicalAgentWriterClient } from "../canonical-agent-writer-client.js"; import { CanonicalExecutionWriterPort } from "../canonical-execution-writer-port.js"; @@ -124,4 +126,66 @@ describe("execution semantic writer transport", () => { expect(db.prepare("SELECT COUNT(*) AS count FROM messages WHERE id = ?").get("semantic-worker-user")) .toEqual({ count: 1 }); }); + + it("stages and finalizes assistant rows without a main-connection projection write", async () => { + writer = new CanonicalAgentWriterClient(NodePath.join(directory, "app.sqlite")); + const published: string[] = []; + const port = new CanonicalExecutionWriterPort(writer, (events) => { + published.push(...events.map((event) => event.eventId)); + }); + const op = (ordinal: number, mutation: ExecutionSemanticOperation["mutation"]): ExecutionSemanticOperation => ({ + operationId: `${lease.leaseId}:${ordinal}`, execution, lease, ordinal, mutation, + }); + expect(await port.transact(beginOperation())).toMatchObject({ kind: "committed" }); + const terminalInput = { + threadId: THREAD_ID, executionId: EXECUTION_ID, outcome: "completed" as const, endedAt: NOW, + assistant: { content: "Answer", model: "fixture", attachments: [] }, narrative: [], + }; + const staged = op(2, { kind: "stage-terminal", input: terminalInput }); + expect(await port.transact(staged)).toMatchObject({ kind: "committed" }); + expect(db.prepare("SELECT content, is_internal FROM messages WHERE role = 'assistant'").get()) + .toEqual({ content: "Answer", is_internal: 1 }); + const stagedId = deriveTurnAssistantMessageId(THREAD_ID, `execution:${EXECUTION_ID}`); + expect(new MessageRepo(db).findByIdInThreadIncludingInternal(THREAD_ID, stagedId)) + .toMatchObject({ content: "Answer", is_internal: true }); + expect(published).not.toContain(`${EXECUTION_ID}:turn.completed`); + + const finish = op(3, { kind: "finish", outcome: "completed", input: { + threadId: THREAD_ID, turnId: TURN_ID, executionId: EXECUTION_ID, providerId: "codex", + providerIdentities: [], outcome: "completed", projection: { kind: "writer-staged" }, + } }); + expect(await port.transact(finish)).toMatchObject({ kind: "committed" }); + expect(db.prepare("SELECT content, is_internal, outcome FROM messages WHERE role = 'assistant'").get()) + .toEqual({ content: "Answer", is_internal: 0, outcome: "completed" }); + expect(published).toContain(`${EXECUTION_ID}:turn.completed`); + expect(await port.transact(staged)).toMatchObject({ kind: "committed" }); + expect(await port.transact(finish)).toMatchObject({ kind: "committed" }); + expect(db.prepare("SELECT COUNT(*) AS count FROM messages WHERE role = 'assistant'").get()) + .toEqual({ count: 1 }); + }); + + it("rolls back a failed semantic terminal stage and retries at the same ordinal", async () => { + writer = new CanonicalAgentWriterClient(NodePath.join(directory, "app.sqlite")); + const port = new CanonicalExecutionWriterPort(writer, () => {}); + expect(await port.transact(beginOperation())).toMatchObject({ kind: "committed" }); + const stage: ExecutionSemanticOperation = { + operationId: `${lease.leaseId}:2`, execution, lease, ordinal: 2, + mutation: { kind: "stage-terminal", input: { + threadId: THREAD_ID, executionId: EXECUTION_ID, outcome: "completed", endedAt: NOW, + assistant: { content: "Answer", model: null, attachments: [] }, narrative: [], + } }, + }; + db.run(`CREATE TRIGGER reject_terminal_stage BEFORE INSERT ON messages + WHEN NEW.role = 'assistant' BEGIN SELECT RAISE(FAIL, 'injected terminal failure'); END`); + await expect(port.transact(stage)).rejects.toThrow("Canonical writer write-failed"); + expect(db.prepare("SELECT COUNT(*) AS count FROM messages WHERE role = 'assistant'").get()) + .toEqual({ count: 0 }); + expect(db.prepare("SELECT COUNT(*) AS count FROM canonical_writer_operation_receipts WHERE operation_id = ?") + .get(stage.operationId)).toEqual({ count: 0 }); + + db.run("DROP TRIGGER reject_terminal_stage"); + expect(await port.transact(stage)).toMatchObject({ kind: "committed", operationId: stage.operationId }); + expect(db.prepare("SELECT COUNT(*) AS count FROM messages WHERE role = 'assistant' AND is_internal = 1").get()) + .toEqual({ count: 1 }); + }); }); diff --git a/apps/server/src/features/agents/canonical/canonical-execution-semantic-writer.ts b/apps/server/src/features/agents/canonical/canonical-execution-semantic-writer.ts index a8604d0cb..df935bd47 100644 --- a/apps/server/src/features/agents/canonical/canonical-execution-semantic-writer.ts +++ b/apps/server/src/features/agents/canonical/canonical-execution-semantic-writer.ts @@ -92,6 +92,7 @@ export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter try { if (operation.mutation.kind === "begin") return this.begin(operation, hash); if (operation.mutation.kind === "append-events") return this.append(operation, hash); + if (operation.mutation.kind === "stage-terminal") return this.stageTerminal(operation, hash); if (operation.mutation.kind === "finish") return await this.finish(operation, hash); return conflict(operation); } catch (error) { @@ -163,6 +164,7 @@ export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter if (mutation.kind !== "finish" || !validFinish(operation, mutation)) return conflict(operation); const head = this.loadHead(operation.execution.executionId); if (!head || !nextHead(head, operation) || head.providerId !== mutation.input.providerId) return conflict(operation); + if ("kind" in mutation.input.projection && !this.hasStagedTerminalPredecessor(operation)) return conflict(operation); this.reserveFinish(operation, hash); this.publishStoredEvents(operation.execution.executionId, hash); const result = await this.turns.finish(mutation.input, (batch) => { @@ -180,6 +182,25 @@ export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter return stored ? this.readReceipt(operation, hash, stored) : conflict(operation); } + private stageTerminal(operation: ExecutionSemanticOperation, hash: string): ExecutionWriteReceipt { + const mutation = operation.mutation; + if (mutation.kind !== "stage-terminal" || mutation.input.threadId !== operation.execution.threadId + || mutation.input.executionId !== operation.execution.executionId) return conflict(operation); + return this.db.transaction(() => { + const head = this.requireNextHead(operation); + this.turns.stageTerminalProjection(mutation.input); + const receipt = committed(operation, head.durableRevision); + this.storeHead({ ...head, ordinal: operation.ordinal }); + this.storeReceipt(operation, hash, receipt); + return receipt; + })(); + } + + private hasStagedTerminalPredecessor(operation: ExecutionSemanticOperation): boolean { + const previousId = `${operation.lease.leaseId}:${operation.ordinal - 1}`; + return this.loadOperation(operation.execution.executionId, previousId)?.kind === "semantic:stage-terminal"; + } + private requireNextHead(operation: ExecutionSemanticOperation): SemanticHead { const head = this.loadHead(operation.execution.executionId); if (!head || !nextHead(head, operation)) throw new SemanticConflict(); diff --git a/apps/server/src/features/agents/canonical/canonical-parent-turn-write.ts b/apps/server/src/features/agents/canonical/canonical-parent-turn-write.ts index f8808deae..504761c57 100644 --- a/apps/server/src/features/agents/canonical/canonical-parent-turn-write.ts +++ b/apps/server/src/features/agents/canonical/canonical-parent-turn-write.ts @@ -64,7 +64,7 @@ export interface DataOnlyParentEventInput extends Omit { - projection: ParentTurnProjection; + projection: ParentTurnProjection | { readonly kind: "writer-staged" }; } /** Cloneable terminal data whose compatibility rows are staged on the writer connection. */ @@ -182,7 +182,7 @@ export class CanonicalParentTurnWrite { private stageAssistant(input: DataOnlyParentTerminalProjectionInput) { const id = deriveTurnAssistantMessageId(input.threadId, `execution:${input.executionId}`); - const existing = this.messages.findByIdInThread(input.threadId, id); + const existing = this.messages.findByIdInThreadIncludingInternal(input.threadId, id); if (existing) { this.assertAssistantMatches(existing, input); return existing; @@ -224,17 +224,20 @@ export class CanonicalParentTurnWrite { input: DataOnlyParentTurnFinishInput, onBatchWrite?: (batch: CanonicalTerminalBatchWrite) => void, ): Promise { - this.assertStagedAssistant(input); + const staged = "kind" in input.projection + ? this.loadStagedTerminalProjection(input.threadId, input.executionId) + : input.projection; + this.assertStagedAssistant(input.threadId, staged); const projection: ParentTurnProjection = { - message: input.projection.message + message: staged.message ? { - ...input.projection.message, + ...staged.message, is_internal: false, outcome: input.outcome, outcomeExecutionId: input.executionId, } : null, - narrative: input.projection.narrative, + narrative: staged.narrative, }; return this.canonical.finishParentTurnBatched({ ...input, @@ -274,10 +277,20 @@ export class CanonicalParentTurnWrite { ); } - private assertStagedAssistant(input: DataOnlyParentTurnFinishInput): void { - const projected = input.projection.message; + private loadStagedTerminalProjection(threadId: string, executionId: string): ParentTurnProjection { + const id = deriveTurnAssistantMessageId(threadId, `execution:${executionId}`); + const message = this.messages.findByIdInThreadIncludingInternal(threadId, id); + if (!message) return { message: null, narrative: [] }; + if (message.role !== "assistant" || !message.is_internal) { + throw new Error(`Staged assistant projection is not private: ${id}`); + } + return { message, narrative: this.narrative.loadForMessages([message]) }; + } + + private assertStagedAssistant(threadId: string, projection: ParentTurnProjection): void { + const projected = projection.message; if (!projected) return; - const staged = stagedAssistantSchema.nullable().parse(this.stagedAssistant.get(projected.id, input.threadId)); + const staged = stagedAssistantSchema.nullable().parse(this.stagedAssistant.get(projected.id, threadId)); if (!staged || staged.role !== "assistant" || staged.content !== projected.content) { throw new Error(`Staged assistant projection not found: ${projected.id}`); } @@ -292,8 +305,10 @@ function settleTerminalNarrative( assistantContent: string, ): ParentNarrativeRecoveryItem[] { const finalText = assistantContent.trim(); - const lastThoughtOrder = Math.max(...items.filter((item) => item.kind === "narrationSegment") - .map((item) => item.record.sort_order)); + let lastThoughtOrder = -Infinity; + for (const item of items) { + if (item.kind === "narrationSegment") lastThoughtOrder = Math.max(lastThoughtOrder, item.record.sort_order); + } return items.map((item) => { if (item.kind === "toolCall") return settleToolCall(item, messageId, outcome, endedAt); if (item.kind === "hook") return settleHook(item, messageId, endedAt); diff --git a/apps/server/src/features/agents/conversation/persistence/message-repo.ts b/apps/server/src/features/agents/conversation/persistence/message-repo.ts index e779949f8..abe415d40 100644 --- a/apps/server/src/features/agents/conversation/persistence/message-repo.ts +++ b/apps/server/src/features/agents/conversation/persistence/message-repo.ts @@ -1080,6 +1080,14 @@ export class MessageRepo { return row ? rowToMessage(row) : null; } + /** Find one private or published message for writer-owned turn materialization. */ + findByIdInThreadIncludingInternal(threadId: string, messageId: string): Message | null { + const row = this.orm.select().from(messages) + .where(and(eq(messages.id, messageId), eq(messages.threadId, threadId))) + .get(); + return row ? rowToMessage(row) : null; + } + /** Look up a single message by its primary key. */ findById(id: string): Message | undefined { const row = this.orm diff --git a/apps/server/src/features/agents/execution/execution-worker-handler.ts b/apps/server/src/features/agents/execution/execution-worker-handler.ts index 891b88751..c7ed3d965 100644 --- a/apps/server/src/features/agents/execution/execution-worker-handler.ts +++ b/apps/server/src/features/agents/execution/execution-worker-handler.ts @@ -2,6 +2,7 @@ import type { TurnOutcome } from "@mcode/contracts"; import type { ProviderEventDraft } from "@mcode/providers"; import type { + DataOnlyParentTerminalProjectionInput, DataOnlyParentTurnFinishInput, DataOnlyParentTurnStartInput, } from "../canonical/canonical-parent-turn-write.js"; @@ -21,6 +22,7 @@ export type ExecutionWorkCommand = | { readonly kind: "checkpoint"; readonly phase: string; readonly nativeCursor: unknown | null } | { readonly kind: "effect-result"; readonly effectId: string; readonly settled: boolean } | { readonly kind: "provider-outcome"; readonly outcome: TurnOutcome } + | { readonly kind: "stage-terminal"; readonly input: DataOnlyParentTerminalProjectionInput } | { readonly kind: "finalize"; readonly outcome: TurnOutcome; readonly input: DataOnlyParentTurnFinishInput } | { readonly kind: "release" }; @@ -38,6 +40,7 @@ export interface ExecutionSemanticOperation { | { readonly kind: "stop-requested"; readonly requestId: string; readonly lastAdmittedOrdinal: number } | { readonly kind: "effect-result"; readonly effectId: string; readonly settled: boolean } | { readonly kind: "provider-outcome"; readonly outcome: TurnOutcome } + | { readonly kind: "stage-terminal"; readonly input: DataOnlyParentTerminalProjectionInput } | { readonly kind: "finish"; readonly outcome: TurnOutcome; readonly input: DataOnlyParentTurnFinishInput }; } @@ -62,6 +65,9 @@ export type ExecutionWorkerResult = | { readonly kind: "rejected"; readonly reason: "no-execution" | "stale-execution" | "out-of-order" | "invalid-transition" | "invalid-event-routing" | "invalid-stop-watermark" | "writer-conflict" }; type WorkerCommand = ExecutionMailboxCommand; +const METADATA_MUTATIONS: ReadonlySet = new Set([ + "checkpoint", "effect-result", "provider-outcome", "stage-terminal", +]); interface ExecutionState { readonly execution: ExecutionIdentity; @@ -165,17 +171,12 @@ function mutationFor( state: ExecutionState, ): ExecutionSemanticOperation["mutation"] | undefined { const command = request.command; + if (isMetadataMutation(command)) return command; switch (command.kind) { case "event": return eventMutation(command, request.execution, state); case "stop": return stopMutation(command, request, state); - case "checkpoint": - return { kind: "checkpoint", phase: command.phase, nativeCursor: command.nativeCursor }; - case "effect-result": - return { kind: "effect-result", effectId: command.effectId, settled: command.settled }; - case "provider-outcome": - return { kind: "provider-outcome", outcome: command.outcome }; case "finalize": return finishMutation(command, request.execution, state.providerId); case "start": @@ -189,6 +190,13 @@ function mutationFor( } } +function isMetadataMutation(command: WorkerCommand): command is Extract< + WorkerCommand, + { readonly kind: "checkpoint" | "effect-result" | "provider-outcome" | "stage-terminal" } +> { + return METADATA_MUTATIONS.has(command.kind); +} + function eventMutation( command: Extract, execution: ExecutionIdentity, From 888f53ed8798c87fe3ed1fc73449c7d824998300 Mon Sep 17 00:00:00 2001 From: "(CJ) Chukwudi Nwobodo" <142016413+chuks-qua@users.noreply.github.com> Date: Thu, 24 Sep 2026 20:45:23 +0100 Subject: [PATCH 043/136] test(server): run execution and writer workers together --- .../canonical-execution-writer-port.test.ts | 65 ++++++++++++++++++- 1 file changed, 64 insertions(+), 1 deletion(-) diff --git a/apps/server/src/features/agents/canonical/__tests__/canonical-execution-writer-port.test.ts b/apps/server/src/features/agents/canonical/__tests__/canonical-execution-writer-port.test.ts index 0de7e7d11..6d33d3197 100644 --- a/apps/server/src/features/agents/canonical/__tests__/canonical-execution-writer-port.test.ts +++ b/apps/server/src/features/agents/canonical/__tests__/canonical-execution-writer-port.test.ts @@ -8,7 +8,10 @@ import { afterEach, beforeEach, describe, expect, it } from "vitest"; import { openDatabase } from "../../../../runtime/persistence/sqlite/database.js"; import { MessageRepo } from "../../conversation/persistence/message-repo.js"; import { deriveTurnAssistantMessageId } from "../../turns/turn-assistant-message-id.js"; -import type { ExecutionSemanticOperation } from "../../execution/execution-worker-handler.js"; +import { ExecutionMailboxScheduler } from "../../execution/execution-mailbox-scheduler.js"; +import type { ExecutionLease } from "../../execution/execution-mailbox-protocol.js"; +import { ExecutionThreadWorkerPort } from "../../execution/execution-worker-port.js"; +import type { ExecutionSemanticOperation, ExecutionWorkCommand, ExecutionWorkerResult } from "../../execution/execution-worker-handler.js"; import { CanonicalAgentWriterClient } from "../canonical-agent-writer-client.js"; import { CanonicalExecutionWriterPort } from "../canonical-execution-writer-port.js"; @@ -188,4 +191,64 @@ describe("execution semantic writer transport", () => { expect(db.prepare("SELECT COUNT(*) AS count FROM messages WHERE role = 'assistant' AND is_internal = 1").get()) .toEqual({ count: 1 }); }); + + it("runs a complete assistant turn through an execution worker and the sole writer worker", async () => { + writer = new CanonicalAgentWriterClient(NodePath.join(directory, "app.sqlite")); + const published: string[] = []; + const port = new CanonicalExecutionWriterPort(writer, (events) => { + published.push(...events.map((event) => event.eventId)); + }); + const scheduler = new ExecutionMailboxScheduler({ + workerCount: 1, + limits: { + maxPending: 16, maxPendingBytes: 64_000, reservedControl: 4, reservedControlBytes: 16_000, + maxPerExecutionPending: 12, maxPerExecutionBytes: 48_000, + reservedPerExecutionControl: 3, reservedPerExecutionControlBytes: 12_000, + }, + createWorker: () => new ExecutionThreadWorkerPort(port), + onWorkerLost: () => { throw new Error("Execution worker was lost"); }, + }); + try { + const claim = scheduler.claim(execution, 1); + if (claim.kind !== "claimed") throw new Error(`Execution claim failed: ${claim.kind}`); + const send = (lease: ExecutionLease, command: Parameters[0]["command"]) => { + const admission = scheduler.submit({ execution, lease, command, byteLength: 1_000 }); + if (admission.kind !== "admitted") throw new Error(`Execution admission failed: ${admission.kind}`); + return admission.completion; + }; + const start = beginOperation().mutation; + if (start.kind !== "begin") throw new Error("Unexpected begin operation"); + await expect(send(claim.lease, { kind: "start", providerId: "codex", input: start.input })) + .resolves.toMatchObject({ kind: "reply", result: { kind: "committed" } }); + await expect(send(claim.lease, { kind: "event", events: [{ + eventId: `${EXECUTION_ID}:worker-item`, + routing: { ...execution, itemId: "worker-item" }, + sourceProviderId: "codex", sourceIdentities: [], sourceSequence: 1, + payload: { type: "item.recorded", item: { + id: "worker-item", threadId: THREAD_ID, turnId: TURN_ID, kind: "message", + providerIdentities: [], payload: { projection: "message", content: "Worker event" }, + createdAt: NOW, updatedAt: NOW, + } }, + }] })).resolves.toMatchObject({ kind: "reply", result: { kind: "committed" } }); + await expect(send(claim.lease, { kind: "stage-terminal", input: { + threadId: THREAD_ID, executionId: EXECUTION_ID, outcome: "completed", endedAt: NOW, + assistant: { content: "Worker answer", model: null, attachments: [] }, narrative: [], + } })).resolves.toMatchObject({ kind: "reply", result: { kind: "committed" } }); + await expect(send(claim.lease, { kind: "finalize", outcome: "completed", input: { + threadId: THREAD_ID, turnId: TURN_ID, executionId: EXECUTION_ID, providerId: "codex", + providerIdentities: [], outcome: "completed", projection: { kind: "writer-staged" }, + } })).resolves.toMatchObject({ kind: "reply", result: { kind: "committed" } }); + await expect(send(claim.lease, { kind: "release" })) + .resolves.toEqual({ kind: "reply", result: { kind: "released" } }); + expect(scheduler.release(execution, claim.lease)).toBe(true); + expect(db.prepare("SELECT content, outcome FROM messages WHERE role = 'assistant' AND is_internal = 0").get()) + .toEqual({ content: "Worker answer", outcome: "completed" }); + expect(published).toContain(`${EXECUTION_ID}:worker-item`); + expect(published.indexOf(`${EXECUTION_ID}:worker-item`)) + .toBeLessThan(published.indexOf(`${EXECUTION_ID}:turn.completed`)); + expect(published).toContain(`${EXECUTION_ID}:turn.completed`); + } finally { + scheduler.shutdown(); + } + }); }); From 9d25e26c21d233f4c1ee3d78667a8606a5949eb3 Mon Sep 17 00:00:00 2001 From: "(CJ) Chukwudi Nwobodo" <142016413+chuks-qua@users.noreply.github.com> Date: Thu, 24 Sep 2026 20:47:01 +0100 Subject: [PATCH 044/136] feat(server): fence Stop and terminal outcome in writer --- .../canonical-execution-writer-port.test.ts | 40 ++++++++++++- .../canonical-execution-semantic-writer.ts | 58 ++++++++++++++++--- 2 files changed, 88 insertions(+), 10 deletions(-) diff --git a/apps/server/src/features/agents/canonical/__tests__/canonical-execution-writer-port.test.ts b/apps/server/src/features/agents/canonical/__tests__/canonical-execution-writer-port.test.ts index 6d33d3197..86e93a4d7 100644 --- a/apps/server/src/features/agents/canonical/__tests__/canonical-execution-writer-port.test.ts +++ b/apps/server/src/features/agents/canonical/__tests__/canonical-execution-writer-port.test.ts @@ -144,7 +144,9 @@ describe("execution semantic writer transport", () => { threadId: THREAD_ID, executionId: EXECUTION_ID, outcome: "completed" as const, endedAt: NOW, assistant: { content: "Answer", model: "fixture", attachments: [] }, narrative: [], }; - const staged = op(2, { kind: "stage-terminal", input: terminalInput }); + expect(await port.transact(op(2, { kind: "provider-outcome", outcome: "completed" }))) + .toMatchObject({ kind: "committed" }); + const staged = op(3, { kind: "stage-terminal", input: terminalInput }); expect(await port.transact(staged)).toMatchObject({ kind: "committed" }); expect(db.prepare("SELECT content, is_internal FROM messages WHERE role = 'assistant'").get()) .toEqual({ content: "Answer", is_internal: 1 }); @@ -153,7 +155,7 @@ describe("execution semantic writer transport", () => { .toMatchObject({ content: "Answer", is_internal: true }); expect(published).not.toContain(`${EXECUTION_ID}:turn.completed`); - const finish = op(3, { kind: "finish", outcome: "completed", input: { + const finish = op(4, { kind: "finish", outcome: "completed", input: { threadId: THREAD_ID, turnId: TURN_ID, executionId: EXECUTION_ID, providerId: "codex", providerIdentities: [], outcome: "completed", projection: { kind: "writer-staged" }, } }); @@ -171,8 +173,12 @@ describe("execution semantic writer transport", () => { writer = new CanonicalAgentWriterClient(NodePath.join(directory, "app.sqlite")); const port = new CanonicalExecutionWriterPort(writer, () => {}); expect(await port.transact(beginOperation())).toMatchObject({ kind: "committed" }); - const stage: ExecutionSemanticOperation = { + expect(await port.transact({ operationId: `${lease.leaseId}:2`, execution, lease, ordinal: 2, + mutation: { kind: "provider-outcome", outcome: "completed" }, + })).toMatchObject({ kind: "committed" }); + const stage: ExecutionSemanticOperation = { + operationId: `${lease.leaseId}:3`, execution, lease, ordinal: 3, mutation: { kind: "stage-terminal", input: { threadId: THREAD_ID, executionId: EXECUTION_ID, outcome: "completed", endedAt: NOW, assistant: { content: "Answer", model: null, attachments: [] }, narrative: [], @@ -192,6 +198,32 @@ describe("execution semantic writer transport", () => { .toEqual({ count: 1 }); }); + it("fences a stopped execution to a cancelled durable outcome", async () => { + writer = new CanonicalAgentWriterClient(NodePath.join(directory, "app.sqlite")); + const port = new CanonicalExecutionWriterPort(writer, () => {}); + const op = (ordinal: number, mutation: ExecutionSemanticOperation["mutation"]): ExecutionSemanticOperation => ({ + operationId: `${lease.leaseId}:${ordinal}`, execution, lease, ordinal, mutation, + }); + expect(await port.transact(beginOperation())).toMatchObject({ kind: "committed" }); + expect(await port.transact(op(2, { + kind: "stop-requested", requestId: "stop-1", lastAdmittedOrdinal: 1, + }))).toMatchObject({ kind: "committed" }); + expect(await port.transact(op(3, { kind: "provider-outcome", outcome: "completed" }))) + .toEqual({ kind: "conflict", operationId: `${lease.leaseId}:3` }); + expect(await port.transact(op(3, { kind: "provider-outcome", outcome: "cancelled" }))) + .toMatchObject({ kind: "committed" }); + expect(await port.transact(op(4, { kind: "stage-terminal", input: { + threadId: THREAD_ID, executionId: EXECUTION_ID, outcome: "cancelled", endedAt: NOW, + assistant: { content: "Partial answer", model: null, attachments: [] }, narrative: [], + } }))).toMatchObject({ kind: "committed" }); + expect(await port.transact(op(5, { kind: "finish", outcome: "cancelled", input: { + threadId: THREAD_ID, turnId: TURN_ID, executionId: EXECUTION_ID, providerId: "codex", + providerIdentities: [], outcome: "cancelled", projection: { kind: "writer-staged" }, + } }))).toMatchObject({ kind: "committed" }); + expect(db.prepare("SELECT outcome FROM messages WHERE role = 'assistant'").get()) + .toEqual({ outcome: "cancelled" }); + }); + it("runs a complete assistant turn through an execution worker and the sole writer worker", async () => { writer = new CanonicalAgentWriterClient(NodePath.join(directory, "app.sqlite")); const published: string[] = []; @@ -230,6 +262,8 @@ describe("execution semantic writer transport", () => { createdAt: NOW, updatedAt: NOW, } }, }] })).resolves.toMatchObject({ kind: "reply", result: { kind: "committed" } }); + await expect(send(claim.lease, { kind: "provider-outcome", outcome: "completed" })) + .resolves.toMatchObject({ kind: "reply", result: { kind: "committed" } }); await expect(send(claim.lease, { kind: "stage-terminal", input: { threadId: THREAD_ID, executionId: EXECUTION_ID, outcome: "completed", endedAt: NOW, assistant: { content: "Worker answer", model: null, attachments: [] }, narrative: [], diff --git a/apps/server/src/features/agents/canonical/canonical-execution-semantic-writer.ts b/apps/server/src/features/agents/canonical/canonical-execution-semantic-writer.ts index df935bd47..16cab6604 100644 --- a/apps/server/src/features/agents/canonical/canonical-execution-semantic-writer.ts +++ b/apps/server/src/features/agents/canonical/canonical-execution-semantic-writer.ts @@ -1,6 +1,6 @@ import type { Database } from "bun:sqlite"; import * as NodeCrypto from "node:crypto"; -import { CanonicalAgentEventEnvelopeSchema } from "@mcode/contracts"; +import { CanonicalAgentEventEnvelopeSchema, TurnOutcomeSchema, type TurnOutcome } from "@mcode/contracts"; import { z } from "zod"; import type { ExecutionIdentity, ExecutionLease } from "../execution/execution-mailbox-protocol.js"; @@ -31,6 +31,9 @@ const storedHeadSchema = z.object({ ordinal: z.number().int(), durableRevision: z.number().int(), terminal: z.boolean(), + stopRequestId: z.string().nullable(), + stopWatermark: z.number().int().nullable(), + providerOutcome: TurnOutcomeSchema.nullable(), }); const storedReceiptSchema = z.object({ kind: z.literal("committed"), @@ -53,6 +56,9 @@ interface SemanticHead { readonly ordinal: number; readonly durableRevision: number; readonly terminal: boolean; + readonly stopRequestId: string | null; + readonly stopWatermark: number | null; + readonly providerOutcome: TurnOutcome | null; } type StoredOperation = z.infer; @@ -90,17 +96,25 @@ export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter const existing = this.existingReceipt(operation, hash); if (existing) return existing; try { - if (operation.mutation.kind === "begin") return this.begin(operation, hash); - if (operation.mutation.kind === "append-events") return this.append(operation, hash); - if (operation.mutation.kind === "stage-terminal") return this.stageTerminal(operation, hash); - if (operation.mutation.kind === "finish") return await this.finish(operation, hash); - return conflict(operation); + return await this.applySupported(operation, hash); } catch (error) { if (error instanceof SemanticConflict) return conflict(operation); throw error; } } + private async applySupported(operation: ExecutionSemanticOperation, hash: string): Promise { + switch (operation.mutation.kind) { + case "begin": return this.begin(operation, hash); + case "append-events": return this.append(operation, hash); + case "stop-requested": + case "provider-outcome": return this.control(operation, hash); + case "stage-terminal": return this.stageTerminal(operation, hash); + case "finish": return await this.finish(operation, hash); + default: return conflict(operation); + } + } + private existingReceipt(operation: ExecutionSemanticOperation, hash: string): ExecutionWriteReceipt | null { const row = this.loadOperation(operation.execution.executionId, operation.operationId); if (!row) return null; @@ -129,6 +143,9 @@ export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter ordinal: operation.ordinal, durableRevision: receipt.durableRevision, terminal: false, + stopRequestId: null, + stopWatermark: null, + providerOutcome: null, }; this.insertOperation.run(operation.execution.executionId, HEAD_ID, HEAD_KIND, fingerprint(head.lease), JSON.stringify(head)); this.storePublicationChunks(operation.execution.executionId, hash, result.events.map((event) => event.acceptedSequence)); @@ -164,7 +181,7 @@ export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter if (mutation.kind !== "finish" || !validFinish(operation, mutation)) return conflict(operation); const head = this.loadHead(operation.execution.executionId); if (!head || !nextHead(head, operation) || head.providerId !== mutation.input.providerId) return conflict(operation); - if ("kind" in mutation.input.projection && !this.hasStagedTerminalPredecessor(operation)) return conflict(operation); + if (!this.validStagedFinish(operation, head)) return conflict(operation); this.reserveFinish(operation, hash); this.publishStoredEvents(operation.execution.executionId, hash); const result = await this.turns.finish(mutation.input, (batch) => { @@ -188,6 +205,7 @@ export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter || mutation.input.executionId !== operation.execution.executionId) return conflict(operation); return this.db.transaction(() => { const head = this.requireNextHead(operation); + if (head.providerOutcome !== mutation.input.outcome) throw new SemanticConflict(); this.turns.stageTerminalProjection(mutation.input); const receipt = committed(operation, head.durableRevision); this.storeHead({ ...head, ordinal: operation.ordinal }); @@ -196,11 +214,37 @@ export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter })(); } + private control(operation: ExecutionSemanticOperation, hash: string): ExecutionWriteReceipt { + return this.db.transaction(() => { + const head = this.requireNextHead(operation); + const mutation = operation.mutation; + let next: SemanticHead; + if (mutation.kind === "stop-requested") { + if (!mutation.requestId || head.stopRequestId || head.providerOutcome + || mutation.lastAdmittedOrdinal !== operation.ordinal - 1) throw new SemanticConflict(); + next = { ...head, stopRequestId: mutation.requestId, stopWatermark: mutation.lastAdmittedOrdinal }; + } else if (mutation.kind === "provider-outcome") { + if (head.providerOutcome || head.stopRequestId && mutation.outcome !== "cancelled") throw new SemanticConflict(); + next = { ...head, providerOutcome: mutation.outcome }; + } else throw new SemanticConflict(); + const receipt = committed(operation, head.durableRevision); + this.storeHead({ ...next, ordinal: operation.ordinal }); + this.storeReceipt(operation, hash, receipt); + return receipt; + })(); + } + private hasStagedTerminalPredecessor(operation: ExecutionSemanticOperation): boolean { const previousId = `${operation.lease.leaseId}:${operation.ordinal - 1}`; return this.loadOperation(operation.execution.executionId, previousId)?.kind === "semantic:stage-terminal"; } + private validStagedFinish(operation: ExecutionSemanticOperation, head: SemanticHead): boolean { + const mutation = operation.mutation; + if (mutation.kind !== "finish" || !("kind" in mutation.input.projection)) return true; + return head.providerOutcome === mutation.outcome && this.hasStagedTerminalPredecessor(operation); + } + private requireNextHead(operation: ExecutionSemanticOperation): SemanticHead { const head = this.loadHead(operation.execution.executionId); if (!head || !nextHead(head, operation)) throw new SemanticConflict(); From ea85532504e526a49005dc1dbbcbefb7d4f65918 Mon Sep 17 00:00:00 2001 From: "(CJ) Chukwudi Nwobodo" <142016413+chuks-qua@users.noreply.github.com> Date: Thu, 24 Sep 2026 20:48:55 +0100 Subject: [PATCH 045/136] feat(server): checkpoint worker execution through writer --- ...anonical-execution-semantic-writer.test.ts | 8 ++- .../canonical-execution-writer-port.test.ts | 15 +++-- .../canonical-execution-semantic-writer.ts | 57 +++++++++++++++---- 3 files changed, 63 insertions(+), 17 deletions(-) diff --git a/apps/server/src/features/agents/canonical/__tests__/canonical-execution-semantic-writer.test.ts b/apps/server/src/features/agents/canonical/__tests__/canonical-execution-semantic-writer.test.ts index c0df75dc1..12db02c11 100644 --- a/apps/server/src/features/agents/canonical/__tests__/canonical-execution-semantic-writer.test.ts +++ b/apps/server/src/features/agents/canonical/__tests__/canonical-execution-semantic-writer.test.ts @@ -149,7 +149,7 @@ describe("CanonicalExecutionSemanticWriter through ExecutionWorkerHandler", () = .toEqual({ count: 4 }); }); - it("rejects stale leases, skipped ordinals and unsupported commands before canonical mutation", async () => { + it("rejects stale leases and skipped ordinals while checkpointing without a new event", async () => { expect((await send(1, { kind: "start", providerId: "codex", input: startInput() })).kind).toBe("committed"); const eventsBefore = db.prepare("SELECT COUNT(*) AS count FROM canonical_agent_events WHERE execution_id = ?").get(EXECUTION_ID); const append = operation(2, { kind: "append-events", events: [event()] }); @@ -158,11 +158,15 @@ describe("CanonicalExecutionSemanticWriter through ExecutionWorkerHandler", () = expect(await writer.transact({ ...append, ordinal: 3, operationId: "lease-1:3" })) .toEqual({ kind: "conflict", operationId: "lease-1:3" }); expect(await send(2, { kind: "checkpoint", phase: "running", nativeCursor: null })) + .toMatchObject({ kind: "committed", operationId: "lease-1:2" }); + expect(await send(3, { kind: "effect-result", effectId: "unsupported", settled: true })) .toEqual({ kind: "rejected", reason: "writer-conflict" }); expect(db.prepare("SELECT COUNT(*) AS count FROM canonical_agent_events WHERE execution_id = ?").get(EXECUTION_ID)) .toEqual(eventsBefore); expect(db.prepare("SELECT receipt_json FROM canonical_writer_operation_receipts WHERE execution_id = ? AND operation_id = ?") - .get(EXECUTION_ID, "semantic:head")).toMatchObject({ receipt_json: expect.stringContaining('"ordinal":1') }); + .get(EXECUTION_ID, "semantic:head")).toMatchObject({ receipt_json: expect.stringContaining('"ordinal":2') }); + expect(db.prepare("SELECT phase, native_cursor_json FROM canonical_agent_ingest_checkpoints WHERE execution_id = ?") + .get(EXECUTION_ID)).toEqual({ phase: "running", native_cursor_json: null }); }); it("rolls back terminal checkpoint and semantic receipt together when receipt storage fails", async () => { diff --git a/apps/server/src/features/agents/canonical/__tests__/canonical-execution-writer-port.test.ts b/apps/server/src/features/agents/canonical/__tests__/canonical-execution-writer-port.test.ts index 86e93a4d7..3597d0fc8 100644 --- a/apps/server/src/features/agents/canonical/__tests__/canonical-execution-writer-port.test.ts +++ b/apps/server/src/features/agents/canonical/__tests__/canonical-execution-writer-port.test.ts @@ -208,15 +208,20 @@ describe("execution semantic writer transport", () => { expect(await port.transact(op(2, { kind: "stop-requested", requestId: "stop-1", lastAdmittedOrdinal: 1, }))).toMatchObject({ kind: "committed" }); - expect(await port.transact(op(3, { kind: "provider-outcome", outcome: "completed" }))) - .toEqual({ kind: "conflict", operationId: `${lease.leaseId}:3` }); - expect(await port.transact(op(3, { kind: "provider-outcome", outcome: "cancelled" }))) + const nativeCursor = { providerId: "codex", scope: "thread", value: "native-thread", provenance: "native" }; + expect(await port.transact(op(3, { kind: "checkpoint", phase: "stopping", nativeCursor }))) .toMatchObject({ kind: "committed" }); - expect(await port.transact(op(4, { kind: "stage-terminal", input: { + expect(db.prepare("SELECT phase, native_cursor_json FROM canonical_agent_ingest_checkpoints WHERE execution_id = ?") + .get(EXECUTION_ID)).toEqual({ phase: "stopping", native_cursor_json: JSON.stringify(nativeCursor) }); + expect(await port.transact(op(4, { kind: "provider-outcome", outcome: "completed" }))) + .toEqual({ kind: "conflict", operationId: `${lease.leaseId}:4` }); + expect(await port.transact(op(4, { kind: "provider-outcome", outcome: "cancelled" }))) + .toMatchObject({ kind: "committed" }); + expect(await port.transact(op(5, { kind: "stage-terminal", input: { threadId: THREAD_ID, executionId: EXECUTION_ID, outcome: "cancelled", endedAt: NOW, assistant: { content: "Partial answer", model: null, attachments: [] }, narrative: [], } }))).toMatchObject({ kind: "committed" }); - expect(await port.transact(op(5, { kind: "finish", outcome: "cancelled", input: { + expect(await port.transact(op(6, { kind: "finish", outcome: "cancelled", input: { threadId: THREAD_ID, turnId: TURN_ID, executionId: EXECUTION_ID, providerId: "codex", providerIdentities: [], outcome: "cancelled", projection: { kind: "writer-staged" }, } }))).toMatchObject({ kind: "committed" }); diff --git a/apps/server/src/features/agents/canonical/canonical-execution-semantic-writer.ts b/apps/server/src/features/agents/canonical/canonical-execution-semantic-writer.ts index 16cab6604..1bc3fbfa0 100644 --- a/apps/server/src/features/agents/canonical/canonical-execution-semantic-writer.ts +++ b/apps/server/src/features/agents/canonical/canonical-execution-semantic-writer.ts @@ -74,6 +74,8 @@ export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter private readonly insertOperation: ReturnType; private readonly commitPendingFinish: ReturnType; private readonly updateHead: ReturnType; + private readonly updateCheckpointPhase: ReturnType; + private readonly updateCheckpoint: ReturnType; private bufferedPublication: Parameters[0][] | null = null; constructor(private readonly db: Database, private readonly publish: CanonicalAgentEventPublisher) { @@ -87,6 +89,8 @@ export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter this.insertOperation = db.prepare("INSERT INTO canonical_writer_operation_receipts (execution_id, operation_id, kind, input_hash, receipt_json) VALUES (?, ?, ?, ?, ?)"); this.commitPendingFinish = db.prepare("UPDATE canonical_writer_operation_receipts SET kind = ?, receipt_json = ? WHERE execution_id = ? AND operation_id = ? AND kind = ? AND input_hash = ?"); this.updateHead = db.prepare("UPDATE canonical_writer_operation_receipts SET receipt_json = ? WHERE execution_id = ? AND operation_id = ? AND kind = ?"); + this.updateCheckpointPhase = db.prepare("UPDATE canonical_agent_ingest_checkpoints SET phase = ?, updated_at = ? WHERE execution_id = ? AND terminal_outcome IS NULL"); + this.updateCheckpoint = db.prepare("UPDATE canonical_agent_ingest_checkpoints SET phase = ?, native_cursor_json = ?, updated_at = ? WHERE execution_id = ? AND terminal_outcome IS NULL"); } /** Commit a supported operation with a durable receipt, or reject it without changing canonical state. */ @@ -107,6 +111,7 @@ export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter switch (operation.mutation.kind) { case "begin": return this.begin(operation, hash); case "append-events": return this.append(operation, hash); + case "checkpoint": case "stop-requested": case "provider-outcome": return this.control(operation, hash); case "stage-terminal": return this.stageTerminal(operation, hash); @@ -217,16 +222,7 @@ export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter private control(operation: ExecutionSemanticOperation, hash: string): ExecutionWriteReceipt { return this.db.transaction(() => { const head = this.requireNextHead(operation); - const mutation = operation.mutation; - let next: SemanticHead; - if (mutation.kind === "stop-requested") { - if (!mutation.requestId || head.stopRequestId || head.providerOutcome - || mutation.lastAdmittedOrdinal !== operation.ordinal - 1) throw new SemanticConflict(); - next = { ...head, stopRequestId: mutation.requestId, stopWatermark: mutation.lastAdmittedOrdinal }; - } else if (mutation.kind === "provider-outcome") { - if (head.providerOutcome || head.stopRequestId && mutation.outcome !== "cancelled") throw new SemanticConflict(); - next = { ...head, providerOutcome: mutation.outcome }; - } else throw new SemanticConflict(); + const next = this.applyControlMutation(head, operation); const receipt = committed(operation, head.durableRevision); this.storeHead({ ...next, ordinal: operation.ordinal }); this.storeReceipt(operation, hash, receipt); @@ -234,6 +230,47 @@ export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter })(); } + private applyControlMutation(head: SemanticHead, operation: ExecutionSemanticOperation): SemanticHead { + const mutation = operation.mutation; + switch (mutation.kind) { + case "stop-requested": return this.recordStop(head, operation, mutation); + case "provider-outcome": return this.recordProviderOutcome(head, mutation.outcome); + case "checkpoint": return this.recordCheckpoint(head, operation.execution.executionId, mutation); + default: throw new SemanticConflict(); + } + } + + private recordStop( + head: SemanticHead, + operation: ExecutionSemanticOperation, + mutation: Extract, + ): SemanticHead { + if (!mutation.requestId || head.stopRequestId || head.providerOutcome + || mutation.lastAdmittedOrdinal !== operation.ordinal - 1) throw new SemanticConflict(); + if (this.updateCheckpointPhase.run("stopping", new Date().toISOString(), operation.execution.executionId).changes !== 1) { + throw new SemanticConflict(); + } + return { ...head, stopRequestId: mutation.requestId, stopWatermark: mutation.lastAdmittedOrdinal }; + } + + private recordProviderOutcome(head: SemanticHead, outcome: TurnOutcome): SemanticHead { + if (head.providerOutcome || head.stopRequestId && outcome !== "cancelled") throw new SemanticConflict(); + return { ...head, providerOutcome: outcome }; + } + + private recordCheckpoint( + head: SemanticHead, + executionId: string, + mutation: Extract, + ): SemanticHead { + const cursor = mutation.nativeCursor === null ? null : JSON.stringify(mutation.nativeCursor); + if (!mutation.phase || mutation.phase.length > 64 || cursor === undefined + || this.updateCheckpoint.run(mutation.phase, cursor, new Date().toISOString(), executionId).changes !== 1) { + throw new SemanticConflict(); + } + return head; + } + private hasStagedTerminalPredecessor(operation: ExecutionSemanticOperation): boolean { const previousId = `${operation.lease.leaseId}:${operation.ordinal - 1}`; return this.loadOperation(operation.execution.executionId, previousId)?.kind === "semantic:stage-terminal"; From 5087a1ecc4b74ff9642354a9f8e4ea0ffdf17f03 Mon Sep 17 00:00:00 2001 From: "(CJ) Chukwudi Nwobodo" <142016413+chuks-qua@users.noreply.github.com> Date: Thu, 24 Sep 2026 20:51:00 +0100 Subject: [PATCH 046/136] feat(server): carry provider phase and cursor through mailbox --- .../canonical-execution-semantic-writer.test.ts | 12 ++++++------ .../canonical-execution-writer-port.test.ts | 5 ++++- .../canonical/canonical-execution-semantic-writer.ts | 6 ++++-- .../__tests__/execution-thread-worker-port.test.ts | 2 +- .../__tests__/execution-worker-handler.test.ts | 10 +++++----- .../agents/execution/execution-worker-handler.ts | 6 +++--- 6 files changed, 23 insertions(+), 18 deletions(-) diff --git a/apps/server/src/features/agents/canonical/__tests__/canonical-execution-semantic-writer.test.ts b/apps/server/src/features/agents/canonical/__tests__/canonical-execution-semantic-writer.test.ts index 12db02c11..fe2f63288 100644 --- a/apps/server/src/features/agents/canonical/__tests__/canonical-execution-semantic-writer.test.ts +++ b/apps/server/src/features/agents/canonical/__tests__/canonical-execution-semantic-writer.test.ts @@ -115,7 +115,7 @@ describe("CanonicalExecutionSemanticWriter through ExecutionWorkerHandler", () = it("commits start, event, and finalization with durable receipts across a database reload", async () => { const begin = operation(1, { kind: "begin", providerId: "codex", input: startInput() }); expect((await send(1, { kind: "start", providerId: "codex", input: startInput() })).kind).toBe("committed"); - expect((await send(2, { kind: "event", events: [event()] })).kind).toBe("committed"); + expect((await send(2, { kind: "event", phase: "running", nativeCursor: null, events: [event()] })).kind).toBe("committed"); const staged = new MessageRepo(db).create(THREAD_ID, "assistant", "Answer", 2, undefined, undefined, undefined, "model", true); const finish = { threadId: THREAD_ID, @@ -152,7 +152,7 @@ describe("CanonicalExecutionSemanticWriter through ExecutionWorkerHandler", () = it("rejects stale leases and skipped ordinals while checkpointing without a new event", async () => { expect((await send(1, { kind: "start", providerId: "codex", input: startInput() })).kind).toBe("committed"); const eventsBefore = db.prepare("SELECT COUNT(*) AS count FROM canonical_agent_events WHERE execution_id = ?").get(EXECUTION_ID); - const append = operation(2, { kind: "append-events", events: [event()] }); + const append = operation(2, { kind: "append-events", phase: "running", nativeCursor: null, events: [event()] }); expect(await writer.transact({ ...append, lease: { ...lease, ownerEpoch: 2, leaseId: "lease-2" }, operationId: "lease-2:2" })) .toEqual({ kind: "conflict", operationId: "lease-2:2" }); expect(await writer.transact({ ...append, ordinal: 3, operationId: "lease-1:3" })) @@ -199,14 +199,14 @@ describe("CanonicalExecutionSemanticWriter through ExecutionWorkerHandler", () = const before = db.prepare("SELECT last_durable_sequence FROM canonical_agent_ingest_checkpoints WHERE execution_id = ?") .get(EXECUTION_ID); db.run("CREATE TRIGGER fail_semantic_event_receipt BEFORE INSERT ON canonical_writer_operation_receipts WHEN NEW.kind = 'semantic:append-events' BEGIN SELECT RAISE(ABORT, 'event receipt unavailable'); END"); - await expect(send(2, { kind: "event", events: [event()] })).rejects.toThrow("event receipt unavailable"); + await expect(send(2, { kind: "event", phase: "running", nativeCursor: null, events: [event()] })).rejects.toThrow("event receipt unavailable"); expect(db.prepare("SELECT last_durable_sequence FROM canonical_agent_ingest_checkpoints WHERE execution_id = ?") .get(EXECUTION_ID)).toEqual(before); expect(db.prepare("SELECT id FROM canonical_agent_items WHERE id = ?").get("item-1")).toBeNull(); expect(published).not.toContain(event().eventId); db.run("DROP TRIGGER fail_semantic_event_receipt"); - expect((await send(2, { kind: "event", events: [event()] })).kind).toBe("committed"); + expect((await send(2, { kind: "event", phase: "running", nativeCursor: null, events: [event()] })).kind).toBe("committed"); expect(published).toContain(event().eventId); }); @@ -219,7 +219,7 @@ describe("CanonicalExecutionSemanticWriter through ExecutionWorkerHandler", () = handler = new ExecutionWorkerHandler(writer); expect((await send(1, { kind: "start", providerId: "codex", input: startInput() })).kind).toBe("committed"); failPublication = true; - await expect(send(2, { kind: "event", events: [event()] })).rejects.toThrow("publisher unavailable"); + await expect(send(2, { kind: "event", phase: "running", nativeCursor: null, events: [event()] })).rejects.toThrow("publisher unavailable"); expect(db.prepare("SELECT event_id FROM canonical_agent_events WHERE event_id = ?").get(event().eventId)) .toEqual({ event_id: event().eventId }); const receipt = db.prepare("SELECT receipt_json FROM canonical_writer_operation_receipts WHERE execution_id = ? AND operation_id = ?") @@ -232,7 +232,7 @@ describe("CanonicalExecutionSemanticWriter through ExecutionWorkerHandler", () = writer = new CanonicalExecutionSemanticWriter(db, (events) => { published.push(...events.map((item) => item.eventId)); }); - expect(await writer.transact(operation(2, { kind: "append-events", events: [event()] }))) + expect(await writer.transact(operation(2, { kind: "append-events", phase: "running", nativeCursor: null, events: [event()] }))) .toMatchObject({ kind: "committed", operationId: "lease-1:2" }); expect(published).toContain(event().eventId); }); diff --git a/apps/server/src/features/agents/canonical/__tests__/canonical-execution-writer-port.test.ts b/apps/server/src/features/agents/canonical/__tests__/canonical-execution-writer-port.test.ts index 3597d0fc8..39157b6d2 100644 --- a/apps/server/src/features/agents/canonical/__tests__/canonical-execution-writer-port.test.ts +++ b/apps/server/src/features/agents/canonical/__tests__/canonical-execution-writer-port.test.ts @@ -257,7 +257,8 @@ describe("execution semantic writer transport", () => { if (start.kind !== "begin") throw new Error("Unexpected begin operation"); await expect(send(claim.lease, { kind: "start", providerId: "codex", input: start.input })) .resolves.toMatchObject({ kind: "reply", result: { kind: "committed" } }); - await expect(send(claim.lease, { kind: "event", events: [{ + const nativeCursor = { providerId: "codex", scope: "thread", value: "native-worker-thread", provenance: "native" }; + await expect(send(claim.lease, { kind: "event", phase: "running", nativeCursor, events: [{ eventId: `${EXECUTION_ID}:worker-item`, routing: { ...execution, itemId: "worker-item" }, sourceProviderId: "codex", sourceIdentities: [], sourceSequence: 1, @@ -267,6 +268,8 @@ describe("execution semantic writer transport", () => { createdAt: NOW, updatedAt: NOW, } }, }] })).resolves.toMatchObject({ kind: "reply", result: { kind: "committed" } }); + expect(db.prepare("SELECT native_cursor_json FROM canonical_agent_ingest_checkpoints WHERE execution_id = ?") + .get(EXECUTION_ID)).toEqual({ native_cursor_json: JSON.stringify(nativeCursor) }); await expect(send(claim.lease, { kind: "provider-outcome", outcome: "completed" })) .resolves.toMatchObject({ kind: "reply", result: { kind: "committed" } }); await expect(send(claim.lease, { kind: "stage-terminal", input: { diff --git a/apps/server/src/features/agents/canonical/canonical-execution-semantic-writer.ts b/apps/server/src/features/agents/canonical/canonical-execution-semantic-writer.ts index 1bc3fbfa0..965778950 100644 --- a/apps/server/src/features/agents/canonical/canonical-execution-semantic-writer.ts +++ b/apps/server/src/features/agents/canonical/canonical-execution-semantic-writer.ts @@ -161,7 +161,8 @@ export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter private append(operation: ExecutionSemanticOperation, hash: string): ExecutionWriteReceipt { const mutation = operation.mutation; - if (mutation.kind !== "append-events" || mutation.events.length === 0 + if (mutation.kind !== "append-events" || !mutation.phase || mutation.phase.length > 64 + || mutation.events.length === 0 || mutation.events.some((event) => event.routing.threadId !== operation.execution.threadId || event.routing.turnId !== operation.execution.turnId || event.routing.executionId !== operation.execution.executionId)) return conflict(operation); @@ -169,7 +170,8 @@ export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter const head = this.requireNextHead(operation); const result = this.turns.append({ ...operation.execution, - phase: "running", + phase: mutation.phase, + nativeCursor: mutation.nativeCursor, events: mutation.events, }); if (result.outcome !== "committed") throw new SemanticConflict(); diff --git a/apps/server/src/features/agents/execution/__tests__/execution-thread-worker-port.test.ts b/apps/server/src/features/agents/execution/__tests__/execution-thread-worker-port.test.ts index 99ba3ac89..18ebfb0f6 100644 --- a/apps/server/src/features/agents/execution/__tests__/execution-thread-worker-port.test.ts +++ b/apps/server/src/features/agents/execution/__tests__/execution-thread-worker-port.test.ts @@ -122,7 +122,7 @@ describe("ExecutionThreadWorkerPort", () => { await expect(submit(scheduler, lease, { kind: "start", providerId: "codex", input: START_INPUT })) .resolves.toMatchObject({ kind: "reply", result: { kind: "committed", durableRevision: 1 } }); await expect(port.whenReady()).resolves.toBe(true); - await expect(submit(scheduler, lease, { kind: "event", events: [eventDraft()] })) + await expect(submit(scheduler, lease, { kind: "event", phase: "running", nativeCursor: null, events: [eventDraft()] })) .resolves.toMatchObject({ kind: "reply", result: { kind: "committed", durableRevision: 2 } }); await expect(submit(scheduler, lease, { kind: "stop", requestId: "stop-real-worker" })) .resolves.toMatchObject({ kind: "reply", result: { kind: "committed", durableRevision: 3 } }); diff --git a/apps/server/src/features/agents/execution/__tests__/execution-worker-handler.test.ts b/apps/server/src/features/agents/execution/__tests__/execution-worker-handler.test.ts index bdeea3c97..5aa2830a8 100644 --- a/apps/server/src/features/agents/execution/__tests__/execution-worker-handler.test.ts +++ b/apps/server/src/features/agents/execution/__tests__/execution-worker-handler.test.ts @@ -151,12 +151,12 @@ describe("ExecutionWorkerHandler through its scheduler", () => { it("runs one synthetic turn from start through Stop, checkpoint, and finalization", async () => { const { scheduler, worker, writer, lease } = fixture(); await committed(submit(scheduler, lease, { kind: "start", providerId: "codex", input: START_INPUT }), 1); - await committed(submit(scheduler, lease, { kind: "event", events: [eventDraft()] }), 2); + await committed(submit(scheduler, lease, { kind: "event", phase: "running", nativeCursor: null, events: [eventDraft()] }), 2); const stop = submit(scheduler, lease, { kind: "stop", requestId: "stop-1" }); expect(scheduler.submit({ execution: EXECUTION, lease, - command: { kind: "event", events: [eventDraft()] }, + command: { kind: "event", phase: "running", nativeCursor: null, events: [eventDraft()] }, byteLength: 1_000, })).toEqual({ kind: "stopping" }); await committed(stop, 3); @@ -189,7 +189,7 @@ describe("ExecutionWorkerHandler through its scheduler", () => { writer.beforeCommit = () => new Promise((resolve) => { releaseWrite = resolve; }); const { scheduler, worker, lease } = fixture(writer); const start = submit(scheduler, lease, { kind: "start", providerId: "codex", input: START_INPUT }); - const event = submit(scheduler, lease, { kind: "event", events: [eventDraft()] }); + const event = submit(scheduler, lease, { kind: "event", phase: "running", nativeCursor: null, events: [eventDraft()] }); await Promise.resolve(); expect(worker.requests).toHaveLength(1); expect(scheduler.depth()).toMatchObject({ pending: 2 }); @@ -225,7 +225,7 @@ describe("ExecutionWorkerHandler through its scheduler", () => { writer.conflictKind = "append-events"; const { scheduler, lease } = fixture(writer); await committed(submit(scheduler, lease, { kind: "start", providerId: "codex", input: START_INPUT }), 1); - await expect(submit(scheduler, lease, { kind: "event", events: [eventDraft()] }).completion).resolves.toEqual({ + await expect(submit(scheduler, lease, { kind: "event", phase: "running", nativeCursor: null, events: [eventDraft()] }).completion).resolves.toEqual({ kind: "reply", result: { kind: "rejected", reason: "writer-conflict" }, }); @@ -241,7 +241,7 @@ describe("ExecutionWorkerHandler through its scheduler", () => { }; const { scheduler, worker, lost, lease } = fixture(writer); await committed(submit(scheduler, lease, { kind: "start", providerId: "codex", input: START_INPUT }), 1); - await expect(submit(scheduler, lease, { kind: "event", events: [eventDraft()] }).completion) + await expect(submit(scheduler, lease, { kind: "event", phase: "running", nativeCursor: null, events: [eventDraft()] }).completion) .resolves.toEqual({ kind: "worker-lost" }); expect(lost).toEqual([[{ execution: EXECUTION, lease }]]); expect(worker.terminated).toBe(true); diff --git a/apps/server/src/features/agents/execution/execution-worker-handler.ts b/apps/server/src/features/agents/execution/execution-worker-handler.ts index c7ed3d965..dccd4511f 100644 --- a/apps/server/src/features/agents/execution/execution-worker-handler.ts +++ b/apps/server/src/features/agents/execution/execution-worker-handler.ts @@ -18,7 +18,7 @@ import type { ExecutionMailboxCommand } from "./execution-mailbox-scheduler.js"; export type ExecutionWorkCommand = | { readonly kind: "start"; readonly providerId: string; readonly input: DataOnlyParentTurnStartInput } | { readonly kind: "resume"; readonly providerId: string; readonly checkpointId: string } - | { readonly kind: "event"; readonly events: readonly ProviderEventDraft[] } + | { readonly kind: "event"; readonly phase: string; readonly nativeCursor: unknown | null; readonly events: readonly ProviderEventDraft[] } | { readonly kind: "checkpoint"; readonly phase: string; readonly nativeCursor: unknown | null } | { readonly kind: "effect-result"; readonly effectId: string; readonly settled: boolean } | { readonly kind: "provider-outcome"; readonly outcome: TurnOutcome } @@ -35,7 +35,7 @@ export interface ExecutionSemanticOperation { readonly mutation: | { readonly kind: "begin"; readonly providerId: string; readonly input: DataOnlyParentTurnStartInput } | { readonly kind: "resume"; readonly providerId: string; readonly checkpointId: string } - | { readonly kind: "append-events"; readonly events: readonly ProviderEventDraft[] } + | { readonly kind: "append-events"; readonly phase: string; readonly nativeCursor: unknown | null; readonly events: readonly ProviderEventDraft[] } | { readonly kind: "checkpoint"; readonly phase: string; readonly nativeCursor: unknown | null } | { readonly kind: "stop-requested"; readonly requestId: string; readonly lastAdmittedOrdinal: number } | { readonly kind: "effect-result"; readonly effectId: string; readonly settled: boolean } @@ -203,7 +203,7 @@ function eventMutation( state: ExecutionState, ): ExecutionSemanticOperation["mutation"] | undefined { if (state.phase !== "running" || !validEventRouting(command.events, execution)) return undefined; - return { kind: "append-events", events: command.events }; + return { kind: "append-events", phase: command.phase, nativeCursor: command.nativeCursor, events: command.events }; } function stopMutation( From 84a80dbe90476c7708a2bd13045ad239f6277a4e Mon Sep 17 00:00:00 2001 From: "(CJ) Chukwudi Nwobodo" <142016413+chuks-qua@users.noreply.github.com> Date: Thu, 24 Sep 2026 20:53:32 +0100 Subject: [PATCH 047/136] feat(server): retain provider commit receipts across worker retries --- ...anonical-execution-semantic-writer.test.ts | 5 +++- .../canonical-execution-writer-port.test.ts | 5 +++- .../canonical-execution-semantic-writer.ts | 27 ++++++++++++++++--- .../execution/execution-worker-handler.ts | 22 ++++++++++++--- 4 files changed, 50 insertions(+), 9 deletions(-) diff --git a/apps/server/src/features/agents/canonical/__tests__/canonical-execution-semantic-writer.test.ts b/apps/server/src/features/agents/canonical/__tests__/canonical-execution-semantic-writer.test.ts index fe2f63288..117f5303a 100644 --- a/apps/server/src/features/agents/canonical/__tests__/canonical-execution-semantic-writer.test.ts +++ b/apps/server/src/features/agents/canonical/__tests__/canonical-execution-semantic-writer.test.ts @@ -233,7 +233,10 @@ describe("CanonicalExecutionSemanticWriter through ExecutionWorkerHandler", () = published.push(...events.map((item) => item.eventId)); }); expect(await writer.transact(operation(2, { kind: "append-events", phase: "running", nativeCursor: null, events: [event()] }))) - .toMatchObject({ kind: "committed", operationId: "lease-1:2" }); + .toMatchObject({ + kind: "committed", operationId: "lease-1:2", + providerCommit: { outcome: "committed", eventCount: 1 }, + }); expect(published).toContain(event().eventId); }); diff --git a/apps/server/src/features/agents/canonical/__tests__/canonical-execution-writer-port.test.ts b/apps/server/src/features/agents/canonical/__tests__/canonical-execution-writer-port.test.ts index 39157b6d2..49afef0bf 100644 --- a/apps/server/src/features/agents/canonical/__tests__/canonical-execution-writer-port.test.ts +++ b/apps/server/src/features/agents/canonical/__tests__/canonical-execution-writer-port.test.ts @@ -267,7 +267,10 @@ describe("execution semantic writer transport", () => { providerIdentities: [], payload: { projection: "message", content: "Worker event" }, createdAt: NOW, updatedAt: NOW, } }, - }] })).resolves.toMatchObject({ kind: "reply", result: { kind: "committed" } }); + }] })).resolves.toMatchObject({ + kind: "reply", + result: { kind: "committed", providerCommit: { outcome: "committed", eventCount: 1 } }, + }); expect(db.prepare("SELECT native_cursor_json FROM canonical_agent_ingest_checkpoints WHERE execution_id = ?") .get(EXECUTION_ID)).toEqual({ native_cursor_json: JSON.stringify(nativeCursor) }); await expect(send(claim.lease, { kind: "provider-outcome", outcome: "completed" })) diff --git a/apps/server/src/features/agents/canonical/canonical-execution-semantic-writer.ts b/apps/server/src/features/agents/canonical/canonical-execution-semantic-writer.ts index 965778950..c8460c7c7 100644 --- a/apps/server/src/features/agents/canonical/canonical-execution-semantic-writer.ts +++ b/apps/server/src/features/agents/canonical/canonical-execution-semantic-writer.ts @@ -5,6 +5,7 @@ import { z } from "zod"; import type { ExecutionIdentity, ExecutionLease } from "../execution/execution-mailbox-protocol.js"; import type { + ExecutionProviderCommitReceipt, ExecutionSemanticOperation, ExecutionSemanticWriter, ExecutionWriteReceipt, @@ -40,6 +41,14 @@ const storedReceiptSchema = z.object({ operationId: z.string(), durableRevision: z.number().int(), publicationVersion: z.literal(1), + providerCommit: z.object({ + outcome: z.enum(["committed", "duplicate", "conflict", "terminal-outcome-confirmed", "ingest-overflow"]), + conversationRevision: z.number().int(), + rosterRevision: z.number().int(), + acceptedThrough: z.number().int(), + durableThrough: z.number().int(), + eventCount: z.number().int().nonnegative(), + }).optional(), }); const storedOperationSchema = z.object({ kind: z.string(), @@ -175,7 +184,15 @@ export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter events: mutation.events, }); if (result.outcome !== "committed") throw new SemanticConflict(); - const receipt = committed(operation, result.durableThrough); + const providerCommit: ExecutionProviderCommitReceipt = { + outcome: result.outcome, + conversationRevision: result.conversationRevision, + rosterRevision: result.rosterRevision, + acceptedThrough: result.acceptedThrough, + durableThrough: result.durableThrough, + eventCount: result.events.length, + }; + const receipt = committed(operation, result.durableThrough, providerCommit); this.storeHead({ ...head, ordinal: operation.ordinal, durableRevision: receipt.durableRevision }); this.storePublicationChunks(operation.execution.executionId, hash, result.events.map((event) => event.acceptedSequence)); this.storeReceipt(operation, hash, receipt); @@ -360,7 +377,7 @@ export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter if (stored.kind !== `semantic:${operation.mutation.kind}` || stored.input_hash !== hash) return conflict(operation); const receipt = storedReceiptSchema.parse(JSON.parse(stored.receipt_json)); return receipt.operationId === operation.operationId && Number.isSafeInteger(receipt.durableRevision) - ? committed(operation, receipt.durableRevision) : conflict(operation); + ? committed(operation, receipt.durableRevision, receipt.providerCommit) : conflict(operation); } private publishStoredEvents(executionId: string, hash: string): void { @@ -436,8 +453,12 @@ function nextHead(head: SemanticHead, operation: ExecutionSemanticOperation): bo function committed( operation: ExecutionSemanticOperation, durableRevision: number, + providerCommit?: ExecutionProviderCommitReceipt, ): Extract { - return { kind: "committed", operationId: operation.operationId, durableRevision }; + return { + kind: "committed", operationId: operation.operationId, durableRevision, + ...(providerCommit ? { providerCommit } : {}), + }; } function conflict(operation: ExecutionSemanticOperation): ExecutionWriteReceipt { diff --git a/apps/server/src/features/agents/execution/execution-worker-handler.ts b/apps/server/src/features/agents/execution/execution-worker-handler.ts index dccd4511f..974efa317 100644 --- a/apps/server/src/features/agents/execution/execution-worker-handler.ts +++ b/apps/server/src/features/agents/execution/execution-worker-handler.ts @@ -6,6 +6,7 @@ import type { DataOnlyParentTurnFinishInput, DataOnlyParentTurnStartInput, } from "../canonical/canonical-parent-turn-write.js"; +import type { CanonicalAgentCommitResult } from "../canonical/canonical-agent-boundary.js"; import type { ExecutionIdentity, ExecutionLease, @@ -44,9 +45,15 @@ export interface ExecutionSemanticOperation { | { readonly kind: "finish"; readonly outcome: TurnOutcome; readonly input: DataOnlyParentTurnFinishInput }; } +/** Provider-facing receipt values retained with the execution operation. */ +export type ExecutionProviderCommitReceipt = Pick< + CanonicalAgentCommitResult, + "outcome" | "conversationRevision" | "rosterRevision" | "acceptedThrough" | "durableThrough" +> & { readonly eventCount: number }; + /** A writer reply is valid only after the semantic operation commits durably. */ export type ExecutionWriteReceipt = - | { readonly kind: "committed"; readonly operationId: string; readonly durableRevision: number } + | { readonly kind: "committed"; readonly operationId: string; readonly durableRevision: number; readonly providerCommit?: ExecutionProviderCommitReceipt } | { readonly kind: "conflict"; readonly operationId: string }; /** @@ -60,7 +67,7 @@ export interface ExecutionSemanticWriter { /** A command result that never calls an uncommitted mutation successful. */ export type ExecutionWorkerResult = - | { readonly kind: "committed"; readonly operationId: string; readonly durableRevision: number } + | { readonly kind: "committed"; readonly operationId: string; readonly durableRevision: number; readonly providerCommit?: ExecutionProviderCommitReceipt } | { readonly kind: "released" } | { readonly kind: "rejected"; readonly reason: "no-execution" | "stale-execution" | "out-of-order" | "invalid-transition" | "invalid-event-routing" | "invalid-stop-watermark" | "writer-conflict" }; @@ -127,7 +134,7 @@ export class ExecutionWorkerHandler { state.durableRevision = receipt.durableRevision; if (request.command.kind === "stop") state.phase = "stopping"; if (request.command.kind === "finalize") state.phase = "finalized"; - return { kind: "committed", operationId: receipt.operationId, durableRevision: receipt.durableRevision }; + return committedResult(receipt); } private async begin( @@ -156,7 +163,7 @@ export class ExecutionWorkerHandler { phase: "running", durableRevision: receipt.durableRevision, }); - return { kind: "committed", operationId: receipt.operationId, durableRevision: receipt.durableRevision }; + return committedResult(receipt); } private release(request: ExecutionWorkerRequest, state: ExecutionState): ExecutionWorkerResult { @@ -294,6 +301,13 @@ function isDurableReceipt( && Number.isSafeInteger(receipt.durableRevision) && receipt.durableRevision >= previousRevision; } +function committedResult(receipt: Extract): ExecutionWorkerResult { + return { + kind: "committed", operationId: receipt.operationId, durableRevision: receipt.durableRevision, + ...(receipt.providerCommit ? { providerCommit: receipt.providerCommit } : {}), + }; +} + function sameIdentity(left: ExecutionIdentity, right: ExecutionIdentity): boolean { return left.threadId === right.threadId && left.turnId === right.turnId && left.executionId === right.executionId; } From 008a97ee5a179b7e08a2a574422b1065edc9fae0 Mon Sep 17 00:00:00 2001 From: "(CJ) Chukwudi Nwobodo" <142016413+chuks-qua@users.noreply.github.com> Date: Thu, 24 Sep 2026 21:00:05 +0100 Subject: [PATCH 048/136] fix(server): allow terminal staging after execution Stop --- .../execution/__tests__/execution-mailbox-scheduler.test.ts | 5 ++++- .../features/agents/execution/execution-mailbox-scheduler.ts | 4 +++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/apps/server/src/features/agents/execution/__tests__/execution-mailbox-scheduler.test.ts b/apps/server/src/features/agents/execution/__tests__/execution-mailbox-scheduler.test.ts index 940612ae8..afe453840 100644 --- a/apps/server/src/features/agents/execution/__tests__/execution-mailbox-scheduler.test.ts +++ b/apps/server/src/features/agents/execution/__tests__/execution-mailbox-scheduler.test.ts @@ -17,6 +17,7 @@ type Work = | { readonly kind: "event"; readonly sequence: number } | { readonly kind: "checkpoint"; readonly revision: number } | { readonly kind: "effect-result"; readonly effectId: string } + | { readonly kind: "stage-terminal" } | { readonly kind: "finalize" }; type Command = Work | { readonly kind: "stop"; readonly requestId: string }; type Result = { readonly revision: number }; @@ -193,12 +194,14 @@ describe("ExecutionMailboxScheduler", () => { .toEqual({ kind: "stopping" }); expect(scheduler.submit({ execution: identity, lease, command: { kind: "stop", requestId: "again" }, byteLength: 100 })) .toEqual({ kind: "stop-already-requested" }); + const staged = admitted(scheduler, identity, lease, { kind: "stage-terminal" }); const finalize = admitted(scheduler, identity, lease, { kind: "finalize" }); - for (let index = 3; index < 6; index += 1) workers[0]?.reply(request(workers[0]!, index), index + 1); + for (let index = 3; index < 7; index += 1) workers[0]?.reply(request(workers[0]!, index), index + 1); expect(await start.completion).toEqual({ kind: "reply", result: { revision: 1 } }); expect(await event.completion).toEqual({ kind: "reply", result: { revision: 2 } }); expect((await checkpoint.completion).kind).toBe("reply"); expect((await effect.completion).kind).toBe("reply"); + expect((await staged.completion).kind).toBe("reply"); expect((await finalize.completion).kind).toBe("reply"); expect(scheduler.release(identity, lease)).toBe(true); scheduler.shutdown(); diff --git a/apps/server/src/features/agents/execution/execution-mailbox-scheduler.ts b/apps/server/src/features/agents/execution/execution-mailbox-scheduler.ts index 9f6856651..715a1a112 100644 --- a/apps/server/src/features/agents/execution/execution-mailbox-scheduler.ts +++ b/apps/server/src/features/agents/execution/execution-mailbox-scheduler.ts @@ -15,7 +15,9 @@ export type ExecutionMailboxCommand = | { readonly kind: "stop"; readonly requestId: string }; /** Lifecycle results that must be allowed to settle after Stop is admitted. */ -export const EXECUTION_CONTROL_KINDS = ["checkpoint", "effect-result", "provider-outcome", "finalize", "release"] as const; +export const EXECUTION_CONTROL_KINDS = [ + "checkpoint", "effect-result", "provider-outcome", "stage-terminal", "finalize", "release", +] as const; export type ExecutionControlKind = typeof EXECUTION_CONTROL_KINDS[number]; const CONTROL_KINDS: ReadonlySet = new Set(EXECUTION_CONTROL_KINDS); From f5bfb217dc9b9535211601f50074123f44d0e9a2 Mon Sep 17 00:00:00 2001 From: "(CJ) Chukwudi Nwobodo" <142016413+chuks-qua@users.noreply.github.com> Date: Thu, 24 Sep 2026 21:01:06 +0100 Subject: [PATCH 049/136] test(server): prove stopped worker preserves partial assistant --- .../canonical-execution-writer-port.test.ts | 41 +++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/apps/server/src/features/agents/canonical/__tests__/canonical-execution-writer-port.test.ts b/apps/server/src/features/agents/canonical/__tests__/canonical-execution-writer-port.test.ts index 49afef0bf..a21beeef8 100644 --- a/apps/server/src/features/agents/canonical/__tests__/canonical-execution-writer-port.test.ts +++ b/apps/server/src/features/agents/canonical/__tests__/canonical-execution-writer-port.test.ts @@ -296,4 +296,45 @@ describe("execution semantic writer transport", () => { scheduler.shutdown(); } }); + + it("stages a partial assistant after Stop before the cancelled terminal commit", async () => { + writer = new CanonicalAgentWriterClient(NodePath.join(directory, "app.sqlite")); + const port = new CanonicalExecutionWriterPort(writer, () => {}); + const scheduler = new ExecutionMailboxScheduler({ + workerCount: 1, + limits: { + maxPending: 16, maxPendingBytes: 64_000, reservedControl: 4, reservedControlBytes: 16_000, + maxPerExecutionPending: 12, maxPerExecutionBytes: 48_000, + reservedPerExecutionControl: 3, reservedPerExecutionControlBytes: 12_000, + }, + createWorker: () => new ExecutionThreadWorkerPort(port), + onWorkerLost: () => { throw new Error("Execution worker was lost"); }, + }); + try { + const claim = scheduler.claim(execution, 1); + if (claim.kind !== "claimed") throw new Error(`Execution claim failed: ${claim.kind}`); + const send = (command: Parameters[0]["command"]) => { + const admission = scheduler.submit({ execution, lease: claim.lease, command, byteLength: 1_000 }); + if (admission.kind !== "admitted") throw new Error(`Execution admission failed: ${admission.kind}`); + return admission.completion; + }; + const start = beginOperation().mutation; + if (start.kind !== "begin") throw new Error("Unexpected begin operation"); + expect((await send({ kind: "start", providerId: "codex", input: start.input })).kind).toBe("reply"); + expect((await send({ kind: "stop", requestId: "stop-partial" })).kind).toBe("reply"); + expect((await send({ kind: "provider-outcome", outcome: "cancelled" })).kind).toBe("reply"); + expect((await send({ kind: "stage-terminal", input: { + threadId: THREAD_ID, executionId: EXECUTION_ID, outcome: "cancelled", endedAt: NOW, + assistant: { content: "Partial answer", model: null, attachments: [] }, narrative: [], + } })).kind).toBe("reply"); + await expect(send({ kind: "finalize", outcome: "cancelled", input: { + threadId: THREAD_ID, turnId: TURN_ID, executionId: EXECUTION_ID, providerId: "codex", + providerIdentities: [], outcome: "cancelled", projection: { kind: "writer-staged" }, + } })).resolves.toMatchObject({ kind: "reply", result: { kind: "committed" } }); + expect(db.prepare("SELECT content, outcome FROM messages WHERE role = 'assistant' AND is_internal = 0").get()) + .toEqual({ content: "Partial answer", outcome: "cancelled" }); + } finally { + scheduler.shutdown(); + } + }); }); From 1d6ff42cb16225c01d0ce4cd167f8b6115b69dbc Mon Sep 17 00:00:00 2001 From: "(CJ) Chukwudi Nwobodo" <142016413+chuks-qua@users.noreply.github.com> Date: Thu, 24 Sep 2026 21:04:28 +0100 Subject: [PATCH 050/136] refactor(server): prepare cloneable parent turn starts during admission --- .../canonical-parent-turn-write.test.ts | 20 ++++ .../canonical/canonical-parent-turn-write.ts | 56 ++++----- .../turn-admission-dispatch-coordinator.ts | 111 ++++++++---------- 3 files changed, 99 insertions(+), 88 deletions(-) diff --git a/apps/server/src/features/agents/canonical/__tests__/canonical-parent-turn-write.test.ts b/apps/server/src/features/agents/canonical/__tests__/canonical-parent-turn-write.test.ts index 55e419691..6b6be5b13 100644 --- a/apps/server/src/features/agents/canonical/__tests__/canonical-parent-turn-write.test.ts +++ b/apps/server/src/features/agents/canonical/__tests__/canonical-parent-turn-write.test.ts @@ -8,6 +8,7 @@ import { afterEach, beforeEach, describe, expect, it } from "vitest"; import { openDatabase } from "../../../../runtime/persistence/sqlite/database.js"; import { MessageRepo } from "../../conversation/persistence/message-repo.js"; +import { PlanQuestionAnswersRepo } from "../../planning/persistence/plan-question-answers-repo.js"; import { ToolCallRecordRepo } from "../../tools/persistence/tool-call-record-repo.js"; import { CanonicalParentTurnWrite, @@ -171,6 +172,25 @@ describe("CanonicalParentTurnWrite", () => { expect(published).toEqual([]); }); + it("reuses a queued user message while reopening the thread and answering a plan question", () => { + const messages = new MessageRepo(db); + const queued = messages.create(THREAD_ID, "user", "Queued answer", 1); + const plan = messages.create(THREAD_ID, "assistant", "Question", 2); + db.prepare("UPDATE threads SET user_completed_at = ? WHERE id = ?").run(NOW, THREAD_ID); + const input: DataOnlyParentTurnStartInput = { + ...startInput(), + userMessage: { kind: "existing", messageId: queued.id }, + reopenThread: true, + answeredPlanQuestionMessageId: plan.id, + }; + + expect(writer.start(structuredClone(input)).outcome).toBe("committed"); + expect(db.prepare("SELECT COUNT(*) AS count FROM messages WHERE role = 'user'").get()).toEqual({ count: 1 }); + expect(db.prepare("SELECT user_completed_at FROM threads WHERE id = ?").get(THREAD_ID)) + .toEqual({ user_completed_at: null }); + expect(new PlanQuestionAnswersRepo(db).isAnswered(plan.id)).toBe(true); + }); + it("does not confirm a terminal checkpoint when assistant publication fails", async () => { writer.start(startInput()); const staged = new MessageRepo(db).create(THREAD_ID, "assistant", "Answer", 2, undefined, undefined, undefined, "model", true); diff --git a/apps/server/src/features/agents/canonical/canonical-parent-turn-write.ts b/apps/server/src/features/agents/canonical/canonical-parent-turn-write.ts index 504761c57..45d961ddc 100644 --- a/apps/server/src/features/agents/canonical/canonical-parent-turn-write.ts +++ b/apps/server/src/features/agents/canonical/canonical-parent-turn-write.ts @@ -50,6 +50,33 @@ export type ParentUserMessageWrite = selectedTextComments?: CreateMessageArgument[13]; }; +/** Project prepared user-message data on the transaction's SQLite connection. */ +export function projectParentUserMessage(messages: MessageRepo, threadId: string, userMessage: ParentUserMessageWrite) { + if (userMessage.kind === "existing") { + const existing = messages.findByIdInThread(threadId, userMessage.messageId); + if (!existing || existing.role !== "user") { + throw new Error(`Queued user message not found: ${userMessage.messageId}`); + } + return existing; + } + return messages.create( + threadId, + "user", + userMessage.content, + userMessage.sequence, + userMessage.attachments, + userMessage.replyToMessageId, + userMessage.quotedText, + undefined, + undefined, + userMessage.mentions, + userMessage.previewAnnotations, + userMessage.origin, + userMessage.messageId, + userMessage.selectedTextComments, + ); +} + /** Values for an atomic start; no caller closure enters the writer. */ export interface DataOnlyParentTurnStartInput extends Omit { userMessage: ParentUserMessageWrite; @@ -118,7 +145,7 @@ export class CanonicalParentTurnWrite { if (input.reopenThread && !this.threads.reopen(input.thread.id)) { throw new Error(`Thread not found: ${input.thread.id}`); } - const message = this.projectUserMessage(input); + const message = projectParentUserMessage(this.messages, input.thread.id, input.userMessage); if (input.answeredPlanQuestionMessageId) { this.planAnswers.markAnswered(input.answeredPlanQuestionMessageId, input.thread.id); } @@ -250,33 +277,6 @@ export class CanonicalParentTurnWrite { }, onBatchWrite); } - private projectUserMessage(input: DataOnlyParentTurnStartInput) { - const { userMessage } = input; - if (userMessage.kind === "existing") { - const existing = this.messages.findByIdInThread(input.thread.id, userMessage.messageId); - if (!existing || existing.role !== "user") { - throw new Error(`Queued user message not found: ${userMessage.messageId}`); - } - return existing; - } - return this.messages.create( - input.thread.id, - "user", - userMessage.content, - userMessage.sequence, - userMessage.attachments, - userMessage.replyToMessageId, - userMessage.quotedText, - undefined, - undefined, - userMessage.mentions, - userMessage.previewAnnotations, - userMessage.origin, - userMessage.messageId, - userMessage.selectedTextComments, - ); - } - private loadStagedTerminalProjection(threadId: string, executionId: string): ParentTurnProjection { const id = deriveTurnAssistantMessageId(threadId, `execution:${executionId}`); const message = this.messages.findByIdInThreadIncludingInternal(threadId, id); diff --git a/apps/server/src/features/agents/turns/turn-admission-dispatch-coordinator.ts b/apps/server/src/features/agents/turns/turn-admission-dispatch-coordinator.ts index 68309a8f2..8558ddd02 100644 --- a/apps/server/src/features/agents/turns/turn-admission-dispatch-coordinator.ts +++ b/apps/server/src/features/agents/turns/turn-admission-dispatch-coordinator.ts @@ -36,6 +36,11 @@ import { } from "../../providers/availability/provider-availability-errors.js"; import { ThreadRepo } from "../../thread-control/persistence/thread-repo.js"; import { MessageRepo } from "../conversation/persistence/message-repo.js"; +import { + projectParentUserMessage, + type DataOnlyParentTurnStartInput, + type ParentUserMessageWrite, +} from "../canonical/canonical-parent-turn-write.js"; import { GoalLifecycleService, type GoalCommandEffectReceipt, @@ -136,6 +141,7 @@ export type ThreadControlLeaseDirective = export interface PreparedTurnDispatch { readonly kind: "dispatch"; readonly lease: TurnRuntimeLease; + readonly parentStartInput: DataOnlyParentTurnStartInput; readonly provider: IAgentProvider; readonly providerId: ProviderId; readonly request: TurnRequest; @@ -416,13 +422,14 @@ export class TurnAdmissionDispatchCoordinator { runtime.activate(lease); const attachmentData = await this.persistAttachments(prepared); const sourceTurnId = prepared.command.sourceTurnId ?? NodeCrypto.randomUUID(); - this.startParentTurn(prepared, lease, sourceTurnId, attachmentData, review); + const parentStartInput = this.startParentTurn(prepared, lease, sourceTurnId, attachmentData, review); this.publishCommittedEffects(prepared, sourceTurnId); const wirePayload = this.buildWirePayload(prepared); const request = await this.buildTurnRequest(prepared, lease, sourceTurnId, attachmentData, cwd, wirePayload, review); return { kind: "dispatch", lease, + parentStartInput, provider: prepared.provider, providerId: prepared.providerId, request, @@ -688,13 +695,47 @@ export class TurnAdmissionDispatchCoordinator { sourceTurnId: string, attachments: PersistedAttachmentData, review: ApprovalReviewDecision, - ): void { - const command = prepared.command; - const wasUserCompleted = prepared.thread.user_completed_at !== null; - const nextSequence = command.persistedUserMessage?.sequence - ?? this.messages.getLatestSequenceIncludingInternal(command.threadId) + 1; + ): DataOnlyParentTurnStartInput { + const input = this.prepareParentTurnStartInput(prepared, lease, sourceTurnId, attachments, review); + const { userMessage, reopenThread, answeredPlanQuestionMessageId, ...start } = input; let reopenedThread: Exclude, null> | null = null; this.parentTurns.startParentTurn({ + ...start, + projectUserMessage: () => { + if (reopenThread) reopenedThread = this.reopenThread(input.thread.id); + const message = projectParentUserMessage(this.messages, input.thread.id, userMessage); + if (answeredPlanQuestionMessageId) this.planAnswers.markAnswered(answeredPlanQuestionMessageId, input.thread.id); + return message; + }, + }); + if (reopenedThread) broadcast("thread.lifecycleChanged", { thread: reopenedThread }); + return input; + } + + private prepareParentTurnStartInput( + prepared: PreparedCommand, + lease: TurnRuntimeLease, + sourceTurnId: string, + attachments: PersistedAttachmentData, + review: ApprovalReviewDecision, + ): DataOnlyParentTurnStartInput { + const command = prepared.command; + const userMessage: ParentUserMessageWrite = command.persistedUserMessage + ? { kind: "existing", messageId: command.persistedUserMessage.id } + : { + kind: "create", + messageId: command.messageId ?? NodeCrypto.randomUUID(), + content: command.displayContent ?? command.content, + sequence: this.messages.getLatestSequenceIncludingInternal(command.threadId) + 1, + attachments: attachments.stored.length > 0 ? [...attachments.stored] : undefined, + replyToMessageId: command.replyToMessageId, + quotedText: command.quotedText, + mentions: prepared.mentions.length > 0 ? [...prepared.mentions] : undefined, + previewAnnotations: command.previewAnnotations, + origin: this.messageOrigin(command), + selectedTextComments: command.selectedTextComments, + }; + return { thread: { id: prepared.thread.id, workspaceId: prepared.workspace.id, @@ -708,14 +749,10 @@ export class TurnAdmissionDispatchCoordinator { approvalReviewReason: review.reason, providerIdentities: this.resumeIdentities(prepared), retryOfExecutionId: command.retryOfExecutionId, - projectUserMessage: () => { - if (wasUserCompleted) reopenedThread = this.reopenThread(command.threadId); - const message = this.persistUserMessage(prepared, nextSequence, attachments); - this.markPlanAnswer(command, prepared.thread.id); - return message; - }, - }); - if (reopenedThread) broadcast("thread.lifecycleChanged", { thread: reopenedThread }); + userMessage, + reopenThread: prepared.thread.user_completed_at !== null, + answeredPlanQuestionMessageId: command.markPlanAnswerForMessageId, + }; } private effectivePermissionMode(requested: PermissionMode | "default" | undefined, persisted: PermissionMode): "full" | "supervised" { @@ -739,19 +776,6 @@ export class TurnAdmissionDispatchCoordinator { return reopened; } - private persistUserMessage(prepared: PreparedCommand, sequence: number, attachments: PersistedAttachmentData) { - const command = prepared.command; - if (command.persistedUserMessage) return this.persistedQueuedMessage(command); - const origin = this.messageOrigin(command); - return this.createUserMessage(prepared, sequence, attachments, origin); - } - - private persistedQueuedMessage(command: SendMessageCommand) { - const message = this.messages.findByIdInThread(command.threadId, command.persistedUserMessage!.id); - if (!message) throw new Error(`Queued Turn message was not found: ${command.persistedUserMessage!.id}`); - return message; - } - private messageOrigin(command: SendMessageCommand) { if (!command.sourceThreadId || !command.originSourceTurnId || !command.sourceProviderId) return undefined; return { @@ -762,39 +786,6 @@ export class TurnAdmissionDispatchCoordinator { }; } - private createUserMessage( - prepared: PreparedCommand, - sequence: number, - attachments: PersistedAttachmentData, - origin: ReturnType, - ) { - const command = prepared.command; - const args = [ - command.threadId, - "user" as const, - command.displayContent ?? command.content, - sequence, - attachments.stored.length > 0 ? [...attachments.stored] : undefined, - command.replyToMessageId, - command.quotedText, - undefined, - undefined, - prepared.mentions.length > 0 ? [...prepared.mentions] : undefined, - command.previewAnnotations, - ] as const; - if (command.selectedTextComments !== undefined) { - return this.messages.create(...args, origin, command.messageId, command.selectedTextComments); - } - if (command.messageId === undefined && origin === undefined) return this.messages.create(...args); - if (command.messageId === undefined) return this.messages.create(...args, origin); - if (origin === undefined) return this.messages.create(...args, undefined, command.messageId); - return this.messages.create(...args, origin, command.messageId); - } - - private markPlanAnswer(command: SendMessageCommand, threadId: string): void { - if (command.markPlanAnswerForMessageId) this.planAnswers.markAnswered(command.markPlanAnswerForMessageId, threadId); - } - private publishCommittedEffects(prepared: PreparedCommand, sourceTurnId: string): void { this.publishPlanAnswer(prepared.command); const command = prepared.command; From 5d203cd0d4396b25cdf5e27be2f189117efb27e8 Mon Sep 17 00:00:00 2001 From: "(CJ) Chukwudi Nwobodo" <142016413+chuks-qua@users.noreply.github.com> Date: Thu, 24 Sep 2026 21:07:50 +0100 Subject: [PATCH 051/136] test(server): align admission fixtures with stable message IDs --- .../__tests__/agent-service-turn-cleanup.test.ts | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/apps/server/src/features/agents/orchestration/__tests__/agent-service-turn-cleanup.test.ts b/apps/server/src/features/agents/orchestration/__tests__/agent-service-turn-cleanup.test.ts index 2ccd1fdea..30de5b646 100644 --- a/apps/server/src/features/agents/orchestration/__tests__/agent-service-turn-cleanup.test.ts +++ b/apps/server/src/features/agents/orchestration/__tests__/agent-service-turn-cleanup.test.ts @@ -530,7 +530,7 @@ describe("AgentService turn cleanup", () => { it("marks a replayed queued plan answer after projecting its persisted user message", async () => { const { service, messageRepo, planQuestionAnswersRepo } = buildService(); - vi.mocked(messageRepo.findByIdInThread).mockReturnValue({ id: "queued-plan-answer", sequence: 1 } as never); + vi.mocked(messageRepo.findByIdInThread).mockReturnValue({ id: "queued-plan-answer", role: "user", sequence: 1 } as never); await service.dispatchQueuedAutomaticTurn({ threadId: THREAD_ID, @@ -584,6 +584,9 @@ describe("AgentService turn cleanup", () => { undefined, undefined, undefined, + undefined, + expect.any(String), + undefined, ); }); @@ -1022,6 +1025,9 @@ describe("AgentService turn cleanup", () => { undefined, undefined, bundle, + undefined, + expect.any(String), + undefined, ); expect((providerEmitter as any).sendTurn).toHaveBeenCalledWith( expect.objectContaining({ From 2fd668de1d0a38cdb09e1c5abb06a12981ba2236 Mon Sep 17 00:00:00 2001 From: "(CJ) Chukwudi Nwobodo" <142016413+chuks-qua@users.noreply.github.com> Date: Thu, 24 Sep 2026 21:07:25 +0100 Subject: [PATCH 052/136] feat(server): reconcile lost execution workers through writer --- ...anonical-execution-semantic-writer.test.ts | 67 ++++++++++++++++ .../canonical-execution-writer-port.test.ts | 24 ++++++ .../canonical-agent-writer-client.ts | 13 ++++ .../canonical-agent-writer-protocol.ts | 2 + .../canonical-agent-writer.worker.ts | 15 ++-- .../canonical-execution-semantic-writer.ts | 73 ++++++++++++++++++ .../canonical-execution-writer-port.ts | 8 ++ .../canonical/canonical-parent-turn-write.ts | 76 +++++++++++++++++++ .../execution/execution-worker-handler.ts | 1 + 9 files changed, 274 insertions(+), 5 deletions(-) diff --git a/apps/server/src/features/agents/canonical/__tests__/canonical-execution-semantic-writer.test.ts b/apps/server/src/features/agents/canonical/__tests__/canonical-execution-semantic-writer.test.ts index 117f5303a..e0cff03dd 100644 --- a/apps/server/src/features/agents/canonical/__tests__/canonical-execution-semantic-writer.test.ts +++ b/apps/server/src/features/agents/canonical/__tests__/canonical-execution-semantic-writer.test.ts @@ -9,6 +9,7 @@ import { openDatabase } from "../../../../runtime/persistence/sqlite/database.js import { MessageRepo } from "../../conversation/persistence/message-repo.js"; import type { ExecutionSemanticOperation } from "../../execution/execution-worker-handler.js"; import { ExecutionWorkerHandler } from "../../execution/execution-worker-handler.js"; +import { ParentAssistantTextCheckpointService } from "../../turns/parent-assistant-text-checkpoint-service.js"; import { CanonicalExecutionSemanticWriter } from "../canonical-execution-semantic-writer.js"; import type { DataOnlyParentTurnStartInput } from "../canonical-parent-turn-write.js"; @@ -112,6 +113,72 @@ describe("CanonicalExecutionSemanticWriter through ExecutionWorkerHandler", () = return (await handler.handle({ requestId: ordinal, execution, lease, ordinal, command })).result; } + const loss = { + execution, lease, + reason: "The execution worker exited before its provider turn could be proved live.", + recoveryIncidentId: "incident-1", + }; + + it("fences a lost worker while retaining durable text and narrative", async () => { + expect((await send(1, { kind: "start", providerId: "codex", input: startInput() })).kind).toBe("committed"); + const text = new ParentAssistantTextCheckpointService(db); + expect(text.appendChunk([{ ...execution, sequence: 1, text: "Partial answer" }]).outcome).toBe("committed"); + text.recoveryJournal.append([{ ...execution, sequence: 2, text: " from journal" }]); + const { CanonicalAgentBoundary } = await import("../canonical-agent-boundary.js"); + const canonical = new CanonicalAgentBoundary(db, () => {}); + expect(canonical.recordParentNarrativeRecovery({ + executionId: EXECUTION_ID, items: toolNarrative("", 1), + })).toBe(true); + + const receipt = writer.interruptWorkerLoss(loss); + expect(receipt).toMatchObject({ kind: "committed", operationId: "lease-1:worker-lost" }); + expect(canonical.loadTurn(TURN_ID)).toMatchObject({ status: "Interrupted" }); + expect(canonical.loadCheckpoint(EXECUTION_ID)).toMatchObject({ phase: "interrupted", terminalOutcome: "interrupted" }); + expect(new MessageRepo(db).listIncludingInternal(THREAD_ID)).toContainEqual(expect.objectContaining({ + role: "assistant", content: "Partial answer from journal", outcome: "interrupted", is_internal: false, + })); + expect(db.prepare("SELECT id, status, message_id FROM tool_call_records WHERE id = ?").get("tool-0")) + .toMatchObject({ id: "tool-0", status: "completed", + message_id: canonical.loadTerminalProjection(TURN_ID).message?.id }); + expect(published).toContain(`${EXECUTION_ID}:recovery-interrupted`); + expect(await writer.transact(operation(2, { kind: "append-events", phase: "running", nativeCursor: null, events: [event()] }))) + .toEqual({ kind: "conflict", operationId: "lease-1:2" }); + + db.close(true); + db = openDatabase({ dbPath: path }); + published = []; + writer = new CanonicalExecutionSemanticWriter(db, (events) => published.push(...events.map((item) => item.eventId))); + expect(writer.interruptWorkerLoss(loss)).toEqual(receipt); + expect(published).toContain(`${EXECUTION_ID}:recovery-interrupted`); + }); + + it("rejects stale worker-loss leases without changing the unfinished turn", async () => { + expect((await send(1, { kind: "start", providerId: "codex", input: startInput() })).kind).toBe("committed"); + expect(writer.interruptWorkerLoss({ ...loss, lease: { ...lease, ownerEpoch: 2 } })) + .toEqual({ kind: "conflict", operationId: "lease-1:worker-lost" }); + expect(db.prepare("SELECT terminal_outcome FROM canonical_agent_ingest_checkpoints WHERE execution_id = ?").get(EXECUTION_ID)) + .toEqual({ terminal_outcome: null }); + expect(published).not.toContain(`${EXECUTION_ID}:recovery-interrupted`); + }); + + it("rolls back worker-loss interruption and publication when its receipt cannot commit", async () => { + expect((await send(1, { kind: "start", providerId: "codex", input: startInput() })).kind).toBe("committed"); + const text = new ParentAssistantTextCheckpointService(db); + text.appendChunk([{ ...execution, sequence: 1, text: "Keep me" }]); + text.recoveryJournal.append([{ ...execution, sequence: 2, text: " through failure" }]); + db.run("CREATE TRIGGER fail_loss_receipt BEFORE INSERT ON canonical_writer_operation_receipts WHEN NEW.kind = 'semantic:worker-lost' BEGIN SELECT RAISE(ABORT, 'loss receipt unavailable'); END"); + expect(() => writer.interruptWorkerLoss(loss)).toThrow("loss receipt unavailable"); + expect(db.prepare("SELECT terminal_outcome FROM canonical_agent_ingest_checkpoints WHERE execution_id = ?").get(EXECUTION_ID)) + .toEqual({ terminal_outcome: null }); + expect(new MessageRepo(db).listIncludingInternal(THREAD_ID).some((message) => message.role === "assistant")).toBe(false); + expect(text.restore(EXECUTION_ID)).toBe("Keep me through failure"); + expect(published).not.toContain(`${EXECUTION_ID}:recovery-interrupted`); + db.run("DROP TRIGGER fail_loss_receipt"); + expect(writer.interruptWorkerLoss(loss).kind).toBe("committed"); + expect(db.prepare("SELECT content FROM messages WHERE role = 'assistant'").get()) + .toEqual({ content: "Keep me through failure" }); + }); + it("commits start, event, and finalization with durable receipts across a database reload", async () => { const begin = operation(1, { kind: "begin", providerId: "codex", input: startInput() }); expect((await send(1, { kind: "start", providerId: "codex", input: startInput() })).kind).toBe("committed"); diff --git a/apps/server/src/features/agents/canonical/__tests__/canonical-execution-writer-port.test.ts b/apps/server/src/features/agents/canonical/__tests__/canonical-execution-writer-port.test.ts index a21beeef8..ae69f4660 100644 --- a/apps/server/src/features/agents/canonical/__tests__/canonical-execution-writer-port.test.ts +++ b/apps/server/src/features/agents/canonical/__tests__/canonical-execution-writer-port.test.ts @@ -93,6 +93,30 @@ describe("execution semantic writer transport", () => { expect(await port.transact(beginOperation())).toMatchObject({ kind: "committed" }); }); + it("commits and replays worker-loss interruption through the SQLite writer worker", async () => { + writer = new CanonicalAgentWriterClient(NodePath.join(directory, "app.sqlite")); + const published: string[] = []; + const port = new CanonicalExecutionWriterPort(writer, (events) => { + published.push(...events.map((event) => event.eventId)); + }); + expect(await port.transact(beginOperation())).toMatchObject({ kind: "committed" }); + const input = { execution, lease, reason: "Execution worker exited", recoveryIncidentId: "incident-1" }; + expect(await port.interruptWorkerLoss({ ...input, lease: { ...lease, ownerEpoch: 2 } })) + .toEqual({ kind: "conflict", operationId: `${lease.leaseId}:worker-lost` }); + const first = await port.interruptWorkerLoss(input); + expect(first).toMatchObject({ kind: "committed", operationId: `${lease.leaseId}:worker-lost` }); + expect(db.prepare("SELECT terminal_outcome FROM canonical_agent_ingest_checkpoints WHERE execution_id = ?") + .get(EXECUTION_ID)).toEqual({ terminal_outcome: "interrupted" }); + expect(published).toContain(`${EXECUTION_ID}:recovery-interrupted`); + published.length = 0; + expect(await port.interruptWorkerLoss(input)).toEqual(first); + expect(published).toContain(`${EXECUTION_ID}:recovery-interrupted`); + expect(await port.transact({ + operationId: `${lease.leaseId}:2`, execution, lease, ordinal: 2, + mutation: { kind: "checkpoint", phase: "running", nativeCursor: null }, + })).toEqual({ kind: "conflict", operationId: `${lease.leaseId}:2` }); + }); + it("replays a durable semantic receipt after the writer loses its reply", async () => { let dropReply = true; const createWorker = (): Worker => { diff --git a/apps/server/src/features/agents/canonical/canonical-agent-writer-client.ts b/apps/server/src/features/agents/canonical/canonical-agent-writer-client.ts index b1ac13d87..d3dc3d31a 100644 --- a/apps/server/src/features/agents/canonical/canonical-agent-writer-client.ts +++ b/apps/server/src/features/agents/canonical/canonical-agent-writer-client.ts @@ -11,6 +11,7 @@ import type { } from "./canonical-agent-writer-protocol.js"; import type { ParentNarrativeRecoveryCommitInput } from "./canonical-agent-boundary.js"; import type { ExecutionSemanticOperation } from "../execution/execution-worker-handler.js"; +import type { LostExecutionInterruption } from "./canonical-execution-semantic-writer.js"; const MAX_PENDING_WRITES = 64; const MAX_WORKER_ATTEMPTS = 3; @@ -81,6 +82,18 @@ export class CanonicalAgentWriterClient { return response.result; } + /** Reconcile a lost execution on the writer, then return committed envelopes for publication. */ + async interruptWorkerLoss(input: LostExecutionInterruption): Promise { + const operationId = `${input.lease.leaseId}:worker-lost`; + await this.retryPendingAcknowledgements(MAX_ACK_RETRIES_BEFORE_WRITE); + const response = await this.sendWithRetry({ + kind: "semantic-worker-loss", requestId: NodeCrypto.randomUUID(), operationId, + executionId: input.execution.executionId, input, + }); + if (response.kind !== "semantic-transacted") throw new Error("Canonical writer returned an unexpected response"); + return response.result; + } + /** Resolves after recovery writes finish; a lost reply replays the durable receipt. */ async recordParentNarrativeRecovery( operationId: string, diff --git a/apps/server/src/features/agents/canonical/canonical-agent-writer-protocol.ts b/apps/server/src/features/agents/canonical/canonical-agent-writer-protocol.ts index a53986271..3d45be1e4 100644 --- a/apps/server/src/features/agents/canonical/canonical-agent-writer-protocol.ts +++ b/apps/server/src/features/agents/canonical/canonical-agent-writer-protocol.ts @@ -1,5 +1,6 @@ import type { CanonicalAgentEventEnvelope } from "@mcode/contracts"; import type { ExecutionSemanticOperation, ExecutionWriteReceipt } from "../execution/execution-worker-handler.js"; +import type { LostExecutionInterruption } from "./canonical-execution-semantic-writer.js"; import type { CanonicalAgentCommitInput, CanonicalAgentCommitResult, @@ -50,6 +51,7 @@ export type CanonicalWriterRequest = | (Correlation & { kind: "open"; dbPath: string }) | (Correlation & { kind: "commit"; input: CanonicalProviderWriteInput }) | (Correlation & { kind: "semantic-transact"; operation: ExecutionSemanticOperation }) + | (Correlation & { kind: "semantic-worker-loss"; input: LostExecutionInterruption }) | (Correlation & { kind: "record-parent-narrative-recovery"; input: ParentNarrativeRecoveryCommitInput }) | (Correlation & { kind: "classify-parent-narrative-recovery"; input: ParentNarrativeRecoveryCommitInput }) | (Correlation & { kind: "ack-operation" }) diff --git a/apps/server/src/features/agents/canonical/canonical-agent-writer.worker.ts b/apps/server/src/features/agents/canonical/canonical-agent-writer.worker.ts index 55c0f3f9f..c6c1c82ad 100644 --- a/apps/server/src/features/agents/canonical/canonical-agent-writer.worker.ts +++ b/apps/server/src/features/agents/canonical/canonical-agent-writer.worker.ts @@ -116,24 +116,29 @@ async function handle(request: CanonicalWriterRequest): Promise, + request: Extract, correlation: Pick, ): Promise { try { if (!semanticWriter || semanticPublication) throw new Error("Semantic writer is not ready"); - if (request.operation.operationId !== request.operationId - || request.operation.execution.executionId !== request.executionId) { + if (request.kind === "semantic-transact" + ? request.operation.operationId !== request.operationId + || request.operation.execution.executionId !== request.executionId + : `${request.input.lease.leaseId}:worker-lost` !== request.operationId + || request.input.execution.executionId !== request.executionId) { throw new Error("Semantic writer request identity mismatch"); } semanticPublication = []; - const receipt = await semanticWriter.transact(request.operation); + const receipt = request.kind === "semantic-transact" + ? await semanticWriter.transact(request.operation) + : semanticWriter.interruptWorkerLoss(request.input); const events = semanticPublication; return { ...correlation, kind: "semantic-transacted", result: { receipt, events } }; } catch { diff --git a/apps/server/src/features/agents/canonical/canonical-execution-semantic-writer.ts b/apps/server/src/features/agents/canonical/canonical-execution-semantic-writer.ts index c8460c7c7..9f47f0acf 100644 --- a/apps/server/src/features/agents/canonical/canonical-execution-semantic-writer.ts +++ b/apps/server/src/features/agents/canonical/canonical-execution-semantic-writer.ts @@ -74,6 +74,14 @@ type StoredOperation = z.infer; class SemanticConflict extends Error {} +/** The revoked ownership and recovery incident for a worker that cannot prove its live turn. */ +export interface LostExecutionInterruption { + readonly execution: ExecutionIdentity; + readonly lease: ExecutionLease; + readonly reason: string; + readonly recoveryIncidentId: string; +} + /** Adapts the supported execution mutations to one writer-local canonical SQLite connection. */ export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter { private readonly turns: CanonicalParentTurnWrite; @@ -116,6 +124,50 @@ export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter } } + /** Fence a lost worker and apply the existing interrupted-turn recovery contract. */ + interruptWorkerLoss(input: LostExecutionInterruption): ExecutionWriteReceipt { + const operation = workerLossOperation(input); + if (!validLostExecutionInput(input, operation)) return conflict(operation); + const hash = fingerprint(input); + const existing = this.loadOperation(input.execution.executionId, operation.operationId); + if (existing) { + const receipt = this.replay(operation, hash, existing); + if (receipt.kind === "committed") this.turns.retireInterruptedText(input.execution.executionId); + return receipt; + } + const head = this.loadHead(input.execution.executionId); + if (!head || head.terminal || !sameExecutionAndLease(head, input)) return conflict(operation); + this.turns.importRecoveryJournals(); + try { + const receipt = this.withBufferedPublication(() => this.db.transaction( + () => this.writeLostInterruption(input, operation, hash), + )()); + this.turns.retireInterruptedText(input.execution.executionId); + return receipt; + } catch (error) { + if (error instanceof SemanticConflict) return conflict(operation); + throw error; + } + } + + private writeLostInterruption( + input: LostExecutionInterruption, + operation: ExecutionSemanticOperation, + hash: string, + ): ExecutionWriteReceipt { + const current = this.loadHead(input.execution.executionId); + if (!current || current.terminal || !sameExecutionAndLease(current, input)) throw new SemanticConflict(); + const result = this.turns.interruptLostExecution({ ...input.execution, + reason: input.reason, recoveryIncidentId: input.recoveryIncidentId }); + const receipt = committed(operation, result.durableThrough); + const sequences = this.bufferedPublication?.flatMap((batch) => batch.map((event) => event.acceptedSequence)) ?? []; + this.storePublicationChunks(input.execution.executionId, hash, sequences); + this.storeHead({ ...current, ordinal: current.ordinal + 1, + durableRevision: receipt.durableRevision, terminal: true }); + this.storeReceipt(operation, hash, receipt); + return receipt; + } + private async applySupported(operation: ExecutionSemanticOperation, hash: string): Promise { switch (operation.mutation.kind) { case "begin": return this.begin(operation, hash); @@ -450,6 +502,27 @@ function nextHead(head: SemanticHead, operation: ExecutionSemanticOperation): bo && lease.workerGeneration === candidate.workerGeneration && lease.leaseId === candidate.leaseId; } +function validLostExecutionInput(input: LostExecutionInterruption, operation: ExecutionSemanticOperation): boolean { + return validIdentity(input.execution) && validLease(input.lease) + && Boolean(input.reason && input.recoveryIncidentId) && operation.operationId.length <= 256; +} + +function sameExecutionAndLease(head: SemanticHead, input: LostExecutionInterruption): boolean { + return head.execution.threadId === input.execution.threadId + && head.execution.turnId === input.execution.turnId + && head.execution.executionId === input.execution.executionId + && head.lease.ownerEpoch === input.lease.ownerEpoch + && head.lease.workerIndex === input.lease.workerIndex + && head.lease.workerGeneration === input.lease.workerGeneration + && head.lease.leaseId === input.lease.leaseId; +} + +function workerLossOperation(input: LostExecutionInterruption): ExecutionSemanticOperation { + return { operationId: `${input.lease.leaseId}:worker-lost`, execution: input.execution, + lease: input.lease, ordinal: 0, + mutation: { kind: "worker-lost", reason: input.reason, recoveryIncidentId: input.recoveryIncidentId } }; +} + function committed( operation: ExecutionSemanticOperation, durableRevision: number, diff --git a/apps/server/src/features/agents/canonical/canonical-execution-writer-port.ts b/apps/server/src/features/agents/canonical/canonical-execution-writer-port.ts index 8003f92c0..1cf5a114e 100644 --- a/apps/server/src/features/agents/canonical/canonical-execution-writer-port.ts +++ b/apps/server/src/features/agents/canonical/canonical-execution-writer-port.ts @@ -5,6 +5,7 @@ import type { } from "../execution/execution-worker-handler.js"; import type { CanonicalAgentEventPublisher } from "./canonical-agent-boundary.js"; import { CanonicalAgentWriterClient } from "./canonical-agent-writer-client.js"; +import type { LostExecutionInterruption } from "./canonical-execution-semantic-writer.js"; /** Bridges an execution worker to the sole SQLite writer and publishes only committed events. */ export class CanonicalExecutionWriterPort implements ExecutionSemanticWriter { @@ -19,4 +20,11 @@ export class CanonicalExecutionWriterPort implements ExecutionSemanticWriter { if (receipt.kind === "committed" && events.length > 0) this.publish(events); return receipt; } + + /** Reconcile a lost worker through the sole writer and publish its committed interruption. */ + async interruptWorkerLoss(input: LostExecutionInterruption): Promise { + const { receipt, events } = await this.writer.interruptWorkerLoss(input); + if (receipt.kind === "committed" && events.length > 0) this.publish(events); + return receipt; + } } diff --git a/apps/server/src/features/agents/canonical/canonical-parent-turn-write.ts b/apps/server/src/features/agents/canonical/canonical-parent-turn-write.ts index 45d961ddc..b0b5723ab 100644 --- a/apps/server/src/features/agents/canonical/canonical-parent-turn-write.ts +++ b/apps/server/src/features/agents/canonical/canonical-parent-turn-write.ts @@ -14,6 +14,7 @@ import { HookExecutionRepo } from "../events/persistence/hook-execution-repo.js" import { PlanQuestionAnswersRepo } from "../planning/persistence/plan-question-answers-repo.js"; import { ToolCallRecordRepo } from "../tools/persistence/tool-call-record-repo.js"; import { deriveTurnAssistantMessageId } from "../turns/turn-assistant-message-id.js"; +import { ParentAssistantTextCheckpointService } from "../turns/parent-assistant-text-checkpoint-service.js"; import type { ParentTurnFinishInput, ParentTurnProjection, @@ -111,6 +112,15 @@ export interface StagedParentTerminalProjection { toolCallCount: number; } +/** Recovery data for an execution whose owning worker has been lost. */ +export interface LostParentExecutionInput { + threadId: string; + turnId: string; + executionId: string; + reason: string; + recoveryIncidentId: string; +} + /** Writer-local semantic operations with cloneable inputs and durable receipts. */ export class CanonicalParentTurnWrite { private readonly db: Database; @@ -120,6 +130,7 @@ export class CanonicalParentTurnWrite { private readonly threads: ThreadRepo; private readonly planAnswers: PlanQuestionAnswersRepo; private readonly stagedAssistant: ReturnType; + private readonly assistantTextCheckpoints: ParentAssistantTextCheckpointService; constructor(db: Database, publish: CanonicalAgentEventPublisher) { this.db = db; @@ -135,6 +146,61 @@ export class CanonicalParentTurnWrite { this.threads = new ThreadRepo(db); this.planAnswers = new PlanQuestionAnswersRepo(db); this.stagedAssistant = db.prepare("SELECT role, content FROM messages WHERE id = ? AND thread_id = ?"); + this.assistantTextCheckpoints = new ParentAssistantTextCheckpointService(db); + } + + /** Import fsynced text before the interruption transaction reads its durable prefix. */ + importRecoveryJournals(): void { + this.assistantTextCheckpoints.importRecoveryJournals(); + } + + /** Apply the existing interrupted-turn contract on the writer connection. Caller owns the transaction. */ + interruptLostExecution(input: LostParentExecutionInput): CanonicalAgentCommitResult { + const turn = this.canonical.loadTurnByExecution(input.executionId); + const checkpoint = this.canonical.loadCheckpoint(input.executionId); + if (!isUnfinishedLostTurn(turn, checkpoint, input)) { + throw new Error(`Unfinished parent execution not found: ${input.executionId}`); + } + const existingAssistant = this.canonical.loadTerminalProjection(input.turnId).message; + const recoveredNarrative = this.canonical.loadParentNarrativeRecovery(input.turnId); + const text = existingAssistant ? "" : this.assistantTextCheckpoints.restore(input.executionId); + const assistant = existingAssistant ?? this.stageRecoveredAssistant(input, text, recoveredNarrative.length); + this.canonical.markUnresolvedCodexChildDeliveriesUnknown(input.executionId); + const result = this.canonical.interruptUnfinishedExecution( + input.executionId, + input.reason, + assistant ?? undefined, + recoveredNarrative.length > 0 + ? (message, narrative) => this.narrative.persistRecoveredNarrative(message.id, narrative) + : undefined, + recoveredNarrative, + input.recoveryIncidentId, + ); + if (result.outcome !== "committed") throw new Error(`Parent interruption did not commit: ${input.executionId}`); + this.threads.updateStatus(input.threadId, "interrupted"); + return result; + } + + /** Retire provisional text only after the canonical interruption and receipt commit. */ + retireInterruptedText(executionId: string): void { + if (!this.assistantTextCheckpoints.retire(executionId)) { + throw new Error(`Interrupted assistant text checkpoint was not retired: ${executionId}`); + } + } + + private stageRecoveredAssistant(input: LostParentExecutionInput, text: string, narrativeCount: number) { + if (text.length === 0 && narrativeCount === 0) return null; + const id = deriveTurnAssistantMessageId(input.threadId, `recovery:${input.executionId}`); + const existing = this.messages.findByIdInThreadIncludingInternal(input.threadId, id); + if (existing) return existing; + return this.messages.createAssistantIdempotent({ + id, + threadId: input.threadId, + content: text, + sequence: this.messages.getLatestSequenceIncludingInternal(input.threadId) + 1, + model: this.threads.findById(input.threadId)?.model ?? null, + isInternal: true, + }); } /** Commit the user message, optional thread reopen and plan answer with canonical start. */ @@ -297,6 +363,16 @@ export class CanonicalParentTurnWrite { } } +function isUnfinishedLostTurn( + turn: ReturnType, + checkpoint: ReturnType, + input: LostParentExecutionInput, +): boolean { + return Boolean(turn && checkpoint && turn.id === input.turnId + && checkpoint.threadId === input.threadId && checkpoint.turnId === input.turnId + && checkpoint.terminalOutcome === null); +} + function settleTerminalNarrative( items: readonly ParentNarrativeRecoveryItem[], messageId: string, diff --git a/apps/server/src/features/agents/execution/execution-worker-handler.ts b/apps/server/src/features/agents/execution/execution-worker-handler.ts index 974efa317..63e799706 100644 --- a/apps/server/src/features/agents/execution/execution-worker-handler.ts +++ b/apps/server/src/features/agents/execution/execution-worker-handler.ts @@ -42,6 +42,7 @@ export interface ExecutionSemanticOperation { | { readonly kind: "effect-result"; readonly effectId: string; readonly settled: boolean } | { readonly kind: "provider-outcome"; readonly outcome: TurnOutcome } | { readonly kind: "stage-terminal"; readonly input: DataOnlyParentTerminalProjectionInput } + | { readonly kind: "worker-lost"; readonly reason: string; readonly recoveryIncidentId: string } | { readonly kind: "finish"; readonly outcome: TurnOutcome; readonly input: DataOnlyParentTurnFinishInput }; } From de4e55fbab14360126dbd49fbb0e0e8d4205b342 Mon Sep 17 00:00:00 2001 From: "(CJ) Chukwudi Nwobodo" <142016413+chuks-qua@users.noreply.github.com> Date: Thu, 24 Sep 2026 21:08:02 +0100 Subject: [PATCH 053/136] feat(server): project Codex collaboration on canonical writer --- .../canonical-execution-writer-port.test.ts | 88 +++++++++++++++++++ .../canonical-committed-provider-projector.ts | 39 ++++++++ .../canonical-execution-semantic-writer.ts | 79 +++++++++++++++-- .../execution/execution-worker-handler.ts | 15 +++- 4 files changed, 210 insertions(+), 11 deletions(-) create mode 100644 apps/server/src/features/agents/canonical/canonical-committed-provider-projector.ts diff --git a/apps/server/src/features/agents/canonical/__tests__/canonical-execution-writer-port.test.ts b/apps/server/src/features/agents/canonical/__tests__/canonical-execution-writer-port.test.ts index ae69f4660..e0cec6544 100644 --- a/apps/server/src/features/agents/canonical/__tests__/canonical-execution-writer-port.test.ts +++ b/apps/server/src/features/agents/canonical/__tests__/canonical-execution-writer-port.test.ts @@ -361,4 +361,92 @@ describe("execution semantic writer transport", () => { scheduler.shutdown(); } }); + it("projects Codex parent and child events on the writer and replays without repeating child writes", async () => { + writer = new CanonicalAgentWriterClient(NodePath.join(directory, "app.sqlite")); + const published: string[] = []; + const port = new CanonicalExecutionWriterPort(writer, (events) => { + published.push(...events.map((event) => event.eventId)); + }); + const scheduler = new ExecutionMailboxScheduler({ + workerCount: 1, + limits: { + maxPending: 16, maxPendingBytes: 64_000, reservedControl: 4, reservedControlBytes: 16_000, + maxPerExecutionPending: 12, maxPerExecutionBytes: 48_000, + reservedPerExecutionControl: 3, reservedPerExecutionControlBytes: 12_000, + }, + createWorker: () => new ExecutionThreadWorkerPort(port), + onWorkerLost: () => { throw new Error("Execution worker was lost"); }, + }); + try { + const claim = scheduler.claim(execution, 1); + if (claim.kind !== "claimed") throw new Error(`Execution claim failed: ${claim.kind}`); + const send = (command: Parameters[0]["command"]) => { + const admission = scheduler.submit({ execution, lease: claim.lease, command, byteLength: 1_000 }); + if (admission.kind !== "admitted") throw new Error(`Execution admission failed: ${admission.kind}`); + return admission.completion; + }; + const start = beginOperation().mutation; + if (start.kind !== "begin") throw new Error("Unexpected begin operation"); + await expect(send({ kind: "start", providerId: "codex", input: start.input })) + .resolves.toMatchObject({ kind: "reply", result: { kind: "committed" } }); + + const parent = runtimeDraft(1, { + type: AgentEventType.ToolUse, threadId: THREAD_ID, turnExecutionId: EXECUTION_ID, + toolCallId: "spawn-1", toolName: "Agent", toolInput: {}, + }, { + providerId: "codex", kind: "codex-collaboration", + collaboration: { kind: "spawnAgent", receiverThreadIds: ["native-child"], prompt: "Inspect the task" }, + }); + const parentCommand = { kind: "event" as const, phase: "running", nativeCursor: null, events: [parent] }; + await expect(send(parentCommand)).resolves.toMatchObject({ + kind: "reply", result: { kind: "committed", providerEvents: [{ + event: { type: AgentEventType.ToolUse, toolInput: {}, subagentPresentation: { + detail: { kind: "canonical-child" }, + } }, + }] }, + }); + const delegation = db.prepare("SELECT target_thread_id FROM canonical_collaboration_actions WHERE source_item_id = ?") + .get("toolCall:spawn-1"); + expect(delegation).toMatchObject({ target_thread_id: expect.any(String) }); + const parentOperation: ExecutionSemanticOperation = { + operationId: `${claim.lease.leaseId}:2`, execution, lease: claim.lease, ordinal: 2, + mutation: { kind: "append-events", phase: "running", nativeCursor: null, events: [parent] }, + }; + const replayedParent = await port.transact(parentOperation); + expect(replayedParent).toMatchObject({ kind: "committed", providerEvents: [{ + event: { toolInput: {}, subagentPresentation: { detail: { kind: "canonical-child" } } }, + }] }); + expect(replayedParent).toMatchObject(db.prepare("SELECT last_durable_sequence AS durableRevision FROM canonical_agent_ingest_checkpoints WHERE execution_id = ?") + .get(EXECUTION_ID)); + expect(db.prepare("SELECT COUNT(*) AS count FROM canonical_collaboration_actions WHERE source_item_id = ?") + .get("toolCall:spawn-1")).toEqual({ count: 1 }); + + const child = runtimeDraft(2, { + type: AgentEventType.TurnStarted, threadId: THREAD_ID, turnExecutionId: EXECUTION_ID, + }, { + providerId: "codex", kind: "codex-collaboration", + child: { nativeThreadId: "native-child", nativeTurnId: "native-turn", parentCollaborationItemId: "spawn-1" }, + }); + await expect(send({ kind: "event", phase: "running", nativeCursor: null, events: [child] })) + .resolves.toMatchObject({ kind: "reply", result: { kind: "committed", providerEvents: [] } }); + expect(db.prepare("SELECT COUNT(*) AS count FROM canonical_agent_turns WHERE thread_id = (SELECT target_thread_id FROM canonical_collaboration_actions WHERE source_item_id = ?)") + .get("toolCall:spawn-1")).toEqual({ count: 1 }); + const childEventsBefore = db.prepare("SELECT COUNT(*) AS count FROM canonical_agent_events WHERE thread_id = (SELECT target_thread_id FROM canonical_collaboration_actions WHERE source_item_id = ?)") + .get("toolCall:spawn-1"); + expect(childEventsBefore).toMatchObject({ count: expect.any(Number) }); + const publishedBeforeReplay = published.length; + expect(await port.transact({ + operationId: `${claim.lease.leaseId}:3`, execution, lease: claim.lease, ordinal: 3, + mutation: { kind: "append-events", phase: "running", nativeCursor: null, events: [child] }, + })).toMatchObject({ kind: "committed", providerEvents: [] }); + expect(db.prepare("SELECT COUNT(*) AS count FROM canonical_agent_events WHERE thread_id = (SELECT target_thread_id FROM canonical_collaboration_actions WHERE source_item_id = ?)") + .get("toolCall:spawn-1")).toEqual(childEventsBefore); + expect(published.length).toBeGreaterThan(publishedBeforeReplay); + expect(published).toContain(parent.eventId); + expect(published).toContain(child.eventId); + } finally { + scheduler.shutdown(); + } + }); + }); diff --git a/apps/server/src/features/agents/canonical/canonical-committed-provider-projector.ts b/apps/server/src/features/agents/canonical/canonical-committed-provider-projector.ts new file mode 100644 index 000000000..37a7cdae3 --- /dev/null +++ b/apps/server/src/features/agents/canonical/canonical-committed-provider-projector.ts @@ -0,0 +1,39 @@ +import type { CanonicalAgentEventEnvelope } from "@mcode/contracts"; + +import { CodexCollaborationEventAdapter } from "../collaboration/adapters/codex-collaboration-event-adapter.js"; +import type { CodexCollaborationDurability } from "../collaboration/codex-collaboration-durability.js"; +import type { ProjectedCommittedProviderEvent } from "../execution/execution-worker-handler.js"; +import { processProviderEventWorkerTask } from "../../providers/composition/provider-event-worker-protocol.js"; + +/** Interprets committed runtime envelopes beside the SQLite connection that owns their effects. */ +export class CanonicalCommittedProviderProjector { + private readonly codex: CodexCollaborationEventAdapter; + + constructor(durability: CodexCollaborationDurability) { + this.codex = new CodexCollaborationEventAdapter(durability); + } + + /** Return cloneable, provider-neutral deliveries after any Codex child writes have committed. */ + project(envelopes: readonly CanonicalAgentEventEnvelope[]): ProjectedCommittedProviderEvent[] { + const projected: ProjectedCommittedProviderEvent[] = []; + for (const envelope of envelopes) { + if (envelope.payload.type !== "item.recorded" + || envelope.payload.item.payload.projection !== "providerRuntimeEvent") continue; + const outcome = processProviderEventWorkerTask({ kind: "canonical-commit", envelope }); + if (outcome.status !== "accepted" || !outcome.event.canonicalReceipt) { + throw new Error(`Committed provider envelope cannot be interpreted: ${envelope.eventId}`); + } + const interpretation = outcome.event.providerId === "codex" + ? this.codex.project(outcome.event) + : { status: "forward" as const, event: outcome.event.event }; + if (interpretation.status !== "forward") continue; + projected.push({ + providerId: outcome.event.providerId, + sourceKind: "canonical-commit", + event: interpretation.event, + canonicalReceipt: outcome.event.canonicalReceipt, + }); + } + return projected; + } +} diff --git a/apps/server/src/features/agents/canonical/canonical-execution-semantic-writer.ts b/apps/server/src/features/agents/canonical/canonical-execution-semantic-writer.ts index 9f47f0acf..6d09c851c 100644 --- a/apps/server/src/features/agents/canonical/canonical-execution-semantic-writer.ts +++ b/apps/server/src/features/agents/canonical/canonical-execution-semantic-writer.ts @@ -1,16 +1,25 @@ import type { Database } from "bun:sqlite"; import * as NodeCrypto from "node:crypto"; -import { CanonicalAgentEventEnvelopeSchema, TurnOutcomeSchema, type TurnOutcome } from "@mcode/contracts"; +import { + AgentEventSchema, + CanonicalAgentEventEnvelopeSchema, + ProviderIdSchema, + TurnOutcomeSchema, + type TurnOutcome, +} from "@mcode/contracts"; import { z } from "zod"; import type { ExecutionIdentity, ExecutionLease } from "../execution/execution-mailbox-protocol.js"; import type { ExecutionProviderCommitReceipt, + ProjectedCommittedProviderEvent, ExecutionSemanticOperation, ExecutionSemanticWriter, ExecutionWriteReceipt, } from "../execution/execution-worker-handler.js"; import type { CanonicalAgentEventPublisher } from "./canonical-agent-boundary.js"; +import { CanonicalAgentBoundary } from "./canonical-agent-boundary.js"; +import { CanonicalCommittedProviderProjector } from "./canonical-committed-provider-projector.js"; import { CanonicalParentTurnWrite } from "./canonical-parent-turn-write.js"; const HEAD_ID = "semantic:head"; @@ -18,8 +27,24 @@ const HEAD_KIND = "semantic-head"; const PUBLICATION_KIND = "semantic-publication"; const PENDING_FINISH_KIND = "semantic:finish-pending"; const PUBLICATION_CHUNK_SIZE = 64; -const storedPublicationSequencesSchema = z.array(z.number().int().positive().max(Number.MAX_SAFE_INTEGER)) +const MAX_BUFFERED_PUBLICATION_EVENTS = 2_048; +const storedPublicationSequencesSchema = z.array(z.union([ + z.number().int().positive().max(Number.MAX_SAFE_INTEGER), + z.object({ executionId: z.string(), sequence: z.number().int().positive().max(Number.MAX_SAFE_INTEGER) }), +])) .min(1).max(PUBLICATION_CHUNK_SIZE); +const projectedProviderEventSchema = z.object({ + providerId: ProviderIdSchema, + sourceKind: z.literal("canonical-commit"), + event: AgentEventSchema(), + canonicalReceipt: z.object({ + eventId: z.string(), + sourceSequence: z.number().int().optional(), + acceptedSequence: z.number().int(), + durableRevision: z.number().int(), + serverTimestamps: z.object({ acceptedAt: z.string(), persistedAt: z.string().optional() }), + }), +}); const storedHeadSchema = z.object({ execution: z.object({ threadId: z.string(), turnId: z.string(), executionId: z.string() }), providerId: z.string(), @@ -49,6 +74,7 @@ const storedReceiptSchema = z.object({ durableThrough: z.number().int(), eventCount: z.number().int().nonnegative(), }).optional(), + providerEvents: z.array(projectedProviderEventSchema).optional(), }); const storedOperationSchema = z.object({ kind: z.string(), @@ -57,6 +83,7 @@ const storedOperationSchema = z.object({ }); const storedOperationListSchema = z.array(storedOperationSchema); const storedEnvelopeRowSchema = z.object({ envelope_json: z.string() }); +const durableSequenceRowSchema = z.object({ last_durable_sequence: z.number().int() }); interface SemanticHead { readonly execution: ExecutionIdentity; @@ -85,24 +112,33 @@ export interface LostExecutionInterruption { /** Adapts the supported execution mutations to one writer-local canonical SQLite connection. */ export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter { private readonly turns: CanonicalParentTurnWrite; + private readonly providerProjector: CanonicalCommittedProviderProjector; private readonly findOperation: ReturnType; private readonly listPublicationChunks: ReturnType; private readonly findPublishedEvent: ReturnType; + private readonly findDurableSequence: ReturnType; private readonly insertOperation: ReturnType; private readonly commitPendingFinish: ReturnType; private readonly updateHead: ReturnType; private readonly updateCheckpointPhase: ReturnType; private readonly updateCheckpoint: ReturnType; private bufferedPublication: Parameters[0][] | null = null; + private bufferedPublicationCount = 0; constructor(private readonly db: Database, private readonly publish: CanonicalAgentEventPublisher) { this.turns = new CanonicalParentTurnWrite(db, (events) => { - if (this.bufferedPublication) this.bufferedPublication.push(events); + if (this.bufferedPublication) this.bufferPublication(events); else this.publish(events); }); + const codexBoundary = new CanonicalAgentBoundary(db, (events) => { + if (!this.bufferedPublication) throw new Error("Codex projection requires a semantic transaction"); + this.bufferPublication(events); + }); + this.providerProjector = new CanonicalCommittedProviderProjector(codexBoundary); this.findOperation = db.prepare("SELECT kind, input_hash, receipt_json FROM canonical_writer_operation_receipts WHERE execution_id = ? AND operation_id = ?"); this.listPublicationChunks = db.prepare("SELECT kind, input_hash, receipt_json FROM canonical_writer_operation_receipts WHERE execution_id = ? AND operation_id >= ? AND operation_id < ? ORDER BY operation_id"); this.findPublishedEvent = db.prepare("SELECT envelope_json FROM canonical_agent_events WHERE execution_id = ? AND accepted_sequence = ?"); + this.findDurableSequence = db.prepare("SELECT last_durable_sequence FROM canonical_agent_ingest_checkpoints WHERE execution_id = ?"); this.insertOperation = db.prepare("INSERT INTO canonical_writer_operation_receipts (execution_id, operation_id, kind, input_hash, receipt_json) VALUES (?, ?, ?, ?, ?)"); this.commitPendingFinish = db.prepare("UPDATE canonical_writer_operation_receipts SET kind = ?, receipt_json = ? WHERE execution_id = ? AND operation_id = ? AND kind = ? AND input_hash = ?"); this.updateHead = db.prepare("UPDATE canonical_writer_operation_receipts SET receipt_json = ? WHERE execution_id = ? AND operation_id = ? AND kind = ?"); @@ -236,6 +272,8 @@ export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter events: mutation.events, }); if (result.outcome !== "committed") throw new SemanticConflict(); + const providerEvents = this.providerProjector.project(result.events); + const checkpoint = durableSequenceRowSchema.parse(this.findDurableSequence.get(operation.execution.executionId)); const providerCommit: ExecutionProviderCommitReceipt = { outcome: result.outcome, conversationRevision: result.conversationRevision, @@ -244,9 +282,14 @@ export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter durableThrough: result.durableThrough, eventCount: result.events.length, }; - const receipt = committed(operation, result.durableThrough, providerCommit); + const receipt = committed(operation, checkpoint.last_durable_sequence, providerCommit, providerEvents); this.storeHead({ ...head, ordinal: operation.ordinal, durableRevision: receipt.durableRevision }); - this.storePublicationChunks(operation.execution.executionId, hash, result.events.map((event) => event.acceptedSequence)); + const publication = this.bufferedPublication?.flat() ?? []; + // A Codex child write has its own execution sequence, even when the parent event caused it. + this.storePublicationChunks(operation.execution.executionId, hash, publication.map((event) => ({ + executionId: event.routing.executionId, + sequence: event.acceptedSequence, + }))); this.storeReceipt(operation, hash, receipt); return receipt; })()); @@ -383,12 +426,16 @@ export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter this.insertOperation.run(operation.execution.executionId, operation.operationId, PENDING_FINISH_KIND, hash, "{}"); } - private storePublicationChunks(executionId: string, hash: string, sequences: readonly number[]): void { + private storePublicationChunks( + executionId: string, + hash: string, + sequences: readonly (number | { executionId: string; sequence: number })[], + ): void { for (let offset = 0; offset < sequences.length; offset += PUBLICATION_CHUNK_SIZE) { const chunk = sequences.slice(offset, offset + PUBLICATION_CHUNK_SIZE); const first = chunk[0]; if (first === undefined) continue; - const id = publicationChunkId(hash, first); + const id = publicationChunkId(hash, typeof first === "number" ? first : offset + 1); const json = JSON.stringify(chunk); const existing = this.loadOperation(executionId, id); if (existing) { @@ -429,7 +476,7 @@ export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter if (stored.kind !== `semantic:${operation.mutation.kind}` || stored.input_hash !== hash) return conflict(operation); const receipt = storedReceiptSchema.parse(JSON.parse(stored.receipt_json)); return receipt.operationId === operation.operationId && Number.isSafeInteger(receipt.durableRevision) - ? committed(operation, receipt.durableRevision, receipt.providerCommit) : conflict(operation); + ? committed(operation, receipt.durableRevision, receipt.providerCommit, receipt.providerEvents) : conflict(operation); } private publishStoredEvents(executionId: string, hash: string): void { @@ -438,7 +485,9 @@ export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter for (const chunk of chunks) { if (chunk.kind !== PUBLICATION_KIND || chunk.input_hash !== hash) throw new Error("Semantic publication chunk mismatch"); const sequences = storedPublicationSequencesSchema.parse(JSON.parse(chunk.receipt_json)); - const events = sequences.map((sequence) => this.loadPublishedEvent(executionId, sequence)); + const events = sequences.map((sequence) => typeof sequence === "number" + ? this.loadPublishedEvent(executionId, sequence) + : this.loadPublishedEvent(sequence.executionId, sequence.sequence)); this.publish(events); } } @@ -450,10 +499,19 @@ export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter } // Only the synchronous begin and append transactions use this buffer. Finish publishes after each committed batch. + private bufferPublication(events: Parameters[0]): void { + if (!this.bufferedPublication || this.bufferedPublicationCount + events.length > MAX_BUFFERED_PUBLICATION_EVENTS) { + throw new Error("Semantic publication buffer exceeded its limit"); + } + this.bufferedPublication.push(events); + this.bufferedPublicationCount += events.length; + } + private withBufferedPublication(write: () => ExecutionWriteReceipt): ExecutionWriteReceipt { if (this.bufferedPublication) throw new Error("Nested semantic publication buffer"); const pending: Parameters[0][] = []; this.bufferedPublication = pending; + this.bufferedPublicationCount = 0; try { const result = write(); this.bufferedPublication = null; @@ -461,6 +519,7 @@ export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter return result; } finally { this.bufferedPublication = null; + this.bufferedPublicationCount = 0; } } } @@ -527,10 +586,12 @@ function committed( operation: ExecutionSemanticOperation, durableRevision: number, providerCommit?: ExecutionProviderCommitReceipt, + providerEvents?: readonly ProjectedCommittedProviderEvent[], ): Extract { return { kind: "committed", operationId: operation.operationId, durableRevision, ...(providerCommit ? { providerCommit } : {}), + ...(providerEvents ? { providerEvents } : {}), }; } diff --git a/apps/server/src/features/agents/execution/execution-worker-handler.ts b/apps/server/src/features/agents/execution/execution-worker-handler.ts index 63e799706..cda83c9ee 100644 --- a/apps/server/src/features/agents/execution/execution-worker-handler.ts +++ b/apps/server/src/features/agents/execution/execution-worker-handler.ts @@ -7,6 +7,7 @@ import type { DataOnlyParentTurnStartInput, } from "../canonical/canonical-parent-turn-write.js"; import type { CanonicalAgentCommitResult } from "../canonical/canonical-agent-boundary.js"; +import type { ProviderEventIngressEvent } from "../../providers/composition/provider-event-ingress.js"; import type { ExecutionIdentity, ExecutionLease, @@ -52,9 +53,18 @@ export type ExecutionProviderCommitReceipt = Pick< "outcome" | "conversationRevision" | "rosterRevision" | "acceptedThrough" | "durableThrough" > & { readonly eventCount: number }; +/** A committed runtime event after writer-local provider interpretation. */ +export type ProjectedCommittedProviderEvent = Omit< + ProviderEventIngressEvent, + "sourceKind" | "canonicalReceipt" | "runtimeExtension" +> & { + readonly sourceKind: "canonical-commit"; + readonly canonicalReceipt: NonNullable; +}; + /** A writer reply is valid only after the semantic operation commits durably. */ export type ExecutionWriteReceipt = - | { readonly kind: "committed"; readonly operationId: string; readonly durableRevision: number; readonly providerCommit?: ExecutionProviderCommitReceipt } + | { readonly kind: "committed"; readonly operationId: string; readonly durableRevision: number; readonly providerCommit?: ExecutionProviderCommitReceipt; readonly providerEvents?: readonly ProjectedCommittedProviderEvent[] } | { readonly kind: "conflict"; readonly operationId: string }; /** @@ -68,7 +78,7 @@ export interface ExecutionSemanticWriter { /** A command result that never calls an uncommitted mutation successful. */ export type ExecutionWorkerResult = - | { readonly kind: "committed"; readonly operationId: string; readonly durableRevision: number; readonly providerCommit?: ExecutionProviderCommitReceipt } + | { readonly kind: "committed"; readonly operationId: string; readonly durableRevision: number; readonly providerCommit?: ExecutionProviderCommitReceipt; readonly providerEvents?: readonly ProjectedCommittedProviderEvent[] } | { readonly kind: "released" } | { readonly kind: "rejected"; readonly reason: "no-execution" | "stale-execution" | "out-of-order" | "invalid-transition" | "invalid-event-routing" | "invalid-stop-watermark" | "writer-conflict" }; @@ -306,6 +316,7 @@ function committedResult(receipt: Extract Date: Thu, 24 Sep 2026 21:09:41 +0100 Subject: [PATCH 054/136] test(server): preserve both worker transport cases after merge --- .../canonical-execution-writer-port.test.ts | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/apps/server/src/features/agents/canonical/__tests__/canonical-execution-writer-port.test.ts b/apps/server/src/features/agents/canonical/__tests__/canonical-execution-writer-port.test.ts index e0cec6544..2e26fd9b7 100644 --- a/apps/server/src/features/agents/canonical/__tests__/canonical-execution-writer-port.test.ts +++ b/apps/server/src/features/agents/canonical/__tests__/canonical-execution-writer-port.test.ts @@ -3,6 +3,7 @@ import * as NodeFS from "node:fs"; import * as NodeOS from "node:os"; import * as NodePath from "node:path"; import type { Database } from "bun:sqlite"; +import { AgentEventType, type AgentEvent, type ProviderRuntimeExtension } from "@mcode/contracts"; import { afterEach, beforeEach, describe, expect, it } from "vitest"; import { openDatabase } from "../../../../runtime/persistence/sqlite/database.js"; @@ -36,6 +37,27 @@ function beginOperation(): ExecutionSemanticOperation { }; } +function runtimeDraft( + sequence: number, + event: AgentEvent, + extension?: ProviderRuntimeExtension, +): Extract["events"][number] { + const itemId = `runtime-item-${sequence}`; + return { + eventId: `${EXECUTION_ID}:runtime-${sequence}`, + routing: { ...execution, itemId }, + sourceProviderId: "codex", sourceIdentities: [], sourceSequence: sequence, + payload: { type: "item.recorded", item: { + id: itemId, threadId: THREAD_ID, turnId: TURN_ID, kind: "system", + providerIdentities: [], + payload: { projection: "providerRuntimeEvent", runtimeEvent: { + event, ...(extension ? { extension } : {}), + } }, + createdAt: NOW, updatedAt: NOW, + } }, + }; +} + describe("execution semantic writer transport", () => { let directory: string; let db: Database; @@ -361,6 +383,7 @@ describe("execution semantic writer transport", () => { scheduler.shutdown(); } }); + it("projects Codex parent and child events on the writer and replays without repeating child writes", async () => { writer = new CanonicalAgentWriterClient(NodePath.join(directory, "app.sqlite")); const published: string[] = []; From da153f7d1bda405f4c5fdf1c78740c4fa245b671 Mon Sep 17 00:00:00 2001 From: "(CJ) Chukwudi Nwobodo" <142016413+chuks-qua@users.noreply.github.com> Date: Thu, 24 Sep 2026 21:13:20 +0100 Subject: [PATCH 055/136] feat(server): stream bounded committed publication pages --- .../canonical-execution-writer-port.test.ts | 32 +++++++++ .../canonical-agent-writer-client.ts | 65 +++++++++++++++---- .../canonical-agent-writer-protocol.ts | 9 +-- .../canonical-agent-writer.worker.ts | 20 ++++-- .../canonical-execution-writer-port.ts | 8 +-- 5 files changed, 101 insertions(+), 33 deletions(-) diff --git a/apps/server/src/features/agents/canonical/__tests__/canonical-execution-writer-port.test.ts b/apps/server/src/features/agents/canonical/__tests__/canonical-execution-writer-port.test.ts index 2e26fd9b7..7c9636ec8 100644 --- a/apps/server/src/features/agents/canonical/__tests__/canonical-execution-writer-port.test.ts +++ b/apps/server/src/features/agents/canonical/__tests__/canonical-execution-writer-port.test.ts @@ -99,6 +99,38 @@ describe("execution semantic writer transport", () => { .toEqual({ count: 1 }); }); + it("delivers a large committed event batch in bounded publication pages", async () => { + writer = new CanonicalAgentWriterClient(NodePath.join(directory, "app.sqlite")); + const pageSizes: number[] = []; + const published: string[] = []; + const port = new CanonicalExecutionWriterPort(writer, (events) => { + pageSizes.push(events.length); + published.push(...events.map((event) => event.eventId)); + }); + expect(await port.transact(beginOperation())).toMatchObject({ kind: "committed" }); + pageSizes.length = 0; + published.length = 0; + const events = Array.from({ length: 130 }, (_, index) => runtimeDraft(index + 1, { + type: AgentEventType.TextDelta, + threadId: THREAD_ID, + turnExecutionId: EXECUTION_ID, + delta: "x", + isFinalResponse: true, + })); + const operation: ExecutionSemanticOperation = { + operationId: `${lease.leaseId}:2`, execution, lease, ordinal: 2, + mutation: { kind: "append-events", phase: "running", nativeCursor: null, events }, + }; + expect(await port.transact(operation)).toMatchObject({ kind: "committed" }); + expect(pageSizes).toEqual([64, 64, 2]); + expect(published).toEqual(events.map((event) => event.eventId)); + pageSizes.length = 0; + published.length = 0; + expect(await port.transact(operation)).toMatchObject({ kind: "committed" }); + expect(pageSizes).toEqual([64, 64, 2]); + expect(published).toEqual(events.map((event) => event.eventId)); + }); + it("does not acknowledge a failed database write and accepts a clean retry", async () => { writer = new CanonicalAgentWriterClient(NodePath.join(directory, "app.sqlite")); const port = new CanonicalExecutionWriterPort(writer, () => {}); diff --git a/apps/server/src/features/agents/canonical/canonical-agent-writer-client.ts b/apps/server/src/features/agents/canonical/canonical-agent-writer-client.ts index d3dc3d31a..609a6ed26 100644 --- a/apps/server/src/features/agents/canonical/canonical-agent-writer-client.ts +++ b/apps/server/src/features/agents/canonical/canonical-agent-writer-client.ts @@ -1,16 +1,16 @@ import * as NodePath from "node:path"; import * as NodeCrypto from "node:crypto"; +import type { CanonicalAgentEventEnvelope } from "@mcode/contracts"; import type { CanonicalParentNarrativeClassificationReceipt, CanonicalParentNarrativeRecoveryReceipt, CanonicalProviderWriteInput, CanonicalProviderWriteReceipt, - CanonicalSemanticWriteResult, CanonicalWriterRequest, CanonicalWriterResponse, } from "./canonical-agent-writer-protocol.js"; import type { ParentNarrativeRecoveryCommitInput } from "./canonical-agent-boundary.js"; -import type { ExecutionSemanticOperation } from "../execution/execution-worker-handler.js"; +import type { ExecutionSemanticOperation, ExecutionWriteReceipt } from "../execution/execution-worker-handler.js"; import type { LostExecutionInterruption } from "./canonical-execution-semantic-writer.js"; const MAX_PENDING_WRITES = 64; @@ -22,6 +22,7 @@ const WRITER_EXECUTION_ID = "writer:init"; interface PendingRequest { request: CanonicalWriterRequest; + onPublication?: (events: readonly CanonicalAgentEventEnvelope[]) => void; resolve(response: CanonicalWriterResponse): void; reject(error: Error): void; } @@ -69,29 +70,35 @@ export class CanonicalAgentWriterClient { return response.receipt; } - /** Commits one execution semantic operation on the writer worker and returns envelopes for publication. */ - async transactSemantic(operation: ExecutionSemanticOperation): Promise { + /** Commits one execution operation and delivers bounded pages after each durable write. */ + async transactSemantic( + operation: ExecutionSemanticOperation, + onPublication: (events: readonly CanonicalAgentEventEnvelope[]) => void, + ): Promise { if (!operation.operationId || !operation.execution.executionId) { throw new Error("Semantic writer operation and execution IDs are required"); } const response = await this.sendWithRetry({ kind: "semantic-transact", requestId: NodeCrypto.randomUUID(), operationId: operation.operationId, executionId: operation.execution.executionId, operation, - }); + }, false, onPublication); if (response.kind !== "semantic-transacted") throw new Error("Canonical writer returned an unexpected response"); - return response.result; + return response.receipt; } - /** Reconcile a lost execution on the writer, then return committed envelopes for publication. */ - async interruptWorkerLoss(input: LostExecutionInterruption): Promise { + /** Reconcile a lost execution and deliver bounded committed publication pages. */ + async interruptWorkerLoss( + input: LostExecutionInterruption, + onPublication: (events: readonly CanonicalAgentEventEnvelope[]) => void, + ): Promise { const operationId = `${input.lease.leaseId}:worker-lost`; await this.retryPendingAcknowledgements(MAX_ACK_RETRIES_BEFORE_WRITE); const response = await this.sendWithRetry({ kind: "semantic-worker-loss", requestId: NodeCrypto.randomUUID(), operationId, executionId: input.execution.executionId, input, - }); + }, false, onPublication); if (response.kind !== "semantic-transacted") throw new Error("Canonical writer returned an unexpected response"); - return response.result; + return response.receipt; } /** Resolves after recovery writes finish; a lost reply replays the durable receipt. */ @@ -215,13 +222,22 @@ export class CanonicalAgentWriterClient { private async sendWithRetry( request: CanonicalWriterRequest, allowDuringClose = false, + onPublication?: (events: readonly CanonicalAgentEventEnvelope[]) => void, ): Promise { + let publishedCount = 0; for (let attempt = 1; attempt <= MAX_WORKER_ATTEMPTS; attempt++) { this.assertCanSend(allowDuringClose); try { await this.workerReady; this.assertCanSend(allowDuringClose); - return await this.sendRaw(request); + let attemptCount = 0; + const deliver = onPublication ? (events: readonly CanonicalAgentEventEnvelope[]) => { + const skipped = Math.min(events.length, Math.max(0, publishedCount - attemptCount)); + if (skipped < events.length) onPublication(events.slice(skipped)); + attemptCount += events.length; + publishedCount = Math.max(publishedCount, attemptCount); + } : undefined; + return await this.sendRaw(request, deliver); } catch (error) { if (!(error instanceof CanonicalWriterWorkerLost) || attempt === MAX_WORKER_ATTEMPTS) throw error; await this.restartAfterLoss(allowDuringClose); @@ -271,14 +287,17 @@ export class CanonicalAgentWriterClient { if (response.kind !== "opened") throw new Error("Canonical writer did not open its database"); } - private sendRaw(request: CanonicalWriterRequest): Promise { + private sendRaw( + request: CanonicalWriterRequest, + onPublication?: (events: readonly CanonicalAgentEventEnvelope[]) => void, + ): Promise { const worker = this.worker; if (!worker) return Promise.reject(new CanonicalWriterWorkerLost("Canonical writer worker closed")); if (request.kind !== "open" && request.kind !== "close" && this.pending.size >= MAX_PENDING_WRITES) { return Promise.reject(new Error("Canonical writer admission is full")); } return new Promise((resolve, reject) => { - this.pending.set(request.requestId, { request, resolve, reject }); + this.pending.set(request.requestId, { request, onPublication, resolve, reject }); try { worker.postMessage(request); } catch { @@ -295,6 +314,10 @@ export class CanonicalAgentWriterClient { this.loseWorker(new Error("Canonical writer response identity mismatch")); return; } + if (response.kind === "semantic-publication") { + this.receivePublication(pending, response); + return; + } this.pending.delete(response.requestId); if (response.kind === "failed") { pending.reject(new Error(`Canonical writer ${response.reason}`)); @@ -303,6 +326,22 @@ export class CanonicalAgentWriterClient { pending.resolve(response); } + private receivePublication( + pending: PendingRequest, + response: Extract, + ): void { + if (!pending.onPublication || response.events.length === 0 || response.events.length > 64) { + this.loseWorker(new Error("Canonical writer publication page is invalid")); + return; + } + try { + pending.onPublication(response.events); + } catch (error) { + this.pending.delete(response.requestId); + pending.reject(error instanceof Error ? error : new Error(String(error))); + } + } + private loseWorker(error: Error, generation = this.generation): void { if (generation !== this.generation) return; const worker = this.worker; diff --git a/apps/server/src/features/agents/canonical/canonical-agent-writer-protocol.ts b/apps/server/src/features/agents/canonical/canonical-agent-writer-protocol.ts index 3d45be1e4..a97acef42 100644 --- a/apps/server/src/features/agents/canonical/canonical-agent-writer-protocol.ts +++ b/apps/server/src/features/agents/canonical/canonical-agent-writer-protocol.ts @@ -35,12 +35,6 @@ export interface CanonicalParentNarrativeClassificationReceipt { reset: true; } -/** A semantic transaction and the committed envelopes awaiting publication. */ -export interface CanonicalSemanticWriteResult { - receipt: ExecutionWriteReceipt; - events: readonly CanonicalAgentEventEnvelope[]; -} - interface Correlation { requestId: string; operationId: string; @@ -60,7 +54,8 @@ export type CanonicalWriterRequest = export type CanonicalWriterResponse = | (Correlation & { kind: "opened" }) | (Correlation & { kind: "committed"; receipt: CanonicalProviderWriteReceipt }) - | (Correlation & { kind: "semantic-transacted"; result: CanonicalSemanticWriteResult }) + | (Correlation & { kind: "semantic-publication"; events: readonly CanonicalAgentEventEnvelope[] }) + | (Correlation & { kind: "semantic-transacted"; receipt: ExecutionWriteReceipt }) | (Correlation & { kind: "parent-narrative-recovery-recorded"; receipt: CanonicalParentNarrativeRecoveryReceipt }) | (Correlation & { kind: "parent-narrative-recovery-classified"; receipt: CanonicalParentNarrativeClassificationReceipt }) | (Correlation & { kind: "operation-acknowledged" }) diff --git a/apps/server/src/features/agents/canonical/canonical-agent-writer.worker.ts b/apps/server/src/features/agents/canonical/canonical-agent-writer.worker.ts index c6c1c82ad..b99c1d4a5 100644 --- a/apps/server/src/features/agents/canonical/canonical-agent-writer.worker.ts +++ b/apps/server/src/features/agents/canonical/canonical-agent-writer.worker.ts @@ -2,7 +2,6 @@ import "reflect-metadata"; import { Database } from "bun:sqlite"; import * as NodeFS from "node:fs"; import * as NodePath from "node:path"; -import type { CanonicalAgentEventEnvelope } from "@mcode/contracts"; import { applySQLiteConnectionPolicy } from "../../../runtime/persistence/sqlite/sqlite-connection-policy.js"; import { CanonicalAgentBoundary } from "./canonical-agent-boundary.js"; import { CanonicalExecutionSemanticWriter } from "./canonical-execution-semantic-writer.js"; @@ -21,7 +20,8 @@ let boundary: CanonicalAgentBoundary | undefined; let assistantTextCheckpoints: ParentAssistantTextCheckpointService | undefined; let receipts: CanonicalAgentWriterReceipts | undefined; let semanticWriter: CanonicalExecutionSemanticWriter | undefined; -let semanticPublication: CanonicalAgentEventEnvelope[] | undefined; +const PUBLICATION_PAGE_SIZE = 64; +let semanticPublication: Pick | undefined; function openDatabase(dbPath: string): void { if (boundary || !NodePath.isAbsolute(dbPath) || !NodeFS.existsSync(dbPath)) { @@ -34,8 +34,15 @@ function openDatabase(dbPath: string): void { assistantTextCheckpoints = new ParentAssistantTextCheckpointService(connection); receipts = new CanonicalAgentWriterReceipts(connection); semanticWriter = new CanonicalExecutionSemanticWriter(connection, (events) => { - if (!semanticPublication) throw new Error("Semantic publication has no active request"); - semanticPublication.push(...events); + const correlation = semanticPublication; + if (!correlation) throw new Error("Semantic publication has no active request"); + for (let offset = 0; offset < events.length; offset += PUBLICATION_PAGE_SIZE) { + globalThis.postMessage({ + ...correlation, + kind: "semantic-publication", + events: events.slice(offset, offset + PUBLICATION_PAGE_SIZE), + } satisfies CanonicalWriterResponse); + } }); db = connection; } catch (error) { @@ -135,12 +142,11 @@ async function handleSemanticWrite( || request.input.execution.executionId !== request.executionId) { throw new Error("Semantic writer request identity mismatch"); } - semanticPublication = []; + semanticPublication = correlation; const receipt = request.kind === "semantic-transact" ? await semanticWriter.transact(request.operation) : semanticWriter.interruptWorkerLoss(request.input); - const events = semanticPublication; - return { ...correlation, kind: "semantic-transacted", result: { receipt, events } }; + return { ...correlation, kind: "semantic-transacted", receipt }; } catch { return { ...correlation, kind: "failed", reason: "write-failed" }; } finally { diff --git a/apps/server/src/features/agents/canonical/canonical-execution-writer-port.ts b/apps/server/src/features/agents/canonical/canonical-execution-writer-port.ts index 1cf5a114e..0c1b02c90 100644 --- a/apps/server/src/features/agents/canonical/canonical-execution-writer-port.ts +++ b/apps/server/src/features/agents/canonical/canonical-execution-writer-port.ts @@ -16,15 +16,11 @@ export class CanonicalExecutionWriterPort implements ExecutionSemanticWriter { /** Resolve after the durable writer receipt and its canonical event publication. */ async transact(operation: ExecutionSemanticOperation): Promise { - const { receipt, events } = await this.writer.transactSemantic(operation); - if (receipt.kind === "committed" && events.length > 0) this.publish(events); - return receipt; + return this.writer.transactSemantic(operation, this.publish); } /** Reconcile a lost worker through the sole writer and publish its committed interruption. */ async interruptWorkerLoss(input: LostExecutionInterruption): Promise { - const { receipt, events } = await this.writer.interruptWorkerLoss(input); - if (receipt.kind === "committed" && events.length > 0) this.publish(events); - return receipt; + return this.writer.interruptWorkerLoss(input, this.publish); } } From fc5f4907b61c7e7e1a97611230d88286c64676ba Mon Sep 17 00:00:00 2001 From: "(CJ) Chukwudi Nwobodo" <142016413+chuks-qua@users.noreply.github.com> Date: Thu, 24 Sep 2026 21:16:13 +0100 Subject: [PATCH 056/136] fix(server): fence worker replacement until turn recovery --- .../execution-mailbox-scheduler.test.ts | 7 ++++++ .../execution-thread-worker-port.test.ts | 4 +++- .../execution-worker-handler.test.ts | 4 +++- .../execution/execution-mailbox-scheduler.ts | 23 +++++++++++++------ 4 files changed, 29 insertions(+), 9 deletions(-) diff --git a/apps/server/src/features/agents/execution/__tests__/execution-mailbox-scheduler.test.ts b/apps/server/src/features/agents/execution/__tests__/execution-mailbox-scheduler.test.ts index afe453840..a11c4d109 100644 --- a/apps/server/src/features/agents/execution/__tests__/execution-mailbox-scheduler.test.ts +++ b/apps/server/src/features/agents/execution/__tests__/execution-mailbox-scheduler.test.ts @@ -268,6 +268,11 @@ describe("ExecutionMailboxScheduler", () => { expect(scheduler.submit({ execution: oldExecution, lease: oldLease, command: { kind: "stop", requestId: "stale" }, byteLength: 100 })) .toEqual({ kind: "stale-execution" }); + expect(scheduler.replaceWorker(0)).toBe(false); + expect(scheduler.claim(oldExecution, 2)).toEqual({ kind: "thread-busy" }); + expect(scheduler.reconcileLost(oldExecution, { ...oldLease, ownerEpoch: 2 })).toBe(false); + expect(scheduler.reconcileLost(oldExecution, oldLease)).toBe(true); + expect(scheduler.reconcileLost(oldExecution, oldLease)).toBe(false); expect(scheduler.replaceWorker(0)).toBe(true); expect(workers[1]?.requests).toHaveLength(0); @@ -337,6 +342,8 @@ describe("ExecutionMailboxScheduler", () => { workers[0]?.close(); expect(await event.completion).toEqual({ kind: "worker-lost" }); expect(lost).toEqual([[{ execution: identity, lease }]]); + expect(scheduler.depth()).toMatchObject({ pending: 0, activeExecutions: 1 }); + expect(scheduler.reconcileLost(identity, lease)).toBe(true); expect(scheduler.depth()).toMatchObject({ pending: 0, activeExecutions: 0 }); scheduler.shutdown(); }); diff --git a/apps/server/src/features/agents/execution/__tests__/execution-thread-worker-port.test.ts b/apps/server/src/features/agents/execution/__tests__/execution-thread-worker-port.test.ts index 18ebfb0f6..fb76f1d62 100644 --- a/apps/server/src/features/agents/execution/__tests__/execution-thread-worker-port.test.ts +++ b/apps/server/src/features/agents/execution/__tests__/execution-thread-worker-port.test.ts @@ -151,7 +151,9 @@ describe("ExecutionThreadWorkerPort", () => { await expect(submit(scheduler, lease, { kind: "start", providerId: "codex", input: START_INPUT })) .resolves.toEqual({ kind: "worker-lost" }); expect(lost).toEqual([[{ execution: EXECUTION, lease }]]); - expect(scheduler.depth()).toMatchObject({ pending: 0, activeExecutions: 0 }); + expect(scheduler.depth()).toMatchObject({ pending: 0, activeExecutions: 1 }); + expect(scheduler.claim(EXECUTION, 2)).toEqual({ kind: "thread-busy" }); + expect(scheduler.replaceWorker(0)).toBe(false); } finally { scheduler.shutdown(); } diff --git a/apps/server/src/features/agents/execution/__tests__/execution-worker-handler.test.ts b/apps/server/src/features/agents/execution/__tests__/execution-worker-handler.test.ts index 5aa2830a8..21fee6fb8 100644 --- a/apps/server/src/features/agents/execution/__tests__/execution-worker-handler.test.ts +++ b/apps/server/src/features/agents/execution/__tests__/execution-worker-handler.test.ts @@ -245,7 +245,9 @@ describe("ExecutionWorkerHandler through its scheduler", () => { .resolves.toEqual({ kind: "worker-lost" }); expect(lost).toEqual([[{ execution: EXECUTION, lease }]]); expect(worker.terminated).toBe(true); - expect(scheduler.depth()).toMatchObject({ pending: 0, activeExecutions: 0 }); + expect(scheduler.depth()).toMatchObject({ pending: 0, activeExecutions: 1 }); + expect(scheduler.claim(EXECUTION, 2)).toEqual({ kind: "thread-busy" }); + expect(scheduler.replaceWorker(0)).toBe(false); scheduler.shutdown(); }); }); diff --git a/apps/server/src/features/agents/execution/execution-mailbox-scheduler.ts b/apps/server/src/features/agents/execution/execution-mailbox-scheduler.ts index 715a1a112..e82ca56a9 100644 --- a/apps/server/src/features/agents/execution/execution-mailbox-scheduler.ts +++ b/apps/server/src/features/agents/execution/execution-mailbox-scheduler.ts @@ -38,6 +38,7 @@ interface Assignment { normalCount: number; normalBytes: number; stopping: boolean; + revoked: boolean; } interface Slot { @@ -160,6 +161,7 @@ export class ExecutionMailboxScheduler { if (this.stopped) return { kind: "shutdown" }; const assignment = this.currentAssignment(input.execution, input.lease); - if (!assignment) return { kind: "stale-execution" }; + if (!assignment || assignment.revoked) return { kind: "stale-execution" }; const decision = this.admissionDecision(assignment, input.command, input.byteLength); if (decision.kind !== "accept") return { kind: decision.kind }; const ordinal = assignment.nextOrdinal++; @@ -206,7 +208,16 @@ export class ExecutionMailboxScheduler assignment.slot === slot) - .map((assignment) => ({ execution: assignment.execution, lease: assignment.lease })); + const revokedAssignments = [...this.byThread.values()].filter((assignment) => assignment.slot === slot); + for (const assignment of revokedAssignments) assignment.revoked = true; + const revoked = revokedAssignments.map((assignment) => ({ execution: assignment.execution, lease: assignment.lease })); this.settleSlot(slot, "worker-lost"); - for (const assignment of revoked) this.byThread.delete(assignment.execution.threadId); - slot.activeCount = 0; this.onWorkerLost(revoked); } From bca7dbbebca0ea712865aa794a7e5f4449410ab6 Mon Sep 17 00:00:00 2001 From: "(CJ) Chukwudi Nwobodo" <142016413+chuks-qua@users.noreply.github.com> Date: Thu, 24 Sep 2026 21:19:24 +0100 Subject: [PATCH 057/136] feat(server): classify lost worker around durable turn start --- ...anonical-execution-semantic-writer.test.ts | 22 +++++++++++++++++++ .../canonical-execution-writer-port.test.ts | 10 ++++++++- .../canonical-execution-semantic-writer.ts | 6 +++-- .../execution/execution-worker-handler.ts | 2 +- 4 files changed, 36 insertions(+), 4 deletions(-) diff --git a/apps/server/src/features/agents/canonical/__tests__/canonical-execution-semantic-writer.test.ts b/apps/server/src/features/agents/canonical/__tests__/canonical-execution-semantic-writer.test.ts index e0cff03dd..f50011751 100644 --- a/apps/server/src/features/agents/canonical/__tests__/canonical-execution-semantic-writer.test.ts +++ b/apps/server/src/features/agents/canonical/__tests__/canonical-execution-semantic-writer.test.ts @@ -161,6 +161,28 @@ describe("CanonicalExecutionSemanticWriter through ExecutionWorkerHandler", () = expect(published).not.toContain(`${EXECUTION_ID}:recovery-interrupted`); }); + it("distinguishes a lost worker before start from one after a durable terminal", async () => { + expect(writer.interruptWorkerLoss(loss)).toEqual({ + kind: "conflict", operationId: "lease-1:worker-lost", recoveryState: "not-started", + }); + expect((await send(1, { kind: "start", providerId: "codex", input: startInput() })).kind).toBe("committed"); + expect((await send(2, { kind: "provider-outcome", outcome: "completed" })).kind).toBe("committed"); + expect((await send(3, { kind: "stage-terminal", input: { + threadId: THREAD_ID, executionId: EXECUTION_ID, outcome: "completed", endedAt: NOW, + assistant: { content: "Answer", model: null, attachments: [] }, narrative: [], + } })).kind).toBe("committed"); + expect((await send(4, { kind: "finalize", outcome: "completed", input: { + threadId: THREAD_ID, turnId: TURN_ID, executionId: EXECUTION_ID, + providerId: "codex", providerIdentities: [], outcome: "completed", + projection: { kind: "writer-staged" }, + } })).kind).toBe("committed"); + expect(writer.interruptWorkerLoss(loss)).toEqual({ + kind: "conflict", operationId: "lease-1:worker-lost", recoveryState: "already-terminal", + }); + expect(db.prepare("SELECT terminal_outcome FROM canonical_agent_ingest_checkpoints WHERE execution_id = ?") + .get(EXECUTION_ID)).toEqual({ terminal_outcome: "completed" }); + }); + it("rolls back worker-loss interruption and publication when its receipt cannot commit", async () => { expect((await send(1, { kind: "start", providerId: "codex", input: startInput() })).kind).toBe("committed"); const text = new ParentAssistantTextCheckpointService(db); diff --git a/apps/server/src/features/agents/canonical/__tests__/canonical-execution-writer-port.test.ts b/apps/server/src/features/agents/canonical/__tests__/canonical-execution-writer-port.test.ts index 7c9636ec8..c8dbd2180 100644 --- a/apps/server/src/features/agents/canonical/__tests__/canonical-execution-writer-port.test.ts +++ b/apps/server/src/features/agents/canonical/__tests__/canonical-execution-writer-port.test.ts @@ -153,8 +153,11 @@ describe("execution semantic writer transport", () => { const port = new CanonicalExecutionWriterPort(writer, (events) => { published.push(...events.map((event) => event.eventId)); }); - expect(await port.transact(beginOperation())).toMatchObject({ kind: "committed" }); const input = { execution, lease, reason: "Execution worker exited", recoveryIncidentId: "incident-1" }; + expect(await port.interruptWorkerLoss(input)).toEqual({ + kind: "conflict", operationId: `${lease.leaseId}:worker-lost`, recoveryState: "not-started", + }); + expect(await port.transact(beginOperation())).toMatchObject({ kind: "committed" }); expect(await port.interruptWorkerLoss({ ...input, lease: { ...lease, ownerEpoch: 2 } })) .toEqual({ kind: "conflict", operationId: `${lease.leaseId}:worker-lost` }); const first = await port.interruptWorkerLoss(input); @@ -370,6 +373,11 @@ describe("execution semantic writer transport", () => { expect(published.indexOf(`${EXECUTION_ID}:worker-item`)) .toBeLessThan(published.indexOf(`${EXECUTION_ID}:turn.completed`)); expect(published).toContain(`${EXECUTION_ID}:turn.completed`); + expect(await port.interruptWorkerLoss({ + execution, lease: claim.lease, reason: "Execution worker exited", recoveryIncidentId: "incident-complete", + })).toEqual({ + kind: "conflict", operationId: `${claim.lease.leaseId}:worker-lost`, recoveryState: "already-terminal", + }); } finally { scheduler.shutdown(); } diff --git a/apps/server/src/features/agents/canonical/canonical-execution-semantic-writer.ts b/apps/server/src/features/agents/canonical/canonical-execution-semantic-writer.ts index 6d09c851c..8c9e5a520 100644 --- a/apps/server/src/features/agents/canonical/canonical-execution-semantic-writer.ts +++ b/apps/server/src/features/agents/canonical/canonical-execution-semantic-writer.ts @@ -172,7 +172,9 @@ export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter return receipt; } const head = this.loadHead(input.execution.executionId); - if (!head || head.terminal || !sameExecutionAndLease(head, input)) return conflict(operation); + if (!head) return { ...conflict(operation), recoveryState: "not-started" }; + if (!sameExecutionAndLease(head, input)) return conflict(operation); + if (head.terminal) return { ...conflict(operation), recoveryState: "already-terminal" }; this.turns.importRecoveryJournals(); try { const receipt = this.withBufferedPublication(() => this.db.transaction( @@ -595,7 +597,7 @@ function committed( }; } -function conflict(operation: ExecutionSemanticOperation): ExecutionWriteReceipt { +function conflict(operation: ExecutionSemanticOperation): Extract { return { kind: "conflict", operationId: operation.operationId }; } diff --git a/apps/server/src/features/agents/execution/execution-worker-handler.ts b/apps/server/src/features/agents/execution/execution-worker-handler.ts index cda83c9ee..511c88bc7 100644 --- a/apps/server/src/features/agents/execution/execution-worker-handler.ts +++ b/apps/server/src/features/agents/execution/execution-worker-handler.ts @@ -65,7 +65,7 @@ export type ProjectedCommittedProviderEvent = Omit< /** A writer reply is valid only after the semantic operation commits durably. */ export type ExecutionWriteReceipt = | { readonly kind: "committed"; readonly operationId: string; readonly durableRevision: number; readonly providerCommit?: ExecutionProviderCommitReceipt; readonly providerEvents?: readonly ProjectedCommittedProviderEvent[] } - | { readonly kind: "conflict"; readonly operationId: string }; + | { readonly kind: "conflict"; readonly operationId: string; readonly recoveryState?: "not-started" | "already-terminal" }; /** * Implemented by one acknowledged writer, not by a worker-local SQLite connection. From 25ec8c25b88446a3f3440e2215e77aeb264f0ae5 Mon Sep 17 00:00:00 2001 From: "(CJ) Chukwudi Nwobodo" <142016413+chuks-qua@users.noreply.github.com> Date: Thu, 24 Sep 2026 21:19:46 +0100 Subject: [PATCH 058/136] refactor(server): share committed publication page limit --- .../agents/canonical/canonical-agent-writer-client.ts | 4 +++- .../agents/canonical/canonical-agent-writer-protocol.ts | 3 +++ .../agents/canonical/canonical-agent-writer.worker.ts | 6 +++--- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/apps/server/src/features/agents/canonical/canonical-agent-writer-client.ts b/apps/server/src/features/agents/canonical/canonical-agent-writer-client.ts index 609a6ed26..b90880793 100644 --- a/apps/server/src/features/agents/canonical/canonical-agent-writer-client.ts +++ b/apps/server/src/features/agents/canonical/canonical-agent-writer-client.ts @@ -9,6 +9,7 @@ import type { CanonicalWriterRequest, CanonicalWriterResponse, } from "./canonical-agent-writer-protocol.js"; +import { SEMANTIC_PUBLICATION_PAGE_SIZE } from "./canonical-agent-writer-protocol.js"; import type { ParentNarrativeRecoveryCommitInput } from "./canonical-agent-boundary.js"; import type { ExecutionSemanticOperation, ExecutionWriteReceipt } from "../execution/execution-worker-handler.js"; import type { LostExecutionInterruption } from "./canonical-execution-semantic-writer.js"; @@ -330,7 +331,8 @@ export class CanonicalAgentWriterClient { pending: PendingRequest, response: Extract, ): void { - if (!pending.onPublication || response.events.length === 0 || response.events.length > 64) { + if (!pending.onPublication || response.events.length === 0 + || response.events.length > SEMANTIC_PUBLICATION_PAGE_SIZE) { this.loseWorker(new Error("Canonical writer publication page is invalid")); return; } diff --git a/apps/server/src/features/agents/canonical/canonical-agent-writer-protocol.ts b/apps/server/src/features/agents/canonical/canonical-agent-writer-protocol.ts index a97acef42..81b5a09cb 100644 --- a/apps/server/src/features/agents/canonical/canonical-agent-writer-protocol.ts +++ b/apps/server/src/features/agents/canonical/canonical-agent-writer-protocol.ts @@ -8,6 +8,9 @@ import type { ParentNarrativeRecoveryCommitInput, } from "./canonical-agent-boundary.js"; +/** Upper bound for one committed semantic publication message. */ +export const SEMANTIC_PUBLICATION_PAGE_SIZE = 64; + /** Cloneable provider batch accepted by the SQLite writer. Compatibility callbacks stay with their owner. */ export type CanonicalProviderWriteInput = Pick< CanonicalAgentCommitInput, diff --git a/apps/server/src/features/agents/canonical/canonical-agent-writer.worker.ts b/apps/server/src/features/agents/canonical/canonical-agent-writer.worker.ts index b99c1d4a5..072716413 100644 --- a/apps/server/src/features/agents/canonical/canonical-agent-writer.worker.ts +++ b/apps/server/src/features/agents/canonical/canonical-agent-writer.worker.ts @@ -13,6 +13,7 @@ import type { CanonicalWriterRequest, CanonicalWriterResponse, } from "./canonical-agent-writer-protocol.js"; +import { SEMANTIC_PUBLICATION_PAGE_SIZE } from "./canonical-agent-writer-protocol.js"; import type { ParentNarrativeRecoveryCommitInput } from "./canonical-agent-boundary.js"; let db: Database | undefined; @@ -20,7 +21,6 @@ let boundary: CanonicalAgentBoundary | undefined; let assistantTextCheckpoints: ParentAssistantTextCheckpointService | undefined; let receipts: CanonicalAgentWriterReceipts | undefined; let semanticWriter: CanonicalExecutionSemanticWriter | undefined; -const PUBLICATION_PAGE_SIZE = 64; let semanticPublication: Pick | undefined; function openDatabase(dbPath: string): void { @@ -36,11 +36,11 @@ function openDatabase(dbPath: string): void { semanticWriter = new CanonicalExecutionSemanticWriter(connection, (events) => { const correlation = semanticPublication; if (!correlation) throw new Error("Semantic publication has no active request"); - for (let offset = 0; offset < events.length; offset += PUBLICATION_PAGE_SIZE) { + for (let offset = 0; offset < events.length; offset += SEMANTIC_PUBLICATION_PAGE_SIZE) { globalThis.postMessage({ ...correlation, kind: "semantic-publication", - events: events.slice(offset, offset + PUBLICATION_PAGE_SIZE), + events: events.slice(offset, offset + SEMANTIC_PUBLICATION_PAGE_SIZE), } satisfies CanonicalWriterResponse); } }); From 819b08cdca6295f4f33785419d9779276ce9a9fb Mon Sep 17 00:00:00 2001 From: "(CJ) Chukwudi Nwobodo" <142016413+chuks-qua@users.noreply.github.com> Date: Thu, 24 Sep 2026 21:20:40 +0100 Subject: [PATCH 059/136] fix(server): require writer evidence before releasing lost turn --- .../__tests__/execution-mailbox-scheduler.test.ts | 13 +++++++++---- .../execution/execution-mailbox-scheduler.ts | 15 +++++++++++++-- 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/apps/server/src/features/agents/execution/__tests__/execution-mailbox-scheduler.test.ts b/apps/server/src/features/agents/execution/__tests__/execution-mailbox-scheduler.test.ts index a11c4d109..7269d78ed 100644 --- a/apps/server/src/features/agents/execution/__tests__/execution-mailbox-scheduler.test.ts +++ b/apps/server/src/features/agents/execution/__tests__/execution-mailbox-scheduler.test.ts @@ -270,9 +270,12 @@ describe("ExecutionMailboxScheduler", () => { expect(scheduler.replaceWorker(0)).toBe(false); expect(scheduler.claim(oldExecution, 2)).toEqual({ kind: "thread-busy" }); - expect(scheduler.reconcileLost(oldExecution, { ...oldLease, ownerEpoch: 2 })).toBe(false); - expect(scheduler.reconcileLost(oldExecution, oldLease)).toBe(true); - expect(scheduler.reconcileLost(oldExecution, oldLease)).toBe(false); + const recovery = { kind: "conflict" as const, operationId: `${oldLease.leaseId}:worker-lost`, + recoveryState: "not-started" as const }; + expect(scheduler.reconcileLost(oldExecution, oldLease, { ...recovery, operationId: "wrong" })).toBe(false); + expect(scheduler.reconcileLost(oldExecution, { ...oldLease, ownerEpoch: 2 }, recovery)).toBe(false); + expect(scheduler.reconcileLost(oldExecution, oldLease, recovery)).toBe(true); + expect(scheduler.reconcileLost(oldExecution, oldLease, recovery)).toBe(false); expect(scheduler.replaceWorker(0)).toBe(true); expect(workers[1]?.requests).toHaveLength(0); @@ -343,7 +346,9 @@ describe("ExecutionMailboxScheduler", () => { expect(await event.completion).toEqual({ kind: "worker-lost" }); expect(lost).toEqual([[{ execution: identity, lease }]]); expect(scheduler.depth()).toMatchObject({ pending: 0, activeExecutions: 1 }); - expect(scheduler.reconcileLost(identity, lease)).toBe(true); + expect(scheduler.reconcileLost(identity, lease, { + kind: "conflict", operationId: `${lease.leaseId}:worker-lost`, recoveryState: "not-started", + })).toBe(true); expect(scheduler.depth()).toMatchObject({ pending: 0, activeExecutions: 0 }); scheduler.shutdown(); }); diff --git a/apps/server/src/features/agents/execution/execution-mailbox-scheduler.ts b/apps/server/src/features/agents/execution/execution-mailbox-scheduler.ts index e82ca56a9..b72ab6b2e 100644 --- a/apps/server/src/features/agents/execution/execution-mailbox-scheduler.ts +++ b/apps/server/src/features/agents/execution/execution-mailbox-scheduler.ts @@ -69,6 +69,11 @@ export interface ExecutionLostAssignment { readonly lease: ExecutionLease; } +/** Writer evidence that a lost worker's execution can release its thread and slot. */ +export type ExecutionRecoveryReceipt = + | { readonly kind: "committed"; readonly operationId: string; readonly durableRevision: number } + | { readonly kind: "conflict"; readonly operationId: string; readonly recoveryState: "not-started" | "already-terminal" }; + /** The host supplies durable owner epochs and a worker factory for each fixed slot. */ export interface ExecutionMailboxOptions { readonly workerCount: number; @@ -215,9 +220,10 @@ export class ExecutionMailboxScheduler(request: ExecutionWorkerRequest, reply: ExecutionWorkerReply): boolean { return request.requestId === reply.requestId && request.ordinal === reply.ordinal && sameIdentity(request.execution, reply.execution) && sameLease(request.lease, reply.lease); From fcd250c5f8e05a602454a99f24c3a4914b6f57e7 Mon Sep 17 00:00:00 2001 From: "(CJ) Chukwudi Nwobodo" <142016413+chuks-qua@users.noreply.github.com> Date: Thu, 24 Sep 2026 21:23:21 +0100 Subject: [PATCH 060/136] fix(server): page durable publication replay from SQLite --- ...anonical-execution-semantic-writer.test.ts | 26 +++++++++++++++++ .../canonical-execution-semantic-writer.ts | 29 ++++++++++++------- 2 files changed, 45 insertions(+), 10 deletions(-) diff --git a/apps/server/src/features/agents/canonical/__tests__/canonical-execution-semantic-writer.test.ts b/apps/server/src/features/agents/canonical/__tests__/canonical-execution-semantic-writer.test.ts index f50011751..f667b26b4 100644 --- a/apps/server/src/features/agents/canonical/__tests__/canonical-execution-semantic-writer.test.ts +++ b/apps/server/src/features/agents/canonical/__tests__/canonical-execution-semantic-writer.test.ts @@ -500,4 +500,30 @@ describe("CanonicalExecutionSemanticWriter through ExecutionWorkerHandler", () = expect(db.prepare("SELECT COUNT(*) AS count FROM canonical_writer_operation_receipts WHERE kind = 'semantic-head'").get()) .toEqual({ count: 2 }); }); + + it("replays terminal publication across multiple bounded receipt pages", async () => { + expect((await send(1, { kind: "start", providerId: "codex", input: startInput() })).kind).toBe("committed"); + const staged = new MessageRepo(db).create( + THREAD_ID, "assistant", "Answer", 2, undefined, undefined, undefined, "model", true, + ); + const input = { + threadId: THREAD_ID, turnId: TURN_ID, executionId: EXECUTION_ID, providerId: "codex", + providerIdentities: [], outcome: "completed" as const, + projection: { message: staged, narrative: toolNarrative(staged.id, 1_025) }, + }; + published = []; + expect((await send(2, { kind: "finalize", outcome: "completed", input })).kind).toBe("committed"); + const firstPublication = [...published]; + expect(firstPublication).toContain(`${EXECUTION_ID}:turn.completed`); + expect((db.prepare("SELECT COUNT(*) AS count FROM canonical_writer_operation_receipts WHERE execution_id = ? AND kind = 'semantic-publication'") + .get(EXECUTION_ID) as { count: number }).count).toBeGreaterThan(16); + + db.close(true); + db = openDatabase({ dbPath: path }); + published = []; + writer = new CanonicalExecutionSemanticWriter(db, (events) => published.push(...events.map((item) => item.eventId))); + expect(await writer.transact(operation(2, { kind: "finish", outcome: "completed", input }))) + .toMatchObject({ kind: "committed" }); + expect(published).toEqual(firstPublication); + }, 30_000); }); diff --git a/apps/server/src/features/agents/canonical/canonical-execution-semantic-writer.ts b/apps/server/src/features/agents/canonical/canonical-execution-semantic-writer.ts index 8c9e5a520..9463e23ed 100644 --- a/apps/server/src/features/agents/canonical/canonical-execution-semantic-writer.ts +++ b/apps/server/src/features/agents/canonical/canonical-execution-semantic-writer.ts @@ -27,6 +27,7 @@ const HEAD_KIND = "semantic-head"; const PUBLICATION_KIND = "semantic-publication"; const PENDING_FINISH_KIND = "semantic:finish-pending"; const PUBLICATION_CHUNK_SIZE = 64; +const PUBLICATION_CHUNK_PAGE_SIZE = 16; const MAX_BUFFERED_PUBLICATION_EVENTS = 2_048; const storedPublicationSequencesSchema = z.array(z.union([ z.number().int().positive().max(Number.MAX_SAFE_INTEGER), @@ -81,7 +82,8 @@ const storedOperationSchema = z.object({ input_hash: z.string(), receipt_json: z.string(), }); -const storedOperationListSchema = z.array(storedOperationSchema); +const storedPublicationChunkPageSchema = z.array(storedOperationSchema.extend({ operation_id: z.string() })) + .max(PUBLICATION_CHUNK_PAGE_SIZE); const storedEnvelopeRowSchema = z.object({ envelope_json: z.string() }); const durableSequenceRowSchema = z.object({ last_durable_sequence: z.number().int() }); @@ -136,7 +138,7 @@ export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter }); this.providerProjector = new CanonicalCommittedProviderProjector(codexBoundary); this.findOperation = db.prepare("SELECT kind, input_hash, receipt_json FROM canonical_writer_operation_receipts WHERE execution_id = ? AND operation_id = ?"); - this.listPublicationChunks = db.prepare("SELECT kind, input_hash, receipt_json FROM canonical_writer_operation_receipts WHERE execution_id = ? AND operation_id >= ? AND operation_id < ? ORDER BY operation_id"); + this.listPublicationChunks = db.prepare("SELECT operation_id, kind, input_hash, receipt_json FROM canonical_writer_operation_receipts WHERE execution_id = ? AND operation_id > ? AND operation_id < ? ORDER BY operation_id LIMIT ?"); this.findPublishedEvent = db.prepare("SELECT envelope_json FROM canonical_agent_events WHERE execution_id = ? AND accepted_sequence = ?"); this.findDurableSequence = db.prepare("SELECT last_durable_sequence FROM canonical_agent_ingest_checkpoints WHERE execution_id = ?"); this.insertOperation = db.prepare("INSERT INTO canonical_writer_operation_receipts (execution_id, operation_id, kind, input_hash, receipt_json) VALUES (?, ?, ?, ?, ?)"); @@ -483,14 +485,21 @@ export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter private publishStoredEvents(executionId: string, hash: string): void { const prefix = publicationChunkPrefix(hash); - const chunks = storedOperationListSchema.parse(this.listPublicationChunks.all(executionId, prefix, `${prefix}~`)); - for (const chunk of chunks) { - if (chunk.kind !== PUBLICATION_KIND || chunk.input_hash !== hash) throw new Error("Semantic publication chunk mismatch"); - const sequences = storedPublicationSequencesSchema.parse(JSON.parse(chunk.receipt_json)); - const events = sequences.map((sequence) => typeof sequence === "number" - ? this.loadPublishedEvent(executionId, sequence) - : this.loadPublishedEvent(sequence.executionId, sequence.sequence)); - this.publish(events); + let cursor = prefix; + while (true) { + const chunks = storedPublicationChunkPageSchema.parse(this.listPublicationChunks.all( + executionId, cursor, `${prefix}~`, PUBLICATION_CHUNK_PAGE_SIZE, + )); + if (chunks.length === 0) return; + for (const chunk of chunks) { + if (chunk.kind !== PUBLICATION_KIND || chunk.input_hash !== hash) throw new Error("Semantic publication chunk mismatch"); + const sequences = storedPublicationSequencesSchema.parse(JSON.parse(chunk.receipt_json)); + const events = sequences.map((sequence) => typeof sequence === "number" + ? this.loadPublishedEvent(executionId, sequence) + : this.loadPublishedEvent(sequence.executionId, sequence.sequence)); + this.publish(events); + } + cursor = chunks[chunks.length - 1]!.operation_id; } } From 81bc82968810d376234430cb46c2cb93898b4434 Mon Sep 17 00:00:00 2001 From: "(CJ) Chukwudi Nwobodo" <142016413+chuks-qua@users.noreply.github.com> Date: Thu, 24 Sep 2026 21:23:25 +0100 Subject: [PATCH 061/136] refactor(server): isolate narrative turn state from SQLite --- .../__tests__/narrative-store.test.ts | 68 + .../conversation/narrative/narrative-store.ts | 1164 +---------------- .../narrative/narrative-turn-state.ts | 1154 ++++++++++++++++ docs/internals/narrative-pipeline.md | 13 +- 4 files changed, 1255 insertions(+), 1144 deletions(-) create mode 100644 apps/server/src/features/agents/conversation/narrative/narrative-turn-state.ts diff --git a/apps/server/src/features/agents/conversation/narrative/__tests__/narrative-store.test.ts b/apps/server/src/features/agents/conversation/narrative/__tests__/narrative-store.test.ts index 1ce417e1d..7ecbac934 100644 --- a/apps/server/src/features/agents/conversation/narrative/__tests__/narrative-store.test.ts +++ b/apps/server/src/features/agents/conversation/narrative/__tests__/narrative-store.test.ts @@ -7,6 +7,7 @@ import { ToolCallRecordRepo } from "../../../tools/persistence/tool-call-record- import { ThoughtSegmentRepo } from "../persistence/thought-segment-repo.js"; import { HookExecutionRepo } from "../../../events/persistence/hook-execution-repo.js"; import { NarrativeStore } from "../narrative-store.js"; +import { NarrativeTurnState } from "../narrative-turn-state.js"; import { ACTIVE_TURN_WRITE_BATCH_LIMITS } from "../../../../../runtime/persistence/sqlite/bounded-write-batches.js"; import { PARENT_ASSISTANT_TEXT_RETAINED_LIMITS } from "../../../turns/parent-assistant-text-checkpoint-service.js"; import { @@ -638,6 +639,34 @@ describe("NarrativeStore write seam (server-side traps)", () => { return { toolCallId, toolName, toolInput: {}, parentToolCallId }; } + it("emits late Agent identity as data without a repository", () => { + const state = new NarrativeTurnState(); + state.beginTurn(THREAD); + state.resetTurnCounters(THREAD); + state.bufferToolCall(THREAD, { + toolCallId: "late-agent", + toolName: "Agent", + toolInput: { codexCollabKind: "spawnAgent", agentPath: "/root/worker" }, + }); + state.prepareNarrativePersistence(THREAD, "m1", "", "completed"); + state.bufferToolCall(THREAD, { + toolCallId: "late-agent", + toolName: "Agent", + toolInput: { + codexCollabKind: "spawnAgent", + agentPath: "/root/worker", + nativeThreadId: "native-late-agent", + }, + }); + + expect(state.takeEffects()).toEqual([{ + kind: "update-subagent-identity", + toolCallId: "late-agent", + messageId: "m1", + identityKey: encodeSubagentAliasDetailTarget("native-late-agent"), + }]); + }); + it("persists a long active turn in order across bounded transactions", async () => { seedAssistantMessage("m1", "done", 1); store.beginTurn(THREAD); @@ -676,6 +705,45 @@ describe("NarrativeStore write seam (server-side traps)", () => { expect(() => store.recoverySnapshot(THREAD)).toThrow("active-turn byte limit"); }); + it("keeps mixed live recovery and persisted narrative in the same order", () => { + seedAssistantMessage("m1", "Done", 1); + store.beginTurn(THREAD); + store.resetTurnCounters(THREAD); + store.openOrExtendThought(THREAD, "I will check."); + store.closeOpenThought(THREAD); + store.bufferToolCall(THREAD, toolUse("agent", "Agent")); + store.bufferToolCall(THREAD, toolUse("child", "Read", "agent")); + store.updateBufferedToolCallOutput(THREAD, "child", "Found it", false); + store.openHook(THREAD, { + hookName: "AfterTool", + toolName: "Read", + phase: "post-tool", + payload: "{}", + sortOrder: store.nextSortOrder(THREAD), + }); + + const live = store.recoverySnapshot(THREAD); + expect(live.map((item) => [item.kind, item.record.sort_order])).toEqual([ + ["narrationSegment", 0], + ["toolCall", 1], + ["toolCall", 2], + ["hook", 3], + ]); + expect(live.find((item) => item.kind === "toolCall" && item.record.id === "child")) + .toMatchObject({ record: { parent_tool_call_id: "agent", output_summary: "Found it" } }); + + store.persistNarrative(THREAD, "m1", "Done", "completed"); + + expect(store.load(THREAD) + .filter((entry) => entry.kind !== "assistantMessage") + .map((entry) => [entry.kind, entry.sortOrder])).toEqual([ + ["narrationSegment", 0], + ["toolCall", 1], + ["toolCall", 2], + ["hook", 3], + ]); + }); + it("rejects individually valid recovery records that exceed the retained byte budget together", () => { store.beginTurn(THREAD); store.resetTurnCounters(THREAD); diff --git a/apps/server/src/features/agents/conversation/narrative/narrative-store.ts b/apps/server/src/features/agents/conversation/narrative/narrative-store.ts index 99a1f1831..23effb8a3 100644 --- a/apps/server/src/features/agents/conversation/narrative/narrative-store.ts +++ b/apps/server/src/features/agents/conversation/narrative/narrative-store.ts @@ -10,11 +10,11 @@ * race two hydration streams (the old `message.list` + `narrative.list` pair) * and Tool calls never render before the assistant message body. * - * Write seam: this store owns the per-turn buffers (tool calls, the + * Write seam: NarrativeTurnState owns the per-turn buffers (tool calls, the * `agentCallStack`, the open/closed thought segments, hook executions, and the - * shared sort counter) and the enrichment + classification + persistence logic - * that AgentService used to inline. The six narrative-pipeline traps documented - * in `docs/internals/narrative-pipeline.md` are enforced here: + * shared sort counter). This store adds SQLite persistence and applies emitted + * data-only effects. The six narrative-pipeline traps documented in + * `docs/internals/narrative-pipeline.md` are enforced here: * * - Trap 1: {@link bufferToolCall} prefers the SDK `parent_tool_use_id` and only * falls back to {@link getCurrentParentToolCallId} when exactly one Agent on @@ -34,7 +34,6 @@ * the persisted rows verbatim and changes no counts. */ import { injectable, inject } from "tsyringe"; -import * as NodeCrypto from "node:crypto"; import type { Database } from "bun:sqlite"; import { drizzle, type BunSQLiteDatabase } from "drizzle-orm/bun-sqlite"; import { and, asc, desc, eq, gt, gte, lt, ne, sql, type SQL, type SQLWrapper } from "drizzle-orm"; @@ -47,20 +46,11 @@ import { import { logger } from "@mcode/shared"; import { NarrativeEntrySchema, - resolveBrowserNarrativeTool, - resolveSubagentDuration, - resolveSubagentMetadata, - resolveSubagentPrompt, - encodeCanonicalSubagentDetailTarget, - encodeSubagentAliasDetailTarget, - createSubagentPresentation, - mergeSubagentPresentation, type Message, type NarrativeEntry, type NarrativeDetailCursor, type ParentNarrativeRecoveryItem, type TurnRange, - type SubagentPresentation, } from "@mcode/contracts"; import { MessageRepo } from "../persistence/message-repo.js"; import { @@ -77,7 +67,17 @@ import { } from "../../events/persistence/hook-execution-repo.js"; import type { TurnOutcome } from "../../turns/turn-outcome.js"; import { ACTIVE_TURN_WRITE_BATCH_LIMITS } from "../../../../runtime/persistence/sqlite/bounded-write-batches.js"; -import { assertActiveTurnRecoveryRetention } from "../../turns/active-turn-recovery-retention-policy.js"; +import { + NarrativeTurnState, + type PreparedNarrativePersistence, + type NarrativeTurnStateEffect, +} from "./narrative-turn-state.js"; +export type { + BufferedToolCall, + BufferToolCallEvent, + OpenHook, + StagedNarrationSegment, +} from "./narrative-turn-state.js"; /** Default number of recent messages hydrated when no range is supplied. */ const DEFAULT_LOAD_LIMIT = 200; @@ -121,82 +121,12 @@ function compareNarrativeEntries(left: NarrativeEntry, right: NarrativeEntry): n || compareSqliteBinaryIds(narrativeEntryId(left), narrativeEntryId(right)); } -/** Buffered tool call with raw input preserved for deferred summarization. */ -export interface BufferedToolCall extends CreateToolCallRecordInput { - _rawToolInput?: Record; - _subagentPresentation?: SubagentPresentation; -} - -/** Extracts bounded provider metadata that must survive a persisted Agent card. */ -function persistedSubagentMetadata(input: Record) { - return { - subagentPrompt: resolveSubagentPrompt(input.prompt), - subagentType: resolveSubagentMetadata(input.subagentType), - subagentAgentId: resolveSubagentMetadata(input.agentId), - subagentDurationMs: resolveSubagentDuration(input.durationMs), - }; -} - -function persistedSubagentIdentityKey(presentation: SubagentPresentation | undefined): string | undefined { - if (!presentation) return undefined; - if (presentation.detail.kind === "canonical-child") { - return encodeCanonicalSubagentDetailTarget(presentation.detail.threadId); - } - return presentation.detail.kind === "canonical-alias" - ? encodeSubagentAliasDetailTarget(presentation.detail.identityKey) - : undefined; -} - -/** In-flight thought segment accumulated from consecutive textDelta events. */ -interface OpenThought { - id: string; - text: string; - startedAt: string; - sortOrder: number; -} - -/** In-flight hook execution awaiting its paired HookCompleted. */ -export interface OpenHook { - id: string; - hookName: string; - toolName: string | null; - phase: string; - payload: string; - startedAt: string; - sortOrder: number; -} - -/** One narration record staged for a durable ownership transfer. */ -export interface StagedNarrationSegment { - id: string; - text: string; - startedAt: string; - endedAt: string; - sortOrder: number; - openThoughtId?: string; -} - -/** Tool-use event shape consumed by {@link NarrativeStore.bufferToolCall}. */ -export interface BufferToolCallEvent { - toolCallId: string; - toolName: string; - toolInput: Record; - parentToolCallId?: string; - subagentPresentation?: SubagentPresentation; -} - /** Result of persisting a turn's narrative rows. */ export interface PersistNarrativeResult { /** Number of buffered tool calls written (drives the turn.persisted count). */ toolCallCount: number; } -interface PreparedNarrativePersistence { - toolCalls: BufferedToolCall[]; - thoughts: CreateThoughtSegmentInput[]; - hooks: CreateHookExecutionInput[]; -} - interface PendingNarrativePersistence extends PreparedNarrativePersistence {} interface PersistedNarrativeRows { @@ -207,48 +137,24 @@ interface PersistedNarrativeRows { type RecoveredToolCallItem = Extract; -/** Bounds persisted shell commands while retaining enough text for readable expansion. */ -const MAX_PERSISTED_COMMAND_CHARS = 4096; - -const NARRATIVE_INPUT_SUMMARIZERS: Record) => string> = { - read: fileInputSummary, - edit: fileInputSummary, - write: fileInputSummary, - move: renameInputSummary, - rename: renameInputSummary, - bash: commandInputSummary, - shell: commandInputSummary, - terminal: commandInputSummary, - command_execution: commandInputSummary, - grep: patternInputSummary, - glob: patternInputSummary, - agent: agentInputSummary, -}; - @injectable() -export class NarrativeStore { - /** Per-thread buffer of tool calls accumulated during the current turn. */ - private turnToolCalls = new Map(); - /** Stack of active Agent tool call IDs per thread (for nesting inference). */ - private agentCallStack = new Map(); - /** Per-thread sort counter shared across tool calls, thoughts, and hooks. */ - private turnSortCounters = new Map(); - /** In-flight thought segment being accumulated from textDelta events, per thread. */ - private turnOpenThought = new Map(); - /** Closed thought segments awaiting persistence at turn end, per thread. */ - private turnThoughts = new Map(); - /** In-flight hook executions keyed by hookName, per thread. */ - private turnOpenHooks = new Map>(); - /** Closed hook executions awaiting persistence at turn end, per thread. */ - private turnHooks = new Map(); - +export class NarrativeStore extends NarrativeTurnState { constructor( @inject(MessageRepo) _messageRepo: MessageRepo, @inject(ToolCallRecordRepo) private readonly toolCallRecordRepo: ToolCallRecordRepo, @inject(ThoughtSegmentRepo) private readonly thoughtSegmentRepo: ThoughtSegmentRepo, @inject(HookExecutionRepo) private readonly hookExecutionRepo: HookExecutionRepo, @inject("Database") private readonly db?: Database, - ) {} + ) { + super(); + this.setEffectSink((effect) => this.applyStateEffect(effect)); + } + + private applyStateEffect(effect: NarrativeTurnStateEffect): void { + this.toolCallRecordRepo.updateSubagentIdentity( + effect.toolCallId, effect.messageId, effect.identityKey, + ); + } private ormInstance: BunSQLiteDatabase | null = null; @@ -571,749 +477,6 @@ export class NarrativeStore { for (const hook of hooks) entries.push({ kind: "hook", sequence, sortOrder: hook.sort_order, record: hook }); } - // ---------------------------------------------------------------------- - // Write seam - // ---------------------------------------------------------------------- - - /** - * Reset the volatile per-turn buffers at the START of a turn (Trap 3). Mirrors - * the seeding AgentService used to do in its `sendMessage`/turnStarted prelude. - * Note: the sort counter and Agent stack are reset separately via - * {@link resetTurnCounters} on the TurnStarted event so late hooks from the - * prior turn can still increment the old counter. - */ - beginTurn(threadId: string): void { - this.turnToolCalls.set(threadId, []); - this.turnOpenThought.set(threadId, null); - this.turnThoughts.set(threadId, []); - this.turnOpenHooks.set(threadId, new Map()); - this.turnHooks.set(threadId, []); - } - - /** - * Reset the per-turn sort counter and Agent stack. Called from the - * TurnStarted handler rather than {@link beginTurn} so a fresh counter is - * available for each new turn while late hooks from the prior turn can still - * increment the old one (see {@link clearTurn}). - */ - resetTurnCounters(threadId: string): void { - this.turnSortCounters.set(threadId, 0); - this.agentCallStack.set(threadId, []); - } - - /** Allocate the next shared sort order for the thread's current turn. */ - nextSortOrder(threadId: string): number { - const sortOrder = this.turnSortCounters.get(threadId) ?? 0; - this.turnSortCounters.set(threadId, sortOrder + 1); - return sortOrder; - } - - /** - * Open or extend the in-flight thought segment from a non-final `textDelta`. - * The sort order is allocated lazily on the first delta so consecutive deltas - * keep the same slot, taken BEFORE any following tool call's slot — matching - * the live client builder. Never touches the `agentCallStack` (Trap 2). - */ - openOrExtendThought(threadId: string, delta: string): void { - const open = this.turnOpenThought.get(threadId); - if (!open) { - const sortOrder = this.nextSortOrder(threadId); - this.turnOpenThought.set(threadId, { - id: NodeCrypto.randomUUID(), - text: delta, - startedAt: new Date().toISOString(), - sortOrder, - }); - } else { - open.text += delta; - } - } - - /** - * Close any in-flight thought segment for the thread and push it onto the - * closed-thoughts list. Called before a tool call begins (so the thought - * sorts strictly before the tool) and during turn-end drain. - */ - closeOpenThought(threadId: string): void { - const open = this.turnOpenThought.get(threadId); - if (!open) return; - const list = this.turnThoughts.get(threadId) ?? []; - list.push({ - id: open.id, - messageId: "", - text: open.text, - startedAt: open.startedAt, - endedAt: new Date().toISOString(), - sortOrder: open.sortOrder, - }); - this.turnThoughts.set(threadId, list); - this.turnOpenThought.set(threadId, null); - } - - /** - * Discards the open thought without persisting it. - * - * Called when `AssistantMessageBoundary` reports `isFinalResponse: true` — - * the streamed text was actually the final assistant response and will be - * persisted via the `Message` event, so keeping the matching thought row - * would duplicate the body as a ThoughtBlock in the narrative. - */ - dropOpenThought(threadId: string): void { - this.turnOpenThought.set(threadId, null); - } - - /** - * Move the open thought text out of NarrativeStore without persisting it. - * - * Used when an authoritative boundary retroactively classifies previously - * unknown deltas as the final assistant response. Ownership moves to - * TurnFinalizer so the same text is not buffered in both stores. - */ - takeOpenThought(threadId: string): string { - const open = this.turnOpenThought.get(threadId); - this.turnOpenThought.set(threadId, null); - return open?.text ?? ""; - } - - /** - * Get the current parent tool call ID for a thread's active Agent nesting. - * This is the fallback consulted by `index.ts` enrichment and - * {@link bufferToolCall} when the SDK omits `parent_tool_use_id` (Trap 1). - */ - getCurrentParentToolCallId(threadId: string): string | undefined { - return this.getStackDerivedParentFallback(threadId); - } - - /** - * A single running Agent on the stack (buffer `status === "running"`) can - * serve as a parent fallback when the SDK omits `parent_tool_use_id`. - * Zero or multiple running Agents means the fallback is ambiguous (parallel - * dispatch, nested agents, or coordinator work after children); return - * undefined so tools do not attach under the wrong subagent row. - */ - private getStackDerivedParentFallback(threadId: string): string | undefined { - const stack = this.agentCallStack.get(threadId) ?? []; - if (stack.length === 0) return undefined; - - const buffer = this.turnToolCalls.get(threadId) ?? []; - const runningAgentIds: string[] = []; - for (const agentId of stack) { - const row = buffer.find( - (b) => b.toolCallId === agentId && b.toolName === "Agent", - ); - if (row?.status === "running") { - runningAgentIds.push(agentId); - } - } - - return runningAgentIds.length === 1 ? runningAgentIds[0] : undefined; - } - - /** - * Buffer a tool call event for later persistence and return the parent tool - * call ID attributed to it. An explicit provider parent always wins. The - * stack fallback applies only to non-Agent calls when exactly one Agent is - * still running (Trap 1). Agent calls never inherit a stack-derived parent - * and are pushed onto the `agentCallStack` (Trap 2 push site). - */ - bufferToolCall(threadId: string, event: BufferToolCallEvent): string | undefined { - const buffer = this.turnToolCalls.get(threadId) ?? []; - const stack = this.agentCallStack.get(threadId) ?? []; - const parentToolCallId = this.resolveParentToolCallId(threadId, event); - this.logParentToolCallAttribution(threadId, event, parentToolCallId, stack.length); - const existing = buffer.find((tc) => tc.toolCallId === event.toolCallId); - if (existing) { - return this.updateExistingBufferedToolCall(existing, event, parentToolCallId); - } - return this.addBufferedToolCall( - threadId, - buffer, - stack, - event, - parentToolCallId, - ); - } - - private resolveParentToolCallId( - threadId: string, - event: BufferToolCallEvent, - ): string | undefined { - if (event.toolName === "Agent") return event.parentToolCallId; - return event.parentToolCallId ?? this.getStackDerivedParentFallback(threadId); - } - - private logParentToolCallAttribution( - threadId: string, - event: BufferToolCallEvent, - parentToolCallId: string | undefined, - stackDepth: number, - ): void { - if (event.toolName === "Agent" || !parentToolCallId) return; - logger.debug("bufferToolCall: parent attribution", { - threadId, - toolCallId: event.toolCallId, - toolName: event.toolName, - sdkParent: event.parentToolCallId ?? null, - stackDepth, - attributed: parentToolCallId, - source: event.parentToolCallId ? "sdk" : "stack-fallback", - }); - } - - private updateExistingBufferedToolCall( - existing: BufferedToolCall, - event: BufferToolCallEvent, - parentToolCallId: string | undefined, - ): string | undefined { - const shouldMergeDuplicate = this.shouldMergeDuplicateToolCall(existing, event); - if (shouldMergeDuplicate) { - this.mergeDuplicateToolCall(existing, event); - } - this.mergeParentToolCallId( - existing, - event.parentToolCallId, - parentToolCallId, - shouldMergeDuplicate, - ); - return existing.parentToolCallId; - } - - private shouldMergeDuplicateToolCall( - existing: BufferedToolCall, - event: BufferToolCallEvent, - ): boolean { - if (existing.toolName === "Agent" && event.toolName === "Agent") return true; - const rawToolInput = existing._rawToolInput ?? {}; - return existing.status === "running" && ( - Object.keys(rawToolInput).length === 0 || existing.toolName !== event.toolName - ); - } - - private mergeDuplicateToolCall(existing: BufferedToolCall, event: BufferToolCallEvent): void { - existing.toolName = event.toolName; - const mergedToolInput = { - ...existing._rawToolInput, - ...event.toolInput, - }; - existing._rawToolInput = mergedToolInput; - if (event.toolName === "Agent") { - existing._subagentPresentation = mergeSubagentPresentation( - existing._subagentPresentation, - event.subagentPresentation - ?? createSubagentPresentation(mergedToolInput, event.toolCallId), - event.toolCallId, - ); - this.applyAgentPresentation(existing, mergedToolInput, event.toolCallId); - } - } - - private mergeParentToolCallId( - existing: BufferedToolCall, - providerParentToolCallId: string | undefined, - inferredParentToolCallId: string | undefined, - mergedDuplicate: boolean, - ): void { - if (providerParentToolCallId) { - existing.parentToolCallId = providerParentToolCallId; - return; - } - if (mergedDuplicate && inferredParentToolCallId && !existing.parentToolCallId) { - existing.parentToolCallId = inferredParentToolCallId; - } - } - - private applyAgentPresentation( - toolCall: BufferedToolCall, - rawToolInput: Record, - toolCallId: string, - ): void { - const presentation = toolCall._subagentPresentation - ?? createSubagentPresentation(rawToolInput, toolCallId); - toolCall.displayName = presentation.hasExplicitIdentity ? presentation.displayName : undefined; - toolCall.providerAgentKey = presentation.providerAgentKey; - toolCall.subagentIdentityKey = persistedSubagentIdentityKey(presentation) ?? toolCall.subagentIdentityKey; - toolCall.subagentProviderName = this.subagentProviderName(presentation); - Object.assign(toolCall, persistedSubagentMetadata(rawToolInput)); - if (toolCall.messageId && toolCall.subagentIdentityKey) { - this.toolCallRecordRepo.updateSubagentIdentity( - toolCall.toolCallId!, - toolCall.messageId, - toolCall.subagentIdentityKey, - ); - } - toolCall.model = presentation.model; - toolCall.reasoningEffort = presentation.reasoningEffort; - } - - private addBufferedToolCall( - threadId: string, - buffer: BufferedToolCall[], - stack: string[], - event: BufferToolCallEvent, - parentToolCallId: string | undefined, - ): string | undefined { - const sortOrder = this.nextSortOrder(threadId); - if (event.toolName === "Agent") { - stack.push(event.toolCallId); - this.agentCallStack.set(threadId, stack); - } - const presentation = this.subagentPresentation(event); - buffer.push(this.createBufferedToolCall(event, presentation, sortOrder, parentToolCallId)); - this.turnToolCalls.set(threadId, buffer); - return parentToolCallId; - } - - private subagentPresentation(event: BufferToolCallEvent): SubagentPresentation | undefined { - if (event.toolName !== "Agent") return undefined; - return event.subagentPresentation ?? createSubagentPresentation(event.toolInput, event.toolCallId); - } - - private createBufferedToolCall( - event: BufferToolCallEvent, - presentation: SubagentPresentation | undefined, - sortOrder: number, - parentToolCallId: string | undefined, - ): BufferedToolCall { - const isAgent = event.toolName === "Agent"; - return { - toolCallId: event.toolCallId, - messageId: "", - toolName: event.toolName, - displayName: presentation?.hasExplicitIdentity ? presentation.displayName : undefined, - providerAgentKey: presentation?.providerAgentKey, - subagentIdentityKey: isAgent ? persistedSubagentIdentityKey(presentation) : undefined, - subagentProviderName: this.subagentProviderName(presentation), - ...(isAgent ? persistedSubagentMetadata(event.toolInput) : {}), - model: presentation?.model, - reasoningEffort: presentation?.reasoningEffort, - inputSummary: "", - outputSummary: "", - status: "running", - startedAt: new Date().toISOString(), - sortOrder, - parentToolCallId, - _rawToolInput: event.toolInput, - ...(presentation ? { _subagentPresentation: presentation } : {}), - }; - } - - private subagentProviderName(presentation: SubagentPresentation | undefined): string | undefined { - if (presentation?.detail.kind !== "transcript-unavailable") return undefined; - return presentation.detail.providerName; - } - - /** - * Update a buffered tool call with its output when its result arrives, and - * pop the call from the `agentCallStack` if it was an Agent (Trap 2 pop site). - */ - updateBufferedToolCallOutput( - threadId: string, - toolCallId: string, - output: string, - isError: boolean, - toolInput?: Record, - outputMeta?: { - outputTruncated?: boolean; - outputTotalBytes?: number; - outputArtifactPath?: string; - exitCode?: number; - }, - subagentPresentation?: SubagentPresentation, - ): void { - this.removeAgentFromStack(threadId, toolCallId); - const toolCall = this.latestBufferedToolCall(threadId, toolCallId); - if (!toolCall) return; - this.applyToolCallOutput(toolCall, output, isError, outputMeta); - this.mergeToolCallInput(toolCall, toolInput); - if (toolCall.toolName === "Agent") { - const incomingPresentation = subagentPresentation - ?? createSubagentPresentation(toolCall._rawToolInput ?? {}, toolCallId); - toolCall._subagentPresentation = mergeSubagentPresentation( - toolCall._subagentPresentation, - incomingPresentation, - toolCallId, - ); - this.applyAgentPresentation(toolCall, toolCall._rawToolInput ?? {}, toolCallId); - } - } - - private removeAgentFromStack(threadId: string, toolCallId: string): void { - const stack = this.agentCallStack.get(threadId) ?? []; - const stackIndex = stack.indexOf(toolCallId); - if (stackIndex < 0) return; - stack.splice(stackIndex, 1); - this.agentCallStack.set(threadId, stack); - logger.debug("updateBufferedToolCallOutput: popped Agent from stack", { - threadId, - toolCallId, - remainingDepth: stack.length, - }); - } - - private latestBufferedToolCall(threadId: string, toolCallId: string): BufferedToolCall | undefined { - const buffer = this.turnToolCalls.get(threadId) ?? []; - for (let index = buffer.length - 1; index >= 0; index -= 1) { - if (buffer[index].toolCallId === toolCallId) return buffer[index]; - } - return undefined; - } - - private applyToolCallOutput( - toolCall: BufferedToolCall, - output: string, - isError: boolean, - outputMeta: { - outputTruncated?: boolean; - outputTotalBytes?: number; - outputArtifactPath?: string; - exitCode?: number; - } | undefined, - ): void { - const outputLimit = resolveBrowserNarrativeTool(toolCall.toolName) ? 4_000 : 500; - toolCall.outputSummary = output.slice(0, outputLimit); - toolCall.outputTruncated = outputMeta?.outputTruncated === true; - this.applyOutputMetadata(toolCall, outputMeta); - toolCall.status = isError ? "failed" : "completed"; - toolCall.completedAt = new Date().toISOString(); - } - - private applyOutputMetadata( - toolCall: BufferedToolCall, - outputMeta: { - outputTotalBytes?: number; - outputArtifactPath?: string; - exitCode?: number; - } | undefined, - ): void { - delete toolCall.outputTotalBytes; - delete toolCall.outputArtifactPath; - delete toolCall.exitCode; - if (outputMeta?.outputTotalBytes != null) toolCall.outputTotalBytes = outputMeta.outputTotalBytes; - if (outputMeta?.outputArtifactPath) toolCall.outputArtifactPath = outputMeta.outputArtifactPath; - if (outputMeta?.exitCode !== undefined) toolCall.exitCode = outputMeta.exitCode; - } - - private mergeToolCallInput(toolCall: BufferedToolCall, toolInput: Record | undefined): void { - if (!toolInput || Object.keys(toolInput).length === 0) return; - toolCall._rawToolInput = { - ...toolCall._rawToolInput, - ...toolInput, - }; - } - - /** - * Clear the whole Agent stack when a final `Message` event arrives — the turn - * is over and any Agent calls still on the stack are implicitly done (Trap 2 - * end-of-turn clear). No-ops when the stack is already empty. - */ - clearAgentStackOnMessage(threadId: string): void { - const stack = this.agentCallStack.get(threadId); - if (stack && stack.length > 0) { - stack.length = 0; - } - } - - /** Snapshot of the thread's buffered tool calls (read-only inspection). */ - getBufferedToolCalls(threadId: string): readonly BufferedToolCall[] { - return this.turnToolCalls.get(threadId) ?? []; - } - - /** - * Snapshot the visible structured narrative for an unfinished turn without - * retaining provider protocol traffic or private raw tool input. - */ - recoverySnapshot(threadId: string): ParentNarrativeRecoveryItem[] { - const snapshot: ParentNarrativeRecoveryItem[] = []; - let bytes = 0; - bytes = this.appendBufferedToolCallRecoveryItems(snapshot, bytes, threadId); - for (const thought of this.turnThoughts.get(threadId) ?? []) { - bytes = this.appendRecoverySnapshotItem( - snapshot, - bytes, - this.thoughtRecoveryItem(thought), - threadId, - ); - } - const openThought = this.turnOpenThought.get(threadId); - if (openThought) { - bytes = this.appendRecoverySnapshotItem( - snapshot, - bytes, - this.openThoughtRecoveryItem(openThought), - threadId, - ); - } - for (const hook of this.turnHooks.get(threadId) ?? []) { - bytes = this.appendRecoverySnapshotItem( - snapshot, - bytes, - this.hookRecoveryItem(hook), - threadId, - ); - } - for (const hook of this.turnOpenHooks.get(threadId)?.values() ?? []) { - bytes = this.appendRecoverySnapshotItem( - snapshot, - bytes, - this.openHookRecoveryItem(hook), - threadId, - ); - } - return this.sortRecoverySnapshot(snapshot); - } - - private appendBufferedToolCallRecoveryItems( - snapshot: ParentNarrativeRecoveryItem[], - bytes: number, - threadId: string, - ): number { - for (const toolCall of this.turnToolCalls.get(threadId) ?? []) { - bytes = this.appendRecoverySnapshotItem( - snapshot, - bytes, - this.toolCallRecoveryItem(toolCall), - threadId, - ); - } - return bytes; - } - - private appendRecoverySnapshotItem( - snapshot: ParentNarrativeRecoveryItem[], - bytes: number, - item: ParentNarrativeRecoveryItem, - threadId: string, - ): number { - const nextBytes = Buffer.byteLength(JSON.stringify(item), "utf8"); - this.assertRecoveryItemFitsWriteBatch(nextBytes, threadId); - const retainedBytes = bytes + nextBytes; - assertActiveTurnRecoveryRetention(snapshot.length + 1, retainedBytes); - snapshot.push(item); - return retainedBytes; - } - - private toolCallRecoveryItem(toolCall: BufferedToolCall): ParentNarrativeRecoveryItem { - return { - kind: "toolCall", - record: { - ...this.toolCallRecoveryIdentityFields(toolCall), - ...this.toolCallRecoveryPresentationFields(toolCall), - ...this.toolCallRecoveryOutputFields(toolCall), - ...this.toolCallRecoveryStateFields(toolCall), - }, - }; - } - - private toolCallRecoveryIdentityFields(toolCall: BufferedToolCall) { - return { - id: this.requireRecoveryString(toolCall.toolCallId, "tool call id"), - message_id: this.requireRecoveryString(toolCall.messageId, "tool call message id"), - parent_tool_call_id: toolCall.parentToolCallId ?? null, - tool_name: toolCall.toolName, - display_name: toolCall.displayName ?? null, - provider_agent_key: toolCall.providerAgentKey ?? null, - subagent_identity_key: toolCall.subagentIdentityKey ?? null, - subagent_provider_name: toolCall.subagentProviderName ?? null, - }; - } - - private toolCallRecoveryPresentationFields(toolCall: BufferedToolCall) { - return { - subagent_prompt: toolCall.subagentPrompt ?? null, - subagent_type: toolCall.subagentType ?? null, - subagent_agent_id: toolCall.subagentAgentId ?? null, - subagent_duration_ms: toolCall.subagentDurationMs ?? null, - model: toolCall.model ?? null, - reasoning_effort: toolCall.reasoningEffort ?? null, - }; - } - - private toolCallRecoveryOutputFields(toolCall: BufferedToolCall) { - return { - input_summary: this.toolCallRecoveryInputSummary(toolCall), - output_summary: toolCall.outputSummary, - ...(toolCall.outputTruncated ? { output_truncated: 1 } : {}), - output_total_bytes: toolCall.outputTotalBytes ?? null, - output_artifact_path: toolCall.outputArtifactPath ?? null, - exit_code: toolCall.exitCode ?? null, - }; - } - - private toolCallRecoveryInputSummary(toolCall: BufferedToolCall): string { - if (toolCall.inputSummary) return toolCall.inputSummary; - return this.summarizeInput(toolCall.toolName, toolCall._rawToolInput ?? {}); - } - - private toolCallRecoveryStateFields(toolCall: BufferedToolCall) { - return { - status: toolCall.status, - started_at: this.requireRecoveryString(toolCall.startedAt, "tool call start time"), - completed_at: toolCall.completedAt ?? null, - sort_order: toolCall.sortOrder, - }; - } - - private thoughtRecoveryItem(thought: CreateThoughtSegmentInput): ParentNarrativeRecoveryItem { - return { - kind: "narrationSegment", - record: { - id: this.requireRecoveryString(thought.id, "thought id"), - message_id: this.requireRecoveryString(thought.messageId, "thought message id"), - text: this.requireRecoveryString(thought.text, "thought text"), - started_at: this.requireRecoveryString(thought.startedAt, "thought start time"), - ended_at: thought.endedAt ?? null, - sort_order: thought.sortOrder, - ...(thought.isFinalResponse ? { is_final_response: thought.isFinalResponse } : {}), - }, - }; - } - - private openThoughtRecoveryItem(openThought: OpenThought): ParentNarrativeRecoveryItem { - return { - kind: "narrationSegment", - record: { - id: openThought.id, - message_id: "", - text: openThought.text, - started_at: openThought.startedAt, - ended_at: null, - sort_order: openThought.sortOrder, - }, - }; - } - - private hookRecoveryItem(hook: CreateHookExecutionInput): ParentNarrativeRecoveryItem { - return { - kind: "hook", - record: { - id: this.requireRecoveryString(hook.id, "hook id"), - message_id: this.requireRecoveryString(hook.messageId, "hook message id"), - hook_name: hook.hookName, - tool_name: hook.toolName, - phase: hook.phase, - payload: hook.payload, - duration_ms: hook.durationMs ?? null, - did_block: hook.didBlock, - started_at: this.requireRecoveryString(hook.startedAt, "hook start time"), - ended_at: hook.endedAt ?? null, - sort_order: hook.sortOrder, - }, - }; - } - - private openHookRecoveryItem(hook: OpenHook): ParentNarrativeRecoveryItem { - return { - kind: "hook", - record: { - id: hook.id, - message_id: "", - hook_name: hook.hookName, - tool_name: hook.toolName, - phase: hook.phase, - payload: hook.payload, - duration_ms: null, - did_block: false, - started_at: hook.startedAt, - ended_at: null, - sort_order: hook.sortOrder, - }, - }; - } - - private sortRecoverySnapshot( - snapshot: ParentNarrativeRecoveryItem[], - ): ParentNarrativeRecoveryItem[] { - return snapshot.sort((left, right) => ( - left.record.sort_order - right.record.sort_order || left.record.id.localeCompare(right.record.id) - )); - } - - /** Stage narration for recovery without mutating the active turn buffer. */ - stageNarrationSegment(threadId: string, text: string): StagedNarrationSegment | null { - const open = this.turnOpenThought.get(threadId); - const endedAt = new Date().toISOString(); - if (open) { - return { - id: open.id, - text: `${open.text}${text}`, - startedAt: open.startedAt, - endedAt, - sortOrder: open.sortOrder, - openThoughtId: open.id, - }; - } - if (!text) return null; - return { - id: NodeCrypto.randomUUID(), - text, - startedAt: endedAt, - endedAt, - sortOrder: this.turnSortCounters.get(threadId) ?? 0, - }; - } - - /** Add staged narration to a recovery snapshot without mutating the active turn buffer. */ - recoverySnapshotWithStagedNarration( - threadId: string, - staged: StagedNarrationSegment, - ): ParentNarrativeRecoveryItem[] { - const snapshot = this.recoverySnapshot(threadId).filter((item) => ( - item.kind !== "narrationSegment" || item.record.id !== staged.id - )); - snapshot.push({ - kind: "narrationSegment", - record: { - id: staged.id, - message_id: "", - text: staged.text, - started_at: staged.startedAt, - ended_at: staged.endedAt, - sort_order: staged.sortOrder, - }, - }); - let bytes = 0; - for (const [index, item] of snapshot.entries()) { - const itemBytes = Buffer.byteLength(JSON.stringify(item), "utf8"); - this.assertRecoveryItemFitsWriteBatch(itemBytes, threadId); - bytes += itemBytes; - assertActiveTurnRecoveryRetention(index + 1, bytes); - } - return snapshot.sort((left, right) => ( - left.record.sort_order - right.record.sort_order || left.record.id.localeCompare(right.record.id) - )); - } - - /** Apply a narration record only after its recovery projection is durable. */ - applyStagedNarrationSegment(threadId: string, staged: StagedNarrationSegment): void { - if (staged.openThoughtId) { - const open = this.turnOpenThought.get(threadId); - if (!open || open.id !== staged.openThoughtId) { - throw new Error(`Staged narration no longer matches the open thought: ${staged.id}`); - } - this.turnOpenThought.set(threadId, null); - } else { - const nextSortOrder = this.turnSortCounters.get(threadId) ?? 0; - if (nextSortOrder <= staged.sortOrder) { - this.turnSortCounters.set(threadId, staged.sortOrder + 1); - } - } - const thoughts = this.turnThoughts.get(threadId) ?? []; - thoughts.push({ - id: staged.id, - messageId: "", - text: staged.text, - startedAt: staged.startedAt, - endedAt: staged.endedAt, - sortOrder: staged.sortOrder, - }); - this.turnThoughts.set(threadId, thoughts); - } - - private requireRecoveryString(value: string | undefined, field: string): string { - if (value === undefined) throw new Error(`Narrative recovery is missing ${field}`); - return value; - } - /** Persist a durable semantic snapshot against its recovered assistant row. */ persistRecoveredNarrative( messageId: string, @@ -1333,12 +496,6 @@ export class NarrativeStore { if (hooks.length > 0) this.hookExecutionRepo.bulkCreate(hooks, replaceExisting); } - private assertRecoveryItemFitsWriteBatch(byteLength: number, threadId: string): void { - if (byteLength > ACTIVE_TURN_WRITE_BATCH_LIMITS.maxBytes) { - throw new Error(`Parent narrative recovery item exceeds the active-turn byte limit: ${threadId}`); - } - } - private recoveredToolCall( messageId: string, item: RecoveredToolCallItem, @@ -1444,67 +601,6 @@ export class NarrativeStore { }; } - /** - * True when the current turn has buffered at least one narrative contributor — - * a tool call, a narration segment (open or closed), or a hook (open or - * closed). Feeds the {@link TurnFinalizer.hasRecordableActivity} predicate so - * a turn with narrative but no assistant body still earns a persisted row. - */ - hasBufferedNarrative(threadId: string): boolean { - return this.narrativeBufferStates(threadId).some(Boolean); - } - - private narrativeBufferStates(threadId: string): boolean[] { - return [ - (this.turnToolCalls.get(threadId)?.length ?? 0) > 0, - Boolean(this.turnOpenThought.get(threadId)), - (this.turnThoughts.get(threadId)?.length ?? 0) > 0, - (this.turnOpenHooks.get(threadId)?.size ?? 0) > 0, - (this.turnHooks.get(threadId)?.length ?? 0) > 0, - ]; - } - - /** - * Record an in-flight hook execution (HookStarted). The caller supplies the - * already-allocated sort order (the late-hook path in AgentService allocates - * it before deciding routing). Returns the generated row id. - */ - openHook( - threadId: string, - hook: { hookName: string; toolName: string | null; phase: string; payload: string; sortOrder: number }, - ): string { - const map = this.turnOpenHooks.get(threadId) ?? new Map(); - const id = NodeCrypto.randomUUID(); - map.set(hook.hookName, { - id, - hookName: hook.hookName, - toolName: hook.toolName, - phase: hook.phase, - payload: hook.payload, - startedAt: new Date().toISOString(), - sortOrder: hook.sortOrder, - }); - this.turnOpenHooks.set(threadId, map); - return id; - } - - /** Look up (without removing) an open hook by name. */ - peekOpenHook(threadId: string, hookName: string): OpenHook | undefined { - return this.turnOpenHooks.get(threadId)?.get(hookName); - } - - /** Remove an open hook by name (after it has been completed or flushed). */ - removeOpenHook(threadId: string, hookName: string): void { - this.turnOpenHooks.get(threadId)?.delete(hookName); - } - - /** Push a completed hook execution onto the closed-hooks list for persistence. */ - pushClosedHook(threadId: string, hook: CreateHookExecutionInput): void { - const list = this.turnHooks.get(threadId) ?? []; - list.push(hook); - this.turnHooks.set(threadId, list); - } - /** * Persist the buffered narrative rows (tool calls, thoughts, hooks) for a * completed turn against `messageId`, tagging the final-response thought via @@ -1664,212 +760,4 @@ export class NarrativeStore { ): void { for (const item of items) persisted.add(identifier(item)); } - - private prepareNarrativePersistence( - threadId: string, - messageId: string, - messageContent: string, - outcome: TurnOutcome, - ): PreparedNarrativePersistence { - const toolCalls = this.prepareToolCallsForPersistence(threadId, messageId, outcome); - this.closeOpenThought(threadId); - this.closeOpenHooksForPersistence(threadId); - const thoughts = this.prepareThoughtsForPersistence(threadId, messageId, messageContent); - const hooks = this.prepareHooksForPersistence(threadId, messageId); - return { toolCalls, thoughts, hooks }; - } - - private prepareToolCallsForPersistence( - threadId: string, - messageId: string, - outcome: TurnOutcome, - ): BufferedToolCall[] { - const toolCalls = this.turnToolCalls.get(threadId) ?? []; - const settledAt = new Date().toISOString(); - for (const toolCall of toolCalls) { - toolCall.toolCallId ??= NodeCrypto.randomUUID(); - this.settleRunningToolCall(toolCall, outcome, settledAt); - toolCall.messageId = messageId; - this.prepareToolCallInput(toolCall); - } - return toolCalls; - } - - private settleRunningToolCall( - toolCall: BufferedToolCall, - outcome: TurnOutcome, - settledAt: string, - ): void { - if (toolCall.status !== "running") return; - toolCall.status = this.settledToolCallStatus(outcome); - toolCall.completedAt = settledAt; - } - - private settledToolCallStatus(outcome: TurnOutcome): BufferedToolCall["status"] { - if (outcome === "errored" || outcome === "interrupted") return "failed"; - if (outcome === "cancelled") return "cancelled"; - return "completed"; - } - - private prepareToolCallInput(toolCall: BufferedToolCall): void { - const rawToolInput = toolCall._rawToolInput; - if (toolCall.inputSummary || !rawToolInput) return; - if (toolCall.toolName === "Agent") { - this.applyAgentPersistenceMetadata(toolCall, rawToolInput); - } - toolCall.inputSummary = this.summarizeInput(toolCall.toolName, rawToolInput); - delete toolCall._rawToolInput; - } - - private applyAgentPersistenceMetadata( - toolCall: BufferedToolCall, - rawToolInput: Record, - ): void { - const presentation = createSubagentPresentation(rawToolInput, toolCall.toolCallId!); - const persistedPresentation = toolCall._subagentPresentation ?? presentation; - toolCall.displayName = persistedPresentation.hasExplicitIdentity - ? persistedPresentation.displayName - : undefined; - toolCall.providerAgentKey = persistedPresentation.providerAgentKey; - toolCall.subagentIdentityKey = persistedSubagentIdentityKey(persistedPresentation) - ?? toolCall.subagentIdentityKey; - toolCall.subagentProviderName = this.subagentProviderName(persistedPresentation); - Object.assign(toolCall, persistedSubagentMetadata(rawToolInput)); - toolCall.model = persistedPresentation.model; - toolCall.reasoningEffort = persistedPresentation.reasoningEffort; - } - - private closeOpenHooksForPersistence(threadId: string): void { - const openHookMap = this.turnOpenHooks.get(threadId); - if (!openHookMap || openHookMap.size === 0) return; - const list = this.turnHooks.get(threadId) ?? []; - const endedAt = new Date().toISOString(); - for (const open of openHookMap.values()) { - list.push({ - id: open.id, - messageId: "", - hookName: open.hookName, - toolName: open.toolName, - phase: open.phase, - payload: open.payload, - durationMs: Date.parse(endedAt) - Date.parse(open.startedAt), - didBlock: false, - startedAt: open.startedAt, - endedAt, - sortOrder: open.sortOrder, - }); - } - this.turnHooks.set(threadId, list); - openHookMap.clear(); - } - - private prepareThoughtsForPersistence( - threadId: string, - messageId: string, - messageContent: string, - ): CreateThoughtSegmentInput[] { - const bufferedThoughts = this.turnThoughts.get(threadId) ?? []; - for (const thought of bufferedThoughts) thought.id ??= NodeCrypto.randomUUID(); - const thoughts = bufferedThoughts.map((thought) => ({ ...thought, messageId })); - const message = messageContent.trim(); - if (message.length > 0 && thoughts.length > 0) { - this.markFinalResponseThoughts(thoughts, message); - } - return thoughts; - } - - private markFinalResponseThoughts( - thoughts: CreateThoughtSegmentInput[], - message: string, - ): void { - const maxSortOrder = Math.max(...thoughts.map((thought) => thought.sortOrder)); - for (const thought of thoughts) { - const text = thought.text.trim(); - if (text.length > 0 && this.isFinalResponseThought(thought, text, message, maxSortOrder)) { - thought.isFinalResponse = 1; - } - } - } - - private isFinalResponseThought( - thought: CreateThoughtSegmentInput, - text: string, - message: string, - maxSortOrder: number, - ): boolean { - if (text === message) return true; - return thought.sortOrder === maxSortOrder - && (thought.isFinalResponse === 1 || message.endsWith(text)); - } - - private prepareHooksForPersistence( - threadId: string, - messageId: string, - ): CreateHookExecutionInput[] { - const bufferedHooks = this.turnHooks.get(threadId) ?? []; - for (const hook of bufferedHooks) hook.id ??= NodeCrypto.randomUUID(); - return bufferedHooks.map((hook) => ({ ...hook, messageId })); - } - - /** - * Clear the per-turn narrative buffers this store owns. The sort counter and - * Agent stack are intentionally NOT cleared here — they are reset in the - * TurnStarted handler so late hooks that arrive after this point can still - * increment the completed turn's counter (mirrors the old clearTurnState). - */ - clearTurn(threadId: string): void { - this.turnToolCalls.delete(threadId); - this.turnOpenThought.delete(threadId); - this.turnThoughts.delete(threadId); - this.turnOpenHooks.delete(threadId); - this.turnHooks.delete(threadId); - } - - /** Generate a human-readable summary of tool input. */ - private summarizeInput(toolName: string, input: Record): string { - if (resolveBrowserNarrativeTool(toolName)) return JSON.stringify(input).slice(0, 4_000); - return (NARRATIVE_INPUT_SUMMARIZERS[toolName.toLowerCase()] ?? genericInputSummary)(input); - } -} - -function fileInputSummary(input: Record): string { - return String(firstProvidedInputValue(input, ["file_path", "filePath"]) ?? ""); -} - -function renameInputSummary(input: Record): string { - const source = firstProvidedInputValue(input, [ - "oldPath", "old_path", "oldFilePath", "sourcePath", "source_path", "source", "from", - ]); - const destination = firstProvidedInputValue(input, [ - "newPath", "new_path", "destinationPath", "destination_path", "destination", "to", "path", - "file_path", "filePath", "target_file", "targetFile", - ]); - return [source, destination] - .filter((value): value is string => typeof value === "string") - .join(" -> ") - .slice(0, 200); -} - -function commandInputSummary(input: Record): string { - return String(firstProvidedInputValue(input, ["command"]) ?? "").slice(0, MAX_PERSISTED_COMMAND_CHARS); -} - -function patternInputSummary(input: Record): string { - return String(firstProvidedInputValue(input, ["pattern"]) ?? ""); -} - -function agentInputSummary(input: Record): string { - return String(firstProvidedInputValue(input, ["description"]) ?? "").slice(0, 100); -} - -function genericInputSummary(input: Record): string { - return JSON.stringify(input).slice(0, 200); -} - -function firstProvidedInputValue(input: Record, keys: readonly string[]): unknown { - for (const key of keys) { - const value = input[key]; - if (value !== null && value !== undefined) return value; - } - return undefined; } diff --git a/apps/server/src/features/agents/conversation/narrative/narrative-turn-state.ts b/apps/server/src/features/agents/conversation/narrative/narrative-turn-state.ts new file mode 100644 index 000000000..c913db728 --- /dev/null +++ b/apps/server/src/features/agents/conversation/narrative/narrative-turn-state.ts @@ -0,0 +1,1154 @@ +import * as NodeCrypto from "node:crypto"; +import { logger } from "@mcode/shared"; +import { + createSubagentPresentation, mergeSubagentPresentation, resolveBrowserNarrativeTool, + resolveSubagentDuration, resolveSubagentMetadata, resolveSubagentPrompt, + encodeCanonicalSubagentDetailTarget, encodeSubagentAliasDetailTarget, + type ParentNarrativeRecoveryItem, type SubagentPresentation, +} from "@mcode/contracts"; +import type { CreateToolCallRecordInput } from "../../tools/persistence/tool-call-record-repo.js"; +import type { CreateThoughtSegmentInput } from "./persistence/thought-segment-repo.js"; +import type { CreateHookExecutionInput } from "../../events/persistence/hook-execution-repo.js"; +import type { TurnOutcome } from "../../turns/turn-outcome.js"; +import { ACTIVE_TURN_WRITE_BATCH_LIMITS } from "../../../../runtime/persistence/sqlite/bounded-write-batches.js"; +import { assertActiveTurnRecoveryRetention } from "../../turns/active-turn-recovery-retention-policy.js"; + +/** Buffered tool call with raw input preserved for deferred summarization. */ +export interface BufferedToolCall extends CreateToolCallRecordInput { + _rawToolInput?: Record; + _subagentPresentation?: SubagentPresentation; +} + +/** Extracts bounded provider metadata that must survive a persisted Agent card. */ +function persistedSubagentMetadata(input: Record) { + return { + subagentPrompt: resolveSubagentPrompt(input.prompt), + subagentType: resolveSubagentMetadata(input.subagentType), + subagentAgentId: resolveSubagentMetadata(input.agentId), + subagentDurationMs: resolveSubagentDuration(input.durationMs), + }; +} + +function persistedSubagentIdentityKey(presentation: SubagentPresentation | undefined): string | undefined { + if (!presentation) return undefined; + if (presentation.detail.kind === "canonical-child") { + return encodeCanonicalSubagentDetailTarget(presentation.detail.threadId); + } + return presentation.detail.kind === "canonical-alias" + ? encodeSubagentAliasDetailTarget(presentation.detail.identityKey) + : undefined; +} + +/** In-flight thought segment accumulated from consecutive textDelta events. */ +interface OpenThought { + id: string; + text: string; + startedAt: string; + sortOrder: number; +} + +/** In-flight hook execution awaiting its paired HookCompleted. */ +export interface OpenHook { + id: string; + hookName: string; + toolName: string | null; + phase: string; + payload: string; + startedAt: string; + sortOrder: number; +} + +/** One narration record staged for a durable ownership transfer. */ +export interface StagedNarrationSegment { + id: string; + text: string; + startedAt: string; + endedAt: string; + sortOrder: number; + openThoughtId?: string; +} + +/** Tool-use event shape consumed by {@link NarrativeTurnState.bufferToolCall}. */ +export interface BufferToolCallEvent { + toolCallId: string; + toolName: string; + toolInput: Record; + parentToolCallId?: string; + subagentPresentation?: SubagentPresentation; +} + +/** Complete narrative rows prepared for one terminal persistence pass. */ +export interface PreparedNarrativePersistence { + toolCalls: BufferedToolCall[]; + thoughts: CreateThoughtSegmentInput[]; + hooks: CreateHookExecutionInput[]; +} + +/** Bounds persisted shell commands while retaining enough text for readable expansion. */ +const MAX_PERSISTED_COMMAND_CHARS = 4096; + +const NARRATIVE_INPUT_SUMMARIZERS: Record) => string> = { + read: fileInputSummary, + edit: fileInputSummary, + write: fileInputSummary, + move: renameInputSummary, + rename: renameInputSummary, + bash: commandInputSummary, + shell: commandInputSummary, + terminal: commandInputSummary, + command_execution: commandInputSummary, + grep: patternInputSummary, + glob: patternInputSummary, + agent: agentInputSummary, +}; + +function fileInputSummary(input: Record): string { + return String(firstProvidedInputValue(input, ["file_path", "filePath"]) ?? ""); +} + +function renameInputSummary(input: Record): string { + const source = firstProvidedInputValue(input, [ + "oldPath", "old_path", "oldFilePath", "sourcePath", "source_path", "source", "from", + ]); + const destination = firstProvidedInputValue(input, [ + "newPath", "new_path", "destinationPath", "destination_path", "destination", "to", "path", + "file_path", "filePath", "target_file", "targetFile", + ]); + return [source, destination] + .filter((value): value is string => typeof value === "string") + .join(" -> ") + .slice(0, 200); +} + +function commandInputSummary(input: Record): string { + return String(firstProvidedInputValue(input, ["command"]) ?? "").slice(0, MAX_PERSISTED_COMMAND_CHARS); +} + +function patternInputSummary(input: Record): string { + return String(firstProvidedInputValue(input, ["pattern"]) ?? ""); +} + +function agentInputSummary(input: Record): string { + return String(firstProvidedInputValue(input, ["description"]) ?? "").slice(0, 100); +} + +function genericInputSummary(input: Record): string { + return JSON.stringify(input).slice(0, 200); +} + +function firstProvidedInputValue(input: Record, keys: readonly string[]): unknown { + for (const key of keys) { + const value = input[key]; + if (value !== null && value !== undefined) return value; + } + return undefined; +} + +/** Data-only update emitted when a buffered Agent gains its durable identity. */ +export type NarrativeTurnStateEffect = { + readonly kind: "update-subagent-identity"; + readonly toolCallId: string; + readonly messageId: string; + readonly identityKey: string; +}; + +/** Owns per-thread narrative buffers and classification without a database connection. */ +export class NarrativeTurnState { + private turnToolCalls = new Map(); + private agentCallStack = new Map(); + private turnSortCounters = new Map(); + private turnOpenThought = new Map(); + private turnThoughts = new Map(); + private turnOpenHooks = new Map>(); + private turnHooks = new Map(); + private effectSink: ((effect: NarrativeTurnStateEffect) => void) | undefined; + private readonly pendingEffects: NarrativeTurnStateEffect[] = []; + + protected setEffectSink(sink: (effect: NarrativeTurnStateEffect) => void): void { + this.effectSink = sink; + } + + /** Return any data-only effects emitted without a live persistence adapter. */ + takeEffects(): NarrativeTurnStateEffect[] { + return this.pendingEffects.splice(0); + } + + private emitEffect(effect: NarrativeTurnStateEffect): void { + if (this.effectSink) this.effectSink(effect); + else this.pendingEffects.push(effect); + } + + /** + * Reset the volatile per-turn buffers at the START of a turn (Trap 3). Mirrors + * the seeding AgentService used to do in its `sendMessage`/turnStarted prelude. + * Note: the sort counter and Agent stack are reset separately via + * {@link resetTurnCounters} on the TurnStarted event so late hooks from the + * prior turn can still increment the old counter. + */ + beginTurn(threadId: string): void { + this.turnToolCalls.set(threadId, []); + this.turnOpenThought.set(threadId, null); + this.turnThoughts.set(threadId, []); + this.turnOpenHooks.set(threadId, new Map()); + this.turnHooks.set(threadId, []); + } + + /** + * Reset the per-turn sort counter and Agent stack. Called from the + * TurnStarted handler rather than {@link beginTurn} so a fresh counter is + * available for each new turn while late hooks from the prior turn can still + * increment the old one (see {@link clearTurn}). + */ + resetTurnCounters(threadId: string): void { + this.turnSortCounters.set(threadId, 0); + this.agentCallStack.set(threadId, []); + } + + /** Allocate the next shared sort order for the thread's current turn. */ + nextSortOrder(threadId: string): number { + const sortOrder = this.turnSortCounters.get(threadId) ?? 0; + this.turnSortCounters.set(threadId, sortOrder + 1); + return sortOrder; + } + + /** + * Open or extend the in-flight thought segment from a non-final `textDelta`. + * The sort order is allocated lazily on the first delta so consecutive deltas + * keep the same slot, taken BEFORE any following tool call's slot — matching + * the live client builder. Never touches the `agentCallStack` (Trap 2). + */ + openOrExtendThought(threadId: string, delta: string): void { + const open = this.turnOpenThought.get(threadId); + if (!open) { + const sortOrder = this.nextSortOrder(threadId); + this.turnOpenThought.set(threadId, { + id: NodeCrypto.randomUUID(), + text: delta, + startedAt: new Date().toISOString(), + sortOrder, + }); + } else { + open.text += delta; + } + } + + /** + * Close any in-flight thought segment for the thread and push it onto the + * closed-thoughts list. Called before a tool call begins (so the thought + * sorts strictly before the tool) and during turn-end drain. + */ + closeOpenThought(threadId: string): void { + const open = this.turnOpenThought.get(threadId); + if (!open) return; + const list = this.turnThoughts.get(threadId) ?? []; + list.push({ + id: open.id, + messageId: "", + text: open.text, + startedAt: open.startedAt, + endedAt: new Date().toISOString(), + sortOrder: open.sortOrder, + }); + this.turnThoughts.set(threadId, list); + this.turnOpenThought.set(threadId, null); + } + + /** + * Discards the open thought without persisting it. + * + * Called when `AssistantMessageBoundary` reports `isFinalResponse: true` — + * the streamed text was actually the final assistant response and will be + * persisted via the `Message` event, so keeping the matching thought row + * would duplicate the body as a ThoughtBlock in the narrative. + */ + dropOpenThought(threadId: string): void { + this.turnOpenThought.set(threadId, null); + } + + /** + * Move the open thought text out of NarrativeStore without persisting it. + * + * Used when an authoritative boundary retroactively classifies previously + * unknown deltas as the final assistant response. Ownership moves to + * TurnFinalizer so the same text is not buffered in both stores. + */ + takeOpenThought(threadId: string): string { + const open = this.turnOpenThought.get(threadId); + this.turnOpenThought.set(threadId, null); + return open?.text ?? ""; + } + + /** + * Get the current parent tool call ID for a thread's active Agent nesting. + * This is the fallback consulted by `index.ts` enrichment and + * {@link bufferToolCall} when the SDK omits `parent_tool_use_id` (Trap 1). + */ + getCurrentParentToolCallId(threadId: string): string | undefined { + return this.getStackDerivedParentFallback(threadId); + } + + /** + * A single running Agent on the stack (buffer `status === "running"`) can + * serve as a parent fallback when the SDK omits `parent_tool_use_id`. + * Zero or multiple running Agents means the fallback is ambiguous (parallel + * dispatch, nested agents, or coordinator work after children); return + * undefined so tools do not attach under the wrong subagent row. + */ + private getStackDerivedParentFallback(threadId: string): string | undefined { + const stack = this.agentCallStack.get(threadId) ?? []; + if (stack.length === 0) return undefined; + + const buffer = this.turnToolCalls.get(threadId) ?? []; + const runningAgentIds: string[] = []; + for (const agentId of stack) { + const row = buffer.find( + (b) => b.toolCallId === agentId && b.toolName === "Agent", + ); + if (row?.status === "running") { + runningAgentIds.push(agentId); + } + } + + return runningAgentIds.length === 1 ? runningAgentIds[0] : undefined; + } + + /** + * Buffer a tool call event for later persistence and return the parent tool + * call ID attributed to it. An explicit provider parent always wins. The + * stack fallback applies only to non-Agent calls when exactly one Agent is + * still running (Trap 1). Agent calls never inherit a stack-derived parent + * and are pushed onto the `agentCallStack` (Trap 2 push site). + */ + bufferToolCall(threadId: string, event: BufferToolCallEvent): string | undefined { + const buffer = this.turnToolCalls.get(threadId) ?? []; + const stack = this.agentCallStack.get(threadId) ?? []; + const parentToolCallId = this.resolveParentToolCallId(threadId, event); + this.logParentToolCallAttribution(threadId, event, parentToolCallId, stack.length); + const existing = buffer.find((tc) => tc.toolCallId === event.toolCallId); + if (existing) { + return this.updateExistingBufferedToolCall(existing, event, parentToolCallId); + } + return this.addBufferedToolCall( + threadId, + buffer, + stack, + event, + parentToolCallId, + ); + } + + private resolveParentToolCallId( + threadId: string, + event: BufferToolCallEvent, + ): string | undefined { + if (event.toolName === "Agent") return event.parentToolCallId; + return event.parentToolCallId ?? this.getStackDerivedParentFallback(threadId); + } + + private logParentToolCallAttribution( + threadId: string, + event: BufferToolCallEvent, + parentToolCallId: string | undefined, + stackDepth: number, + ): void { + if (event.toolName === "Agent" || !parentToolCallId) return; + logger.debug("bufferToolCall: parent attribution", { + threadId, + toolCallId: event.toolCallId, + toolName: event.toolName, + sdkParent: event.parentToolCallId ?? null, + stackDepth, + attributed: parentToolCallId, + source: event.parentToolCallId ? "sdk" : "stack-fallback", + }); + } + + private updateExistingBufferedToolCall( + existing: BufferedToolCall, + event: BufferToolCallEvent, + parentToolCallId: string | undefined, + ): string | undefined { + const shouldMergeDuplicate = this.shouldMergeDuplicateToolCall(existing, event); + if (shouldMergeDuplicate) { + this.mergeDuplicateToolCall(existing, event); + } + this.mergeParentToolCallId( + existing, + event.parentToolCallId, + parentToolCallId, + shouldMergeDuplicate, + ); + return existing.parentToolCallId; + } + + private shouldMergeDuplicateToolCall( + existing: BufferedToolCall, + event: BufferToolCallEvent, + ): boolean { + if (existing.toolName === "Agent" && event.toolName === "Agent") return true; + const rawToolInput = existing._rawToolInput ?? {}; + return existing.status === "running" && ( + Object.keys(rawToolInput).length === 0 || existing.toolName !== event.toolName + ); + } + + private mergeDuplicateToolCall(existing: BufferedToolCall, event: BufferToolCallEvent): void { + existing.toolName = event.toolName; + const mergedToolInput = { + ...existing._rawToolInput, + ...event.toolInput, + }; + existing._rawToolInput = mergedToolInput; + if (event.toolName === "Agent") { + existing._subagentPresentation = mergeSubagentPresentation( + existing._subagentPresentation, + event.subagentPresentation + ?? createSubagentPresentation(mergedToolInput, event.toolCallId), + event.toolCallId, + ); + this.applyAgentPresentation(existing, mergedToolInput, event.toolCallId); + } + } + + private mergeParentToolCallId( + existing: BufferedToolCall, + providerParentToolCallId: string | undefined, + inferredParentToolCallId: string | undefined, + mergedDuplicate: boolean, + ): void { + if (providerParentToolCallId) { + existing.parentToolCallId = providerParentToolCallId; + return; + } + if (mergedDuplicate && inferredParentToolCallId && !existing.parentToolCallId) { + existing.parentToolCallId = inferredParentToolCallId; + } + } + + private applyAgentPresentation( + toolCall: BufferedToolCall, + rawToolInput: Record, + toolCallId: string, + ): void { + const presentation = toolCall._subagentPresentation + ?? createSubagentPresentation(rawToolInput, toolCallId); + toolCall.displayName = presentation.hasExplicitIdentity ? presentation.displayName : undefined; + toolCall.providerAgentKey = presentation.providerAgentKey; + toolCall.subagentIdentityKey = persistedSubagentIdentityKey(presentation) ?? toolCall.subagentIdentityKey; + toolCall.subagentProviderName = this.subagentProviderName(presentation); + Object.assign(toolCall, persistedSubagentMetadata(rawToolInput)); + if (toolCall.messageId && toolCall.subagentIdentityKey) { + this.emitEffect({ + kind: "update-subagent-identity", + toolCallId: toolCall.toolCallId!, + messageId: toolCall.messageId, + identityKey: toolCall.subagentIdentityKey, + }); + } + toolCall.model = presentation.model; + toolCall.reasoningEffort = presentation.reasoningEffort; + } + + private addBufferedToolCall( + threadId: string, + buffer: BufferedToolCall[], + stack: string[], + event: BufferToolCallEvent, + parentToolCallId: string | undefined, + ): string | undefined { + const sortOrder = this.nextSortOrder(threadId); + if (event.toolName === "Agent") { + stack.push(event.toolCallId); + this.agentCallStack.set(threadId, stack); + } + const presentation = this.subagentPresentation(event); + buffer.push(this.createBufferedToolCall(event, presentation, sortOrder, parentToolCallId)); + this.turnToolCalls.set(threadId, buffer); + return parentToolCallId; + } + + private subagentPresentation(event: BufferToolCallEvent): SubagentPresentation | undefined { + if (event.toolName !== "Agent") return undefined; + return event.subagentPresentation ?? createSubagentPresentation(event.toolInput, event.toolCallId); + } + + private createBufferedToolCall( + event: BufferToolCallEvent, + presentation: SubagentPresentation | undefined, + sortOrder: number, + parentToolCallId: string | undefined, + ): BufferedToolCall { + const isAgent = event.toolName === "Agent"; + return { + toolCallId: event.toolCallId, + messageId: "", + toolName: event.toolName, + displayName: presentation?.hasExplicitIdentity ? presentation.displayName : undefined, + providerAgentKey: presentation?.providerAgentKey, + subagentIdentityKey: isAgent ? persistedSubagentIdentityKey(presentation) : undefined, + subagentProviderName: this.subagentProviderName(presentation), + ...(isAgent ? persistedSubagentMetadata(event.toolInput) : {}), + model: presentation?.model, + reasoningEffort: presentation?.reasoningEffort, + inputSummary: "", + outputSummary: "", + status: "running", + startedAt: new Date().toISOString(), + sortOrder, + parentToolCallId, + _rawToolInput: event.toolInput, + ...(presentation ? { _subagentPresentation: presentation } : {}), + }; + } + + private subagentProviderName(presentation: SubagentPresentation | undefined): string | undefined { + if (presentation?.detail.kind !== "transcript-unavailable") return undefined; + return presentation.detail.providerName; + } + + /** + * Update a buffered tool call with its output when its result arrives, and + * pop the call from the `agentCallStack` if it was an Agent (Trap 2 pop site). + */ + updateBufferedToolCallOutput( + threadId: string, + toolCallId: string, + output: string, + isError: boolean, + toolInput?: Record, + outputMeta?: { + outputTruncated?: boolean; + outputTotalBytes?: number; + outputArtifactPath?: string; + exitCode?: number; + }, + subagentPresentation?: SubagentPresentation, + ): void { + this.removeAgentFromStack(threadId, toolCallId); + const toolCall = this.latestBufferedToolCall(threadId, toolCallId); + if (!toolCall) return; + this.applyToolCallOutput(toolCall, output, isError, outputMeta); + this.mergeToolCallInput(toolCall, toolInput); + if (toolCall.toolName === "Agent") { + const incomingPresentation = subagentPresentation + ?? createSubagentPresentation(toolCall._rawToolInput ?? {}, toolCallId); + toolCall._subagentPresentation = mergeSubagentPresentation( + toolCall._subagentPresentation, + incomingPresentation, + toolCallId, + ); + this.applyAgentPresentation(toolCall, toolCall._rawToolInput ?? {}, toolCallId); + } + } + + private removeAgentFromStack(threadId: string, toolCallId: string): void { + const stack = this.agentCallStack.get(threadId) ?? []; + const stackIndex = stack.indexOf(toolCallId); + if (stackIndex < 0) return; + stack.splice(stackIndex, 1); + this.agentCallStack.set(threadId, stack); + logger.debug("updateBufferedToolCallOutput: popped Agent from stack", { + threadId, + toolCallId, + remainingDepth: stack.length, + }); + } + + private latestBufferedToolCall(threadId: string, toolCallId: string): BufferedToolCall | undefined { + const buffer = this.turnToolCalls.get(threadId) ?? []; + for (let index = buffer.length - 1; index >= 0; index -= 1) { + if (buffer[index].toolCallId === toolCallId) return buffer[index]; + } + return undefined; + } + + private applyToolCallOutput( + toolCall: BufferedToolCall, + output: string, + isError: boolean, + outputMeta: { + outputTruncated?: boolean; + outputTotalBytes?: number; + outputArtifactPath?: string; + exitCode?: number; + } | undefined, + ): void { + const outputLimit = resolveBrowserNarrativeTool(toolCall.toolName) ? 4_000 : 500; + toolCall.outputSummary = output.slice(0, outputLimit); + toolCall.outputTruncated = outputMeta?.outputTruncated === true; + this.applyOutputMetadata(toolCall, outputMeta); + toolCall.status = isError ? "failed" : "completed"; + toolCall.completedAt = new Date().toISOString(); + } + + private applyOutputMetadata( + toolCall: BufferedToolCall, + outputMeta: { + outputTotalBytes?: number; + outputArtifactPath?: string; + exitCode?: number; + } | undefined, + ): void { + delete toolCall.outputTotalBytes; + delete toolCall.outputArtifactPath; + delete toolCall.exitCode; + if (outputMeta?.outputTotalBytes != null) toolCall.outputTotalBytes = outputMeta.outputTotalBytes; + if (outputMeta?.outputArtifactPath) toolCall.outputArtifactPath = outputMeta.outputArtifactPath; + if (outputMeta?.exitCode !== undefined) toolCall.exitCode = outputMeta.exitCode; + } + + private mergeToolCallInput(toolCall: BufferedToolCall, toolInput: Record | undefined): void { + if (!toolInput || Object.keys(toolInput).length === 0) return; + toolCall._rawToolInput = { + ...toolCall._rawToolInput, + ...toolInput, + }; + } + + /** + * Clear the whole Agent stack when a final `Message` event arrives — the turn + * is over and any Agent calls still on the stack are implicitly done (Trap 2 + * end-of-turn clear). No-ops when the stack is already empty. + */ + clearAgentStackOnMessage(threadId: string): void { + const stack = this.agentCallStack.get(threadId); + if (stack && stack.length > 0) { + stack.length = 0; + } + } + + /** Snapshot of the thread's buffered tool calls (read-only inspection). */ + getBufferedToolCalls(threadId: string): readonly BufferedToolCall[] { + return this.turnToolCalls.get(threadId) ?? []; + } + + /** + * Snapshot the visible structured narrative for an unfinished turn without + * retaining provider protocol traffic or private raw tool input. + */ + recoverySnapshot(threadId: string): ParentNarrativeRecoveryItem[] { + const snapshot: ParentNarrativeRecoveryItem[] = []; + let bytes = 0; + bytes = this.appendBufferedToolCallRecoveryItems(snapshot, bytes, threadId); + for (const thought of this.turnThoughts.get(threadId) ?? []) { + bytes = this.appendRecoverySnapshotItem( + snapshot, + bytes, + this.thoughtRecoveryItem(thought), + threadId, + ); + } + const openThought = this.turnOpenThought.get(threadId); + if (openThought) { + bytes = this.appendRecoverySnapshotItem( + snapshot, + bytes, + this.openThoughtRecoveryItem(openThought), + threadId, + ); + } + for (const hook of this.turnHooks.get(threadId) ?? []) { + bytes = this.appendRecoverySnapshotItem( + snapshot, + bytes, + this.hookRecoveryItem(hook), + threadId, + ); + } + for (const hook of this.turnOpenHooks.get(threadId)?.values() ?? []) { + bytes = this.appendRecoverySnapshotItem( + snapshot, + bytes, + this.openHookRecoveryItem(hook), + threadId, + ); + } + return this.sortRecoverySnapshot(snapshot); + } + + private appendBufferedToolCallRecoveryItems( + snapshot: ParentNarrativeRecoveryItem[], + bytes: number, + threadId: string, + ): number { + for (const toolCall of this.turnToolCalls.get(threadId) ?? []) { + bytes = this.appendRecoverySnapshotItem( + snapshot, + bytes, + this.toolCallRecoveryItem(toolCall), + threadId, + ); + } + return bytes; + } + + private appendRecoverySnapshotItem( + snapshot: ParentNarrativeRecoveryItem[], + bytes: number, + item: ParentNarrativeRecoveryItem, + threadId: string, + ): number { + const nextBytes = Buffer.byteLength(JSON.stringify(item), "utf8"); + this.assertRecoveryItemFitsWriteBatch(nextBytes, threadId); + const retainedBytes = bytes + nextBytes; + assertActiveTurnRecoveryRetention(snapshot.length + 1, retainedBytes); + snapshot.push(item); + return retainedBytes; + } + + private toolCallRecoveryItem(toolCall: BufferedToolCall): ParentNarrativeRecoveryItem { + return { + kind: "toolCall", + record: { + ...this.toolCallRecoveryIdentityFields(toolCall), + ...this.toolCallRecoveryPresentationFields(toolCall), + ...this.toolCallRecoveryOutputFields(toolCall), + ...this.toolCallRecoveryStateFields(toolCall), + }, + }; + } + + private toolCallRecoveryIdentityFields(toolCall: BufferedToolCall) { + return { + id: this.requireRecoveryString(toolCall.toolCallId, "tool call id"), + message_id: this.requireRecoveryString(toolCall.messageId, "tool call message id"), + parent_tool_call_id: toolCall.parentToolCallId ?? null, + tool_name: toolCall.toolName, + display_name: toolCall.displayName ?? null, + provider_agent_key: toolCall.providerAgentKey ?? null, + subagent_identity_key: toolCall.subagentIdentityKey ?? null, + subagent_provider_name: toolCall.subagentProviderName ?? null, + }; + } + + private toolCallRecoveryPresentationFields(toolCall: BufferedToolCall) { + return { + subagent_prompt: toolCall.subagentPrompt ?? null, + subagent_type: toolCall.subagentType ?? null, + subagent_agent_id: toolCall.subagentAgentId ?? null, + subagent_duration_ms: toolCall.subagentDurationMs ?? null, + model: toolCall.model ?? null, + reasoning_effort: toolCall.reasoningEffort ?? null, + }; + } + + private toolCallRecoveryOutputFields(toolCall: BufferedToolCall) { + return { + input_summary: this.toolCallRecoveryInputSummary(toolCall), + output_summary: toolCall.outputSummary, + ...(toolCall.outputTruncated ? { output_truncated: 1 } : {}), + output_total_bytes: toolCall.outputTotalBytes ?? null, + output_artifact_path: toolCall.outputArtifactPath ?? null, + exit_code: toolCall.exitCode ?? null, + }; + } + + private toolCallRecoveryInputSummary(toolCall: BufferedToolCall): string { + if (toolCall.inputSummary) return toolCall.inputSummary; + return this.summarizeInput(toolCall.toolName, toolCall._rawToolInput ?? {}); + } + + private toolCallRecoveryStateFields(toolCall: BufferedToolCall) { + return { + status: toolCall.status, + started_at: this.requireRecoveryString(toolCall.startedAt, "tool call start time"), + completed_at: toolCall.completedAt ?? null, + sort_order: toolCall.sortOrder, + }; + } + + private thoughtRecoveryItem(thought: CreateThoughtSegmentInput): ParentNarrativeRecoveryItem { + return { + kind: "narrationSegment", + record: { + id: this.requireRecoveryString(thought.id, "thought id"), + message_id: this.requireRecoveryString(thought.messageId, "thought message id"), + text: this.requireRecoveryString(thought.text, "thought text"), + started_at: this.requireRecoveryString(thought.startedAt, "thought start time"), + ended_at: thought.endedAt ?? null, + sort_order: thought.sortOrder, + ...(thought.isFinalResponse ? { is_final_response: thought.isFinalResponse } : {}), + }, + }; + } + + private openThoughtRecoveryItem(openThought: OpenThought): ParentNarrativeRecoveryItem { + return { + kind: "narrationSegment", + record: { + id: openThought.id, + message_id: "", + text: openThought.text, + started_at: openThought.startedAt, + ended_at: null, + sort_order: openThought.sortOrder, + }, + }; + } + + private hookRecoveryItem(hook: CreateHookExecutionInput): ParentNarrativeRecoveryItem { + return { + kind: "hook", + record: { + id: this.requireRecoveryString(hook.id, "hook id"), + message_id: this.requireRecoveryString(hook.messageId, "hook message id"), + hook_name: hook.hookName, + tool_name: hook.toolName, + phase: hook.phase, + payload: hook.payload, + duration_ms: hook.durationMs ?? null, + did_block: hook.didBlock, + started_at: this.requireRecoveryString(hook.startedAt, "hook start time"), + ended_at: hook.endedAt ?? null, + sort_order: hook.sortOrder, + }, + }; + } + + private openHookRecoveryItem(hook: OpenHook): ParentNarrativeRecoveryItem { + return { + kind: "hook", + record: { + id: hook.id, + message_id: "", + hook_name: hook.hookName, + tool_name: hook.toolName, + phase: hook.phase, + payload: hook.payload, + duration_ms: null, + did_block: false, + started_at: hook.startedAt, + ended_at: null, + sort_order: hook.sortOrder, + }, + }; + } + + private sortRecoverySnapshot( + snapshot: ParentNarrativeRecoveryItem[], + ): ParentNarrativeRecoveryItem[] { + return snapshot.sort((left, right) => ( + left.record.sort_order - right.record.sort_order || left.record.id.localeCompare(right.record.id) + )); + } + + /** Stage narration for recovery without mutating the active turn buffer. */ + stageNarrationSegment(threadId: string, text: string): StagedNarrationSegment | null { + const open = this.turnOpenThought.get(threadId); + const endedAt = new Date().toISOString(); + if (open) { + return { + id: open.id, + text: `${open.text}${text}`, + startedAt: open.startedAt, + endedAt, + sortOrder: open.sortOrder, + openThoughtId: open.id, + }; + } + if (!text) return null; + return { + id: NodeCrypto.randomUUID(), + text, + startedAt: endedAt, + endedAt, + sortOrder: this.turnSortCounters.get(threadId) ?? 0, + }; + } + + /** Add staged narration to a recovery snapshot without mutating the active turn buffer. */ + recoverySnapshotWithStagedNarration( + threadId: string, + staged: StagedNarrationSegment, + ): ParentNarrativeRecoveryItem[] { + const snapshot = this.recoverySnapshot(threadId).filter((item) => ( + item.kind !== "narrationSegment" || item.record.id !== staged.id + )); + snapshot.push({ + kind: "narrationSegment", + record: { + id: staged.id, + message_id: "", + text: staged.text, + started_at: staged.startedAt, + ended_at: staged.endedAt, + sort_order: staged.sortOrder, + }, + }); + let bytes = 0; + for (const [index, item] of snapshot.entries()) { + const itemBytes = Buffer.byteLength(JSON.stringify(item), "utf8"); + this.assertRecoveryItemFitsWriteBatch(itemBytes, threadId); + bytes += itemBytes; + assertActiveTurnRecoveryRetention(index + 1, bytes); + } + return snapshot.sort((left, right) => ( + left.record.sort_order - right.record.sort_order || left.record.id.localeCompare(right.record.id) + )); + } + + /** Apply a narration record only after its recovery projection is durable. */ + applyStagedNarrationSegment(threadId: string, staged: StagedNarrationSegment): void { + if (staged.openThoughtId) { + const open = this.turnOpenThought.get(threadId); + if (!open || open.id !== staged.openThoughtId) { + throw new Error(`Staged narration no longer matches the open thought: ${staged.id}`); + } + this.turnOpenThought.set(threadId, null); + } else { + const nextSortOrder = this.turnSortCounters.get(threadId) ?? 0; + if (nextSortOrder <= staged.sortOrder) { + this.turnSortCounters.set(threadId, staged.sortOrder + 1); + } + } + const thoughts = this.turnThoughts.get(threadId) ?? []; + thoughts.push({ + id: staged.id, + messageId: "", + text: staged.text, + startedAt: staged.startedAt, + endedAt: staged.endedAt, + sortOrder: staged.sortOrder, + }); + this.turnThoughts.set(threadId, thoughts); + } + + private requireRecoveryString(value: string | undefined, field: string): string { + if (value === undefined) throw new Error(`Narrative recovery is missing ${field}`); + return value; + } + + private assertRecoveryItemFitsWriteBatch(byteLength: number, threadId: string): void { + if (byteLength > ACTIVE_TURN_WRITE_BATCH_LIMITS.maxBytes) { + throw new Error(`Parent narrative recovery item exceeds the active-turn byte limit: ${threadId}`); + } + } + + /** + * True when the current turn has buffered at least one narrative contributor — + * a tool call, a narration segment (open or closed), or a hook (open or + * closed). Feeds the {@link TurnFinalizer.hasRecordableActivity} predicate so + * a turn with narrative but no assistant body still earns a persisted row. + */ + hasBufferedNarrative(threadId: string): boolean { + return this.narrativeBufferStates(threadId).some(Boolean); + } + + private narrativeBufferStates(threadId: string): boolean[] { + return [ + (this.turnToolCalls.get(threadId)?.length ?? 0) > 0, + Boolean(this.turnOpenThought.get(threadId)), + (this.turnThoughts.get(threadId)?.length ?? 0) > 0, + (this.turnOpenHooks.get(threadId)?.size ?? 0) > 0, + (this.turnHooks.get(threadId)?.length ?? 0) > 0, + ]; + } + + /** + * Record an in-flight hook execution (HookStarted). The caller supplies the + * already-allocated sort order (the late-hook path in AgentService allocates + * it before deciding routing). Returns the generated row id. + */ + openHook( + threadId: string, + hook: { hookName: string; toolName: string | null; phase: string; payload: string; sortOrder: number }, + ): string { + const map = this.turnOpenHooks.get(threadId) ?? new Map(); + const id = NodeCrypto.randomUUID(); + map.set(hook.hookName, { + id, + hookName: hook.hookName, + toolName: hook.toolName, + phase: hook.phase, + payload: hook.payload, + startedAt: new Date().toISOString(), + sortOrder: hook.sortOrder, + }); + this.turnOpenHooks.set(threadId, map); + return id; + } + + /** Look up (without removing) an open hook by name. */ + peekOpenHook(threadId: string, hookName: string): OpenHook | undefined { + return this.turnOpenHooks.get(threadId)?.get(hookName); + } + + /** Remove an open hook by name (after it has been completed or flushed). */ + removeOpenHook(threadId: string, hookName: string): void { + this.turnOpenHooks.get(threadId)?.delete(hookName); + } + + /** Push a completed hook execution onto the closed-hooks list for persistence. */ + pushClosedHook(threadId: string, hook: CreateHookExecutionInput): void { + const list = this.turnHooks.get(threadId) ?? []; + list.push(hook); + this.turnHooks.set(threadId, list); + } + + /** Settle one turn's buffered records into data-only persistence rows. */ + prepareNarrativePersistence( + threadId: string, + messageId: string, + messageContent: string, + outcome: TurnOutcome, + ): PreparedNarrativePersistence { + const toolCalls = this.prepareToolCallsForPersistence(threadId, messageId, outcome); + this.closeOpenThought(threadId); + this.closeOpenHooksForPersistence(threadId); + const thoughts = this.prepareThoughtsForPersistence(threadId, messageId, messageContent); + const hooks = this.prepareHooksForPersistence(threadId, messageId); + return { toolCalls, thoughts, hooks }; + } + + private prepareToolCallsForPersistence( + threadId: string, + messageId: string, + outcome: TurnOutcome, + ): BufferedToolCall[] { + const toolCalls = this.turnToolCalls.get(threadId) ?? []; + const settledAt = new Date().toISOString(); + for (const toolCall of toolCalls) { + toolCall.toolCallId ??= NodeCrypto.randomUUID(); + this.settleRunningToolCall(toolCall, outcome, settledAt); + toolCall.messageId = messageId; + this.prepareToolCallInput(toolCall); + } + return toolCalls; + } + + private settleRunningToolCall( + toolCall: BufferedToolCall, + outcome: TurnOutcome, + settledAt: string, + ): void { + if (toolCall.status !== "running") return; + toolCall.status = this.settledToolCallStatus(outcome); + toolCall.completedAt = settledAt; + } + + private settledToolCallStatus(outcome: TurnOutcome): BufferedToolCall["status"] { + if (outcome === "errored" || outcome === "interrupted") return "failed"; + if (outcome === "cancelled") return "cancelled"; + return "completed"; + } + + private prepareToolCallInput(toolCall: BufferedToolCall): void { + const rawToolInput = toolCall._rawToolInput; + if (toolCall.inputSummary || !rawToolInput) return; + if (toolCall.toolName === "Agent") { + this.applyAgentPersistenceMetadata(toolCall, rawToolInput); + } + toolCall.inputSummary = this.summarizeInput(toolCall.toolName, rawToolInput); + delete toolCall._rawToolInput; + } + + private applyAgentPersistenceMetadata( + toolCall: BufferedToolCall, + rawToolInput: Record, + ): void { + const presentation = createSubagentPresentation(rawToolInput, toolCall.toolCallId!); + const persistedPresentation = toolCall._subagentPresentation ?? presentation; + toolCall.displayName = persistedPresentation.hasExplicitIdentity + ? persistedPresentation.displayName + : undefined; + toolCall.providerAgentKey = persistedPresentation.providerAgentKey; + toolCall.subagentIdentityKey = persistedSubagentIdentityKey(persistedPresentation) + ?? toolCall.subagentIdentityKey; + toolCall.subagentProviderName = this.subagentProviderName(persistedPresentation); + Object.assign(toolCall, persistedSubagentMetadata(rawToolInput)); + toolCall.model = persistedPresentation.model; + toolCall.reasoningEffort = persistedPresentation.reasoningEffort; + } + + private closeOpenHooksForPersistence(threadId: string): void { + const openHookMap = this.turnOpenHooks.get(threadId); + if (!openHookMap || openHookMap.size === 0) return; + const list = this.turnHooks.get(threadId) ?? []; + const endedAt = new Date().toISOString(); + for (const open of openHookMap.values()) { + list.push({ + id: open.id, + messageId: "", + hookName: open.hookName, + toolName: open.toolName, + phase: open.phase, + payload: open.payload, + durationMs: Date.parse(endedAt) - Date.parse(open.startedAt), + didBlock: false, + startedAt: open.startedAt, + endedAt, + sortOrder: open.sortOrder, + }); + } + this.turnHooks.set(threadId, list); + openHookMap.clear(); + } + + private prepareThoughtsForPersistence( + threadId: string, + messageId: string, + messageContent: string, + ): CreateThoughtSegmentInput[] { + const bufferedThoughts = this.turnThoughts.get(threadId) ?? []; + for (const thought of bufferedThoughts) thought.id ??= NodeCrypto.randomUUID(); + const thoughts = bufferedThoughts.map((thought) => ({ ...thought, messageId })); + const message = messageContent.trim(); + if (message.length > 0 && thoughts.length > 0) { + this.markFinalResponseThoughts(thoughts, message); + } + return thoughts; + } + + private markFinalResponseThoughts( + thoughts: CreateThoughtSegmentInput[], + message: string, + ): void { + const maxSortOrder = Math.max(...thoughts.map((thought) => thought.sortOrder)); + for (const thought of thoughts) { + const text = thought.text.trim(); + if (text.length > 0 && this.isFinalResponseThought(thought, text, message, maxSortOrder)) { + thought.isFinalResponse = 1; + } + } + } + + private isFinalResponseThought( + thought: CreateThoughtSegmentInput, + text: string, + message: string, + maxSortOrder: number, + ): boolean { + if (text === message) return true; + return thought.sortOrder === maxSortOrder + && (thought.isFinalResponse === 1 || message.endsWith(text)); + } + + private prepareHooksForPersistence( + threadId: string, + messageId: string, + ): CreateHookExecutionInput[] { + const bufferedHooks = this.turnHooks.get(threadId) ?? []; + for (const hook of bufferedHooks) hook.id ??= NodeCrypto.randomUUID(); + return bufferedHooks.map((hook) => ({ ...hook, messageId })); + } + + /** + * Clear the per-turn narrative buffers this store owns. The sort counter and + * Agent stack are intentionally NOT cleared here — they are reset in the + * TurnStarted handler so late hooks that arrive after this point can still + * increment the completed turn's counter (mirrors the old clearTurnState). + */ + clearTurn(threadId: string): void { + this.turnToolCalls.delete(threadId); + this.turnOpenThought.delete(threadId); + this.turnThoughts.delete(threadId); + this.turnOpenHooks.delete(threadId); + this.turnHooks.delete(threadId); + } + + /** Generate a human-readable summary of tool input. */ + private summarizeInput(toolName: string, input: Record): string { + if (resolveBrowserNarrativeTool(toolName)) return JSON.stringify(input).slice(0, 4_000); + return (NARRATIVE_INPUT_SUMMARIZERS[toolName.toLowerCase()] ?? genericInputSummary)(input); + } +} diff --git a/docs/internals/narrative-pipeline.md b/docs/internals/narrative-pipeline.md index 93299334f..227437eca 100644 --- a/docs/internals/narrative-pipeline.md +++ b/docs/internals/narrative-pipeline.md @@ -38,9 +38,10 @@ the specific traps we hit so the next person doesn't trip on them. If you are about to touch any of these files, **read this first**: - `apps/server/src/features/providers/` (runtime event source and provider projection) -- `apps/server/src/features/agents/conversation/narrative/narrative-store.ts` (write seam: enrichment + - classification + persistence; owns the per-turn buffers and the - `agentCallStack`. Also owns the read seam, `load`.) +- `apps/server/src/features/agents/conversation/narrative/narrative-turn-state.ts` (per-turn + buffers, `agentCallStack`, enrichment, classification, and recovery snapshot) +- `apps/server/src/features/agents/conversation/narrative/narrative-store.ts` (SQLite + persistence, data-only effects from turn state, and the read seam, `load`) - `apps/server/src/features/agents/orchestration/agent-service.ts` (turn orchestration; delegates the write seam to NarrativeStore and retains turn-level concerns: turn snapshots, `turn.persisted` broadcast, and late-hook flushing) @@ -136,7 +137,7 @@ Provider ingress queues accepted results fairly and selects a provider adapter Adapter forwards a provider-neutral AgentEvent or consumes private provider work │ ▼ -Turn event pipeline ToolUse handler → narrative-store.ts bufferToolCall +Turn event pipeline ToolUse handler → narrative-turn-state.ts bufferToolCall (writes to the per-turn tool-call buffer for later persist) │ ▼ @@ -262,7 +263,7 @@ SDK doesn't surface this field (older paths, edge cases). - `index.ts` checks `if (event.parentToolCallId)` first — if set, leave it. Only falls back to `narrativeStore.getCurrentParentToolCallId` when the SDK omitted it. -- `narrative-store.ts bufferToolCall` does the same dance for the persistence +- `narrative-turn-state.ts bufferToolCall` does the same dance for the persistence buffer: SDK value wins, stack is a fallback. (AgentService's `bufferToolCall` is a thin wrapper that delegates here, then persists TodoWrite task state.) @@ -300,7 +301,7 @@ child `toolUse` events to lose their fallback parent ID. 3. When the session ends. **Never on `textDelta`. Never on streaming events.** The stack now lives on -`narrative-store.ts`; its `openOrExtendThought` (the textDelta path) never +`narrative-turn-state.ts`; its `openOrExtendThought` (the textDelta path) never touches `agentCallStack`. See the explanatory comment in the AgentService `TextDelta` handler. From 1f786fa53372fd0ea6c68c8a21cea3139c01b990 Mon Sep 17 00:00:00 2001 From: "(CJ) Chukwudi Nwobodo" <142016413+chuks-qua@users.noreply.github.com> Date: Thu, 24 Sep 2026 21:25:45 +0100 Subject: [PATCH 062/136] refactor(server): capture bounded file observations before worker handoff --- .../turns/__tests__/turn-file-tracker.test.ts | 55 +++++++-- .../agents/turns/turn-file-tracker.ts | 107 ++++++++++++++++-- 2 files changed, 142 insertions(+), 20 deletions(-) diff --git a/apps/server/src/features/agents/turns/__tests__/turn-file-tracker.test.ts b/apps/server/src/features/agents/turns/__tests__/turn-file-tracker.test.ts index 7e608d470..86fe796f0 100644 --- a/apps/server/src/features/agents/turns/__tests__/turn-file-tracker.test.ts +++ b/apps/server/src/features/agents/turns/__tests__/turn-file-tracker.test.ts @@ -90,16 +90,13 @@ describe("TurnFileTracker", () => { TEST_PLATFORM, ); tracker.beginTurn("t", root, "unavailable-ref"); - const pendingStarts = [tracker.observeToolUse( - "t", - "file-child", - "file_change", - { changes: [{ path: "tracked.txt", kind: "edit" }] }, - )]; + const event = { threadId: "t", toolCallId: "file-child", toolName: "file_change", toolInput: { changes: [{ path: "tracked.txt", kind: "edit" }] } }; + const captured = tracker.captureToolUseObservation(event.threadId, event.toolCallId, event.toolName, event.toolInput); + if (!captured) throw new Error("Expected a captured baseline"); + expect(structuredClone(captured)).toEqual(captured); await NodeFSPromises.writeFile(trackedPath, "after\nextra\n"); - expect(pendingStarts).toHaveLength(1); + expect(await tracker.observeCapturedToolUse(event, structuredClone(captured))).toBe(true); await tracker.observeToolResult("t", "file-child"); - await Promise.all(pendingStarts); const summary = await tracker.finalizeTurn("t"); expect(summary).toMatchObject({ fileCount: 1, additions: 2, deletions: 1 }); @@ -111,6 +108,31 @@ describe("TurnFileTracker", () => { expect(updates.at(-1)).toEqual(summary); }); + it("rejects a captured baseline from another tool, root, or turn generation", async () => { + const root = await tempDir("mcode-stale-observation-"); + const path = NodePath.join(root, "tracked.txt"); + await NodeFSPromises.writeFile(path, "before\n"); + const tracker = new TurnFileTracker(async () => ({ kind: "unavailable" }), () => {}, TEST_PLATFORM); + const event = { threadId: "t", toolCallId: "old-tool", toolName: "Edit", toolInput: { file_path: "tracked.txt" } }; + const firstGeneration = tracker.beginTurn("t", root, null); + const captured = tracker.captureToolUseObservation(event.threadId, event.toolCallId, event.toolName, event.toolInput); + if (!captured) throw new Error("Expected a captured baseline"); + expect(await tracker.observeCapturedToolUse({ ...event, toolCallId: "other-tool" }, captured)).toBe(false); + expect(await tracker.observeCapturedToolUse({ ...event, toolName: "Write" }, captured)).toBe(false); + expect(await tracker.observeCapturedToolUse(event, { ...captured, canonicalRoot: "other-root" })).toBe(false); + expect(await tracker.observeCapturedToolUse({ ...event, toolInput: { file_path: "other.txt" } }, captured)).toBe(false); + const replacement = new TurnFileTracker(async () => ({ kind: "unavailable" }), () => {}, TEST_PLATFORM); + expect(replacement.beginTurn("t", root, null)).toBe(firstGeneration); + expect(await replacement.observeCapturedToolUse(event, captured)).toBe(false); + + const nextGeneration = tracker.beginTurn("t", root, null); + await NodeFSPromises.writeFile(path, "after\n"); + expect(await tracker.observeCapturedToolUse(event, structuredClone(captured))).toBe(false); + await tracker.observeToolResult("t", event.toolCallId); + expect(await tracker.finalizeTurn("t", firstGeneration)).toMatchObject({ fileCount: 0 }); + expect(await tracker.finalizeTurn("t", nextGeneration)).toMatchObject({ fileCount: 0 }); + }); + it("classifies added, edited, and removed files with net line totals", async () => { const root = await tempDir("mcode-file-effects-"); await NodeFSPromises.writeFile(NodePath.join(root, "edited.txt"), "one\ntwo"); @@ -386,6 +408,23 @@ describe("TurnFileTracker", () => { expect(synchronousDurationMs).toBeLessThan(250); }); + it("keeps captured pre-edit observations within the path and byte budgets", async () => { + const root = await tempDir("mcode-captured-observation-budget-"); + const tracker = trackerWithBaseline({}, []); + tracker.beginTurn("t", root, null); + const changes = Array.from({ length: 5 }, (_, index) => ({ path: `file-${index}.txt`, kind: "edit" })); + for (const change of changes) { + await NodeFSPromises.writeFile(NodePath.join(root, change.path), "x".repeat(350_000)); + } + const captured = tracker.captureToolUseObservation("t", "bulk", "file_change", { changes }); + if (!captured) throw new Error("Expected bounded observations"); + expect(captured.observations).toHaveLength(5); + expect(captured.observations.filter(Boolean)).toHaveLength(4); + const capturedBytes = captured.observations.reduce((total, observation) => + total + Buffer.byteLength(observation?.baseline?.text ?? "", "utf8"), 0); + expect(capturedBytes).toBeLessThanOrEqual(1_048_576); + }); + it("falls back to complete async observation when a rename exceeds the remaining path budget", async () => { const root = await tempDir("mcode-file-effects-mixed-budget-"); for (let index = 0; index < 3; index += 1) { diff --git a/apps/server/src/features/agents/turns/turn-file-tracker.ts b/apps/server/src/features/agents/turns/turn-file-tracker.ts index 70e9c392d..afd281655 100644 --- a/apps/server/src/features/agents/turns/turn-file-tracker.ts +++ b/apps/server/src/features/agents/turns/turn-file-tracker.ts @@ -32,10 +32,21 @@ interface MutationCandidate { } interface CandidateObservation { - resolvedPath: Pick | null; - resolvedOldPath: Pick | null; - baseline: FileState | null; - oldBaseline: FileState | null; + readonly resolvedPath: Readonly> | null; + readonly resolvedOldPath: Readonly> | null; + readonly baseline: Readonly | null; + readonly oldBaseline: Readonly | null; +} + +/** Plain, bounded pre-edit evidence that can cross a worker message boundary. */ +export interface CapturedToolUseObservation { + readonly threadId: string; + readonly toolCallId: string; + readonly generation: number; + readonly generationToken: string; + readonly canonicalRoot: string; + readonly candidateIdentity: string; + readonly observations: readonly (CandidateObservation | null)[]; } interface SyncObservationBudget { @@ -73,6 +84,7 @@ interface TrackedPath { interface TurnState { reconstructionRejected?: boolean; generation: number; + generationToken: string; cwd: string; canonicalRoot: string; baselineRef: string | null; @@ -124,6 +136,7 @@ export class TurnFileTracker { const generations = this.turns.get(threadId) ?? new Map(); generations.set(generation, { generation, + generationToken: NodeCrypto.randomUUID(), cwd, canonicalRoot, baselineRef, @@ -160,20 +173,52 @@ export class TurnFileTracker { toolName: string, toolInput: Record, ): Promise { - const candidates = extractMutationCandidates(toolName, toolInput); - if (candidates.length === 0) return Promise.resolve(); + const captured = this.captureToolUseObservation(threadId, toolCallId, toolName, toolInput); + return captured + ? this.observeCapturedToolUse({ threadId, toolCallId, toolName, toolInput }, captured).then(() => undefined) + : Promise.resolve(); + } + + /** Take the bounded filesystem baseline synchronously, before the provider may edit. */ + captureToolUseObservation( + threadId: string, + toolCallId: string, + toolName: string, + toolInput: Record, + ): CapturedToolUseObservation | null { const generation = this.currentGeneration.get(threadId); - if (generation === undefined) return Promise.resolve(); + if (generation === undefined) return null; const turn = this.getTurn(threadId, generation); - if (!turn) return Promise.resolve(); - const boundedCandidates = dedupeMutationCandidates(candidates).slice(0, MAX_TURN_FILE_EFFECTS); - const observations = this.synchronousObservations(turn, boundedCandidates); + if (!turn) return null; + const candidates = boundedMutationCandidates(toolName, toolInput); + if (candidates.length === 0) return null; + const observations = this.synchronousObservations(turn, candidates).map(freezeCandidateObservation); + return Object.freeze({ + threadId, toolCallId, generation, generationToken: turn.generationToken, + canonicalRoot: turn.canonicalRoot, + candidateIdentity: mutationCandidateIdentity(toolName, candidates), + observations: Object.freeze(observations), + }); + } + + /** Apply a captured baseline only to its original thread, tool, root, and active generation. */ + observeCapturedToolUse( + event: { readonly threadId: string; readonly toolCallId: string; readonly toolName: string; readonly toolInput: Record }, + captured: CapturedToolUseObservation, + ): Promise { + const { threadId, toolCallId, toolName, toolInput } = event; + const turn = this.getTurn(threadId); + if (!turn || !capturedMatchesTurn(captured, turn, threadId, toolCallId)) return Promise.resolve(false); + const generation = turn.generation; + const boundedCandidates = boundedMutationCandidates(toolName, toolInput); + if (boundedCandidates.length === 0 || captured.observations.length !== boundedCandidates.length + || captured.candidateIdentity !== mutationCandidateIdentity(toolName, boundedCandidates)) return Promise.resolve(false); this.generationByToolCall.set(toolGenerationKey(threadId, toolCallId), generation); return this.enqueue(threadId, async (queuedTurn) => { for (const [index, candidate] of boundedCandidates.entries()) { - await this.captureCandidate(queuedTurn, toolCallId, candidate, observations[index]); + await this.captureCandidate(queuedTurn, toolCallId, candidate, captured.observations[index] ?? undefined); } - }, generation); + }, generation).then(() => true); } /** Verify all paths attributed to a completed file tool and publish the net summary. */ @@ -674,6 +719,44 @@ function dedupeMutationCandidates(candidates: MutationCandidate[]): MutationCand return [...unique.values()]; } +function boundedMutationCandidates(toolName: string, toolInput: Record): MutationCandidate[] { + return dedupeMutationCandidates(extractMutationCandidates(toolName, toolInput)).slice(0, MAX_TURN_FILE_EFFECTS); +} + +function capturedMatchesTurn( + captured: CapturedToolUseObservation, + turn: TurnState, + threadId: string, + toolCallId: string, +): boolean { + return captured.threadId === threadId && captured.toolCallId === toolCallId + && captured.generation === turn.generation && captured.generationToken === turn.generationToken + && captured.canonicalRoot === turn.canonicalRoot; +} + +function mutationCandidateIdentity(toolName: string, candidates: readonly MutationCandidate[]): string { + const hash = NodeCrypto.createHash("sha256"); + hash.update(toolName); + for (const candidate of candidates) { + hash.update(JSON.stringify([ + candidate.path, candidate.oldPath ?? null, candidate.operationHint, + candidate.providerConfirmed === true, candidate.beforeText !== undefined, + candidate.afterText !== undefined, + ])); + } + return hash.digest("hex"); +} + +function freezeCandidateObservation(observation: CandidateObservation | undefined): CandidateObservation | null { + if (!observation) return null; + return Object.freeze({ + resolvedPath: observation.resolvedPath ? Object.freeze({ ...observation.resolvedPath }) : null, + resolvedOldPath: observation.resolvedOldPath ? Object.freeze({ ...observation.resolvedOldPath }) : null, + baseline: observation.baseline ? Object.freeze({ ...observation.baseline }) : null, + oldBaseline: observation.oldBaseline ? Object.freeze({ ...observation.oldBaseline }) : null, + }); +} + function observeCandidateSynchronously( canonicalRoot: string, candidate: MutationCandidate, From 710991618bf5cee95745b68abd4a80b301a24f87 Mon Sep 17 00:00:00 2001 From: "(CJ) Chukwudi Nwobodo" <142016413+chuks-qua@users.noreply.github.com> Date: Thu, 24 Sep 2026 21:28:07 +0100 Subject: [PATCH 063/136] fix(server): bind file observations to exact evidence --- .../turns/__tests__/turn-file-tracker.test.ts | 36 +++++++++++++++++++ .../agents/turns/turn-file-tracker.ts | 10 +++++- 2 files changed, 45 insertions(+), 1 deletion(-) diff --git a/apps/server/src/features/agents/turns/__tests__/turn-file-tracker.test.ts b/apps/server/src/features/agents/turns/__tests__/turn-file-tracker.test.ts index 86fe796f0..c03f86950 100644 --- a/apps/server/src/features/agents/turns/__tests__/turn-file-tracker.test.ts +++ b/apps/server/src/features/agents/turns/__tests__/turn-file-tracker.test.ts @@ -133,6 +133,42 @@ describe("TurnFileTracker", () => { expect(await tracker.finalizeTurn("t", nextGeneration)).toMatchObject({ fileCount: 0 }); }); + it("binds captured observations to the exact before and after evidence text", async () => { + const root = await tempDir("mcode-observation-evidence-"); + const tracker = trackerWithBaseline({}, []); + tracker.beginTurn("t", root, null); + const mutation = { path: "file.txt", kind: "edit", fullFileContent: true, beforeText: "before\n", afterText: "after\n" }; + const event = { threadId: "t", toolCallId: "edit", toolName: "Edit", toolInput: { _mcodeFileMutations: [mutation] } }; + const captured = tracker.captureToolUseObservation(event.threadId, event.toolCallId, event.toolName, event.toolInput); + if (!captured) throw new Error("Expected captured text evidence"); + + expect(await tracker.observeCapturedToolUse({ + ...event, toolInput: { _mcodeFileMutations: [{ ...mutation, beforeText: "befoXe\n" }] }, + }, captured)).toBe(false); + expect(await tracker.observeCapturedToolUse({ + ...event, toolInput: { _mcodeFileMutations: [{ ...mutation, afterText: "aftEr\n" }] }, + }, captured)).toBe(false); + }); + + it("does not accept an observation if input extraction clears its turn", async () => { + const root = await tempDir("mcode-observation-cleared-turn-"); + const tracker = trackerWithBaseline({}, []); + tracker.beginTurn("t", root, null); + const event = { threadId: "t", toolCallId: "edit", toolName: "Edit", toolInput: { file_path: "file.txt" } }; + const captured = tracker.captureToolUseObservation(event.threadId, event.toolCallId, event.toolName, event.toolInput); + if (!captured) throw new Error("Expected captured tool observation"); + const clearingInput: Record = {}; + Object.defineProperty(clearingInput, "file_path", { + get: () => { + tracker.clearTurn("t"); + return "file.txt"; + }, + }); + + expect(await tracker.observeCapturedToolUse({ ...event, toolInput: clearingInput }, captured)).toBe(false); + expect(tracker.getCurrentTurnId("t")).toBeUndefined(); + }); + it("classifies added, edited, and removed files with net line totals", async () => { const root = await tempDir("mcode-file-effects-"); await NodeFSPromises.writeFile(NodePath.join(root, "edited.txt"), "one\ntwo"); diff --git a/apps/server/src/features/agents/turns/turn-file-tracker.ts b/apps/server/src/features/agents/turns/turn-file-tracker.ts index afd281655..1c701825d 100644 --- a/apps/server/src/features/agents/turns/turn-file-tracker.ts +++ b/apps/server/src/features/agents/turns/turn-file-tracker.ts @@ -213,6 +213,7 @@ export class TurnFileTracker { const boundedCandidates = boundedMutationCandidates(toolName, toolInput); if (boundedCandidates.length === 0 || captured.observations.length !== boundedCandidates.length || captured.candidateIdentity !== mutationCandidateIdentity(toolName, boundedCandidates)) return Promise.resolve(false); + if (this.getTurn(threadId) !== turn) return Promise.resolve(false); this.generationByToolCall.set(toolGenerationKey(threadId, toolCallId), generation); return this.enqueue(threadId, async (queuedTurn) => { for (const [index, candidate] of boundedCandidates.entries()) { @@ -736,13 +737,20 @@ function capturedMatchesTurn( function mutationCandidateIdentity(toolName: string, candidates: readonly MutationCandidate[]): string { const hash = NodeCrypto.createHash("sha256"); - hash.update(toolName); + hash.update(JSON.stringify([toolName])); for (const candidate of candidates) { hash.update(JSON.stringify([ candidate.path, candidate.oldPath ?? null, candidate.operationHint, candidate.providerConfirmed === true, candidate.beforeText !== undefined, candidate.afterText !== undefined, ])); + for (const text of [candidate.beforeText, candidate.afterText]) { + if (text === undefined) hash.update("u"); + else { + hash.update(`t${text.length}:`); + hash.update(text, "utf16le"); + } + } } return hash.digest("hex"); } From 44623138338f5bb1fa69ac3c0105e5916e51c4d6 Mon Sep 17 00:00:00 2001 From: "(CJ) Chukwudi Nwobodo" <142016413+chuks-qua@users.noreply.github.com> Date: Thu, 24 Sep 2026 21:30:38 +0100 Subject: [PATCH 064/136] refactor(server): isolate assistant execution decisions from finalizer --- .../assistant-execution-state.test.ts | 64 +++++++++++++ .../agents/turns/assistant-execution-state.ts | 95 +++++++++++++++++++ .../features/agents/turns/turn-finalizer.ts | 94 ++++-------------- 3 files changed, 179 insertions(+), 74 deletions(-) create mode 100644 apps/server/src/features/agents/turns/__tests__/assistant-execution-state.test.ts create mode 100644 apps/server/src/features/agents/turns/assistant-execution-state.ts diff --git a/apps/server/src/features/agents/turns/__tests__/assistant-execution-state.test.ts b/apps/server/src/features/agents/turns/__tests__/assistant-execution-state.test.ts new file mode 100644 index 000000000..03bbb8057 --- /dev/null +++ b/apps/server/src/features/agents/turns/__tests__/assistant-execution-state.test.ts @@ -0,0 +1,64 @@ +import { describe, expect, it } from "vitest"; + +import { AssistantExecutionState } from "../assistant-execution-state.js"; + +describe("AssistantExecutionState", () => { + it("accumulates final text and clears it at a message boundary", () => { + const state = new AssistantExecutionState(); + state.appendStreamingText(" partial"); + state.appendStreamingText(" answer "); + + expect(state.getStreamingText()).toBe(" partial answer "); + expect(state.materializationInput("fallback-model")).toEqual({ + content: "partial answer", + model: "fallback-model", + attachments: [], + fromProvider: false, + }); + + state.resetStreamingText(); + expect(state.getStreamingText()).toBe(""); + expect(state.hasBufferedBody()).toBe(false); + }); + + it("uses the provider message body and merges generated attachments by id", () => { + const state = new AssistantExecutionState(); + const image = { id: "image", name: "image.png", mimeType: "image/png", sizeBytes: 12 }; + const updatedImage = { ...image, sizeBytes: 20 }; + const file = { id: "file", name: "notes.txt", mimeType: "text/plain", sizeBytes: 8 }; + state.appendStreamingText("draft text"); + state.bufferAttachments([image]); + state.bufferBody("final answer", "provider-model"); + state.bufferAttachments([updatedImage, file]); + + const input = state.materializationInput("fallback-model"); + expect(input).toEqual({ + content: "final answer", + model: "provider-model", + attachments: [updatedImage, file], + fromProvider: true, + }); + expect(structuredClone(input)).toEqual(input); + + state.commitMaterialization(); + expect(state.hasMaterialized()).toBe(true); + expect(state.getStreamingText()).toBe(""); + expect(state.getBufferedAttachments()).toEqual([]); + expect(state.hasBufferedBody()).toBe(false); + }); + + it("keeps a text-only empty turn empty after an undecided message boundary", () => { + const state = new AssistantExecutionState(); + state.appendStreamingText(" \n"); + expect(state.hasBufferedBody()).toBe(false); + + state.resetStreamingText(); + expect(state.materializationInput(null)).toEqual({ + content: "", + model: null, + attachments: [], + fromProvider: false, + }); + expect(state.hasMaterialized()).toBe(false); + }); +}); diff --git a/apps/server/src/features/agents/turns/assistant-execution-state.ts b/apps/server/src/features/agents/turns/assistant-execution-state.ts new file mode 100644 index 000000000..716460bcd --- /dev/null +++ b/apps/server/src/features/agents/turns/assistant-execution-state.ts @@ -0,0 +1,95 @@ +import type { StoredAttachment } from "@mcode/contracts"; + +/** Data needed to decide which assistant body a completed execution persists. */ +export interface AssistantMaterializationInput { + content: string; + model: string | null; + attachments: StoredAttachment[]; + fromProvider: boolean; +} + +interface BufferedBody { + content: string; + model: string | null; + attachments: StoredAttachment[]; +} + +/** Volatile assistant state for one provider execution, with no persistence or transport dependencies. */ +export class AssistantExecutionState { + private streamingText = ""; + private bufferedBody: BufferedBody | undefined; + private bufferedAttachments: StoredAttachment[] = []; + private materialized = false; + + appendStreamingText(delta: string): void { + this.streamingText += delta; + } + + getStreamingText(): string { + return this.streamingText; + } + + resetStreamingText(): void { + this.streamingText = ""; + } + + bufferBody(content: string, model: string | null, attachments = this.bufferedAttachments): void { + this.bufferedBody = { content, model, attachments }; + } + + bufferAttachments(attachments: StoredAttachment[]): void { + if (attachments.length === 0) return; + const byId = new Map(this.bufferedAttachments.map((attachment) => [attachment.id, attachment])); + for (const attachment of attachments) byId.set(attachment.id, attachment); + this.bufferedAttachments = [...byId.values()]; + } + + getBufferedAttachments(): StoredAttachment[] { + return this.bufferedAttachments; + } + + hasBufferedBody(): boolean { + return Boolean(this.bufferedBody?.content.trim() || this.streamingText.trim()); + } + + hasProviderBody(): boolean { + return this.bufferedBody !== undefined; + } + + hasMaterialized(): boolean { + return this.materialized; + } + + materializationInput(fallbackModel: string | null): AssistantMaterializationInput { + const buffered = this.bufferedBody; + if (buffered) { + return { + content: buffered.content, + model: buffered.model, + attachments: mergeAttachments(buffered.attachments, this.bufferedAttachments), + fromProvider: true, + }; + } + return { + content: this.streamingText.trim(), + model: fallbackModel, + attachments: this.bufferedAttachments, + fromProvider: false, + }; + } + + commitMaterialization(): void { + this.streamingText = ""; + this.bufferedBody = undefined; + this.bufferedAttachments = []; + this.materialized = true; + } +} + +function mergeAttachments(first: StoredAttachment[], second: StoredAttachment[]): StoredAttachment[] { + if (first.length === 0) return second; + if (second.length === 0) return first; + const byId = new Map(first.map((attachment) => [attachment.id, attachment])); + for (const attachment of second) byId.set(attachment.id, attachment); + return [...byId.values()]; +} diff --git a/apps/server/src/features/agents/turns/turn-finalizer.ts b/apps/server/src/features/agents/turns/turn-finalizer.ts index 3aecd61f3..2bba7bf1b 100644 --- a/apps/server/src/features/agents/turns/turn-finalizer.ts +++ b/apps/server/src/features/agents/turns/turn-finalizer.ts @@ -37,6 +37,7 @@ import type { ParentTurnDurability } from "./parent-turn-durability.js"; import type { ParentAssistantTextCheckpointService } from "./parent-assistant-text-checkpoint-service.js"; import { deriveTurnAssistantMessageId } from "./turn-assistant-message-id.js"; import type { TurnDiffService, SettleTurnDiff } from "./turn-diff-service.js"; +import { AssistantExecutionState, type AssistantMaterializationInput } from "./assistant-execution-state.js"; /** Pre-turn git ref captured at send time, used to diff the turn's file changes. */ interface TurnRef { @@ -45,13 +46,6 @@ interface TurnRef { fileTrackerGeneration?: number; } -/** Provider assistant body buffered during a turn, materialized at finalize. */ -interface BufferedBody { - content: string; - model: string | null; - attachments: StoredAttachment[]; -} - interface MaterializedAssistantRow { id: string; content: string; @@ -65,13 +59,6 @@ interface CanonicalProjection { narrative: ReturnType; } -interface AssistantMaterializationInput { - content: string; - model: string | null; - attachments: StoredAttachment[]; - fromProvider: boolean; -} - /** Durable snapshot operations required by terminal materialization. */ export type TurnSnapshotPersistence = Pick; @@ -88,14 +75,8 @@ export class TurnFinalizer { private readonly persistingThreads = new Set(); /** Last persisted assistant message id per thread, for late-hook attachment. */ private readonly lastPersistedMessageIdByThread = new Map(); - /** Streaming assistant text accumulated from textDelta events, per thread. */ - private readonly streamingAssistantTextByThread = new Map(); - /** Buffered provider assistant body awaiting materialization at finalize, per thread. */ - private readonly bufferedBodyByThread = new Map(); - /** Generated attachments awaiting materialization with the assistant row. */ - private readonly bufferedAttachmentsByThread = new Map(); - /** Threads whose assistant row was already materialized this turn (e.g. eagerly for a plan FK). */ - private readonly materializedThreads = new Set(); + /** One assistant decision state per active thread execution. */ + private readonly assistantStateByThread = new Map(); private readonly orm: BunSQLiteDatabase; /** Serializes finalize calls per thread so a slow git snapshot cannot drop a later turn. */ private readonly finalizeChainByThread = new Map>(); @@ -118,18 +99,17 @@ export class TurnFinalizer { /** Append a streaming assistant-text delta for the current turn. */ appendStreamingText(threadId: string, delta: string): void { - const prev = this.streamingAssistantTextByThread.get(threadId) ?? ""; - this.streamingAssistantTextByThread.set(threadId, prev + delta); + this.assistantState(threadId).appendStreamingText(delta); } /** Return the unmaterialized assistant text for the active turn. */ getStreamingText(threadId: string): string { - return this.streamingAssistantTextByThread.get(threadId) ?? ""; + return this.assistantStateByThread.get(threadId)?.getStreamingText() ?? ""; } /** Drop accumulated streaming text (a real assistant row now exists, or a new turn began). */ resetStreamingText(threadId: string): void { - this.streamingAssistantTextByThread.delete(threadId); + this.assistantStateByThread.get(threadId)?.resetStreamingText(); } /** @@ -147,24 +127,19 @@ export class TurnFinalizer { model: string | null, attachments = this.getBufferedAssistantAttachments(threadId), ): string { - this.bufferedBodyByThread.set(threadId, { content, model, attachments }); + this.assistantState(threadId).bufferBody(content, model, attachments); return deriveTurnAssistantMessageId(threadId, this.turnAnchorId(threadId)); } /** Buffer assistant-generated attachments until the turn's assistant row is materialized. */ bufferAssistantAttachments(threadId: string, attachments: StoredAttachment[]): void { if (attachments.length === 0) return; - const existing = this.bufferedAttachmentsByThread.get(threadId) ?? []; - const byId = new Map(existing.map((att) => [att.id, att])); - for (const att of attachments) { - byId.set(att.id, att); - } - this.bufferedAttachmentsByThread.set(threadId, [...byId.values()]); + this.assistantState(threadId).bufferAttachments(attachments); } /** Return generated attachments buffered for the current assistant turn. */ getBufferedAssistantAttachments(threadId: string): StoredAttachment[] { - return this.bufferedAttachmentsByThread.get(threadId) ?? []; + return this.assistantStateByThread.get(threadId)?.getBufferedAttachments() ?? []; } /** Record the pre-turn git ref so the turn's file changes can be diffed at finalize. */ @@ -206,20 +181,11 @@ export class TurnFinalizer { hasRecordableActivity(threadId: string): boolean { // An already-materialized row (e.g. eagerly written for a plan FK target) // is itself recordable activity even after its buffered body was consumed. - if (this.materializedThreads.has(threadId)) return true; - if (this.hasBufferedBody(threadId)) return true; - if (this.getBufferedAssistantAttachments(threadId).length > 0) return true; + const assistant = this.assistantStateByThread.get(threadId); + if (assistant?.hasMaterialized() || assistant?.hasBufferedBody() || assistant?.getBufferedAttachments().length) return true; return this.narrativeStore.hasBufferedNarrative(threadId); } - /** True when a non-empty assistant body is buffered (provider body or streaming text). */ - private hasBufferedBody(threadId: string): boolean { - const body = this.bufferedBodyByThread.get(threadId)?.content.trim(); - if (body) return true; - const streamed = this.streamingAssistantTextByThread.get(threadId)?.trim(); - return Boolean(streamed); - } - /** * Resolve the anchor id the turn's synthesized assistant row keys on — the id * of the message immediately preceding the assistant turn (normally the user @@ -731,16 +697,9 @@ export class TurnFinalizer { } private assistantMaterializationInput(threadId: string): AssistantMaterializationInput { - const buffered = this.bufferedBodyByThread.get(threadId); - const streamed = this.streamingAssistantTextByThread.get(threadId)?.trim(); - const fromProvider = buffered != null; - const content = fromProvider ? buffered.content : (streamed ?? ""); - const model = fromProvider ? buffered.model : (this.threadRepo.findById(threadId)?.model ?? null); - const bufferedAttachments = this.getBufferedAssistantAttachments(threadId); - const attachments = fromProvider - ? this.mergeAttachments(buffered.attachments, bufferedAttachments) - : bufferedAttachments; - return { content, model, attachments, fromProvider }; + const state = this.assistantState(threadId); + const fallbackModel = state.hasProviderBody() ? null : (this.threadRepo.findById(threadId)?.model ?? null); + return state.materializationInput(fallbackModel); } private broadcastMaterializedAssistant(threadId: string, materialized: MaterializedAssistantRow): void { @@ -756,10 +715,7 @@ export class TurnFinalizer { } private commitAssistantMaterialization(threadId: string): void { - this.streamingAssistantTextByThread.delete(threadId); - this.bufferedBodyByThread.delete(threadId); - this.bufferedAttachmentsByThread.delete(threadId); - this.materializedThreads.add(threadId); + this.assistantState(threadId).commitMaterialization(); } /** @@ -778,25 +734,15 @@ export class TurnFinalizer { if (turnRef !== undefined && this.turnRefBefore.get(threadId) === turnRef) { this.turnRefBefore.delete(threadId); } - this.streamingAssistantTextByThread.delete(threadId); - this.bufferedBodyByThread.delete(threadId); - this.bufferedAttachmentsByThread.delete(threadId); - this.materializedThreads.delete(threadId); + this.assistantStateByThread.delete(threadId); this.narrativeStore.clearTurn(threadId); this.turnFileTracker?.clearTurn(threadId, turnRef?.fileTrackerGeneration); this.persistingThreads.delete(threadId); } - private mergeAttachments( - first: StoredAttachment[], - second: StoredAttachment[], - ): StoredAttachment[] { - if (first.length === 0) return second; - if (second.length === 0) return first; - const byId = new Map(first.map((att) => [att.id, att])); - for (const att of second) { - byId.set(att.id, att); - } - return [...byId.values()]; + private assistantState(threadId: string): AssistantExecutionState { + const state = this.assistantStateByThread.get(threadId) ?? new AssistantExecutionState(); + this.assistantStateByThread.set(threadId, state); + return state; } } From 2956e48c049b86f8df9b5b39fb8f72d312f11123 Mon Sep 17 00:00:00 2001 From: "(CJ) Chukwudi Nwobodo" <142016413+chuks-qua@users.noreply.github.com> Date: Thu, 24 Sep 2026 21:31:40 +0100 Subject: [PATCH 065/136] feat(server): coordinate lost worker recovery before replacement --- .../execution-worker-loss-coordinator.test.ts | 203 ++++++++++++++++++ .../execution/execution-mailbox-scheduler.ts | 4 +- .../execution-worker-loss-coordinator.ts | 119 ++++++++++ 3 files changed, 324 insertions(+), 2 deletions(-) create mode 100644 apps/server/src/features/agents/execution/__tests__/execution-worker-loss-coordinator.test.ts create mode 100644 apps/server/src/features/agents/execution/execution-worker-loss-coordinator.ts diff --git a/apps/server/src/features/agents/execution/__tests__/execution-worker-loss-coordinator.test.ts b/apps/server/src/features/agents/execution/__tests__/execution-worker-loss-coordinator.test.ts new file mode 100644 index 000000000..2ed8edb66 --- /dev/null +++ b/apps/server/src/features/agents/execution/__tests__/execution-worker-loss-coordinator.test.ts @@ -0,0 +1,203 @@ +import "reflect-metadata"; +import * as NodeFS from "node:fs"; +import * as NodeOS from "node:os"; +import * as NodePath from "node:path"; +import type { Database } from "bun:sqlite"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; + +import { openDatabase } from "../../../../runtime/persistence/sqlite/database.js"; +import { CanonicalAgentWriterClient } from "../../canonical/canonical-agent-writer-client.js"; +import { CanonicalExecutionWriterPort } from "../../canonical/canonical-execution-writer-port.js"; +import { MessageRepo } from "../../conversation/persistence/message-repo.js"; +import { ParentAssistantTextCheckpointService } from "../../turns/parent-assistant-text-checkpoint-service.js"; +import type { ExecutionWorkerPort, ExecutionWorkerReply, ExecutionWorkerRequest } from "../execution-mailbox-protocol.js"; +import type { ExecutionMailboxCommand } from "../execution-mailbox-scheduler.js"; +import type { ExecutionWorkCommand, ExecutionWorkerResult } from "../execution-worker-handler.js"; +import { ExecutionThreadWorkerPort } from "../execution-worker-port.js"; +import { ExecutionWorkerLossCoordinator } from "../execution-worker-loss-coordinator.js"; + +const NOW = "2026-09-24T10:00:00.000Z"; +const execution = { + threadId: "lost-worker-thread", turnId: "lost-worker-turn", + executionId: "00000000-0000-4000-8000-000000000188", +} as const; +type Command = ExecutionMailboxCommand; + +const limits = { + maxPending: 8, maxPendingBytes: 8_000, reservedControl: 2, reservedControlBytes: 2_000, + maxPerExecutionPending: 6, maxPerExecutionBytes: 6_000, + reservedPerExecutionControl: 2, reservedPerExecutionControlBytes: 2_000, +}; + +class CrashingPort implements ExecutionWorkerPort { + onmessage: ((event: MessageEvent>) => void) | null = null; + onerror: ((event: ErrorEvent) => void) | null = null; + onclose: (() => void) | null = null; + private readonly inner: ExecutionThreadWorkerPort; + + constructor(writer: CanonicalExecutionWriterPort) { + this.inner = new ExecutionThreadWorkerPort(writer); + this.inner.onmessage = (event) => this.onmessage?.(event); + this.inner.onerror = (event) => this.onerror?.(event); + this.inner.onclose = () => this.onclose?.(); + } + + postMessage(request: ExecutionWorkerRequest): void { + this.inner.postMessage(request); + } + + terminate(): void { + this.inner.terminate(); + } + + crash(): void { + this.inner.terminate(); + this.onclose?.(); + } +} + +describe("ExecutionWorkerLossCoordinator with a file-backed writer", () => { + let directory: string; + let db: Database; + let writer: CanonicalAgentWriterClient; + let coordinator: ExecutionWorkerLossCoordinator; + let workers: CrashingPort[]; + let published: string[]; + let recoveryIncidentIds: string[]; + + beforeEach(() => { + directory = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "mcode-lost-worker-host-")); + db = openDatabase({ dbPath: NodePath.join(directory, "app.sqlite") }); + db.prepare("INSERT INTO workspaces (id, name, path, created_at, updated_at) VALUES (?, ?, ?, ?, ?)") + .run("lost-worker-workspace", "Workspace", directory, NOW, NOW); + db.prepare("INSERT INTO threads (id, workspace_id, title, branch, provider, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?)") + .run(execution.threadId, "lost-worker-workspace", "Thread", "main", "codex", NOW, NOW); + writer = new CanonicalAgentWriterClient(NodePath.join(directory, "app.sqlite")); + published = []; + const port = new CanonicalExecutionWriterPort(writer, (events) => { + published.push(...events.map((event) => event.eventId)); + }); + recoveryIncidentIds = []; + workers = []; + coordinator = new ExecutionWorkerLossCoordinator({ + interruptWorkerLoss: (input) => { + recoveryIncidentIds.push(input.recoveryIncidentId); + return port.interruptWorkerLoss(input); + }, + }, { + workerCount: 1, + limits, + createWorker: () => { + const worker = new CrashingPort(port); + workers.push(worker); + return worker; + }, + }); + }); + + afterEach(async () => { + coordinator.scheduler.shutdown(); + await writer.close(); + db.close(true); + NodeFS.rmSync(directory, { recursive: true, force: true }); + }); + + async function start(coordinator: ExecutionWorkerLossCoordinator, identity = execution) { + const claim = coordinator.scheduler.claim(identity, 1); + if (claim.kind !== "claimed") throw new Error(`Claim failed: ${claim.kind}`); + const command: ExecutionWorkCommand = { + kind: "start", providerId: "codex", + input: { + thread: { id: identity.threadId, workspaceId: "lost-worker-workspace", providerId: "codex", createdAt: NOW }, + turnId: identity.turnId, executionId: identity.executionId, + permissionMode: "supervised", providerIdentities: [], + userMessage: { kind: "create", messageId: `${identity.threadId}-user`, content: "Question", sequence: 1 }, + }, + }; + const admitted = coordinator.scheduler.submit({ execution: identity, lease: claim.lease, command, byteLength: 1_000 }); + if (admitted.kind !== "admitted") throw new Error(`Start not admitted: ${admitted.kind}`); + expect(await admitted.completion).toMatchObject({ kind: "reply", result: { kind: "committed" } }); + return claim.lease; + } + + it("interrupts a crashed worker before replacing its slot", async () => { + const lease = await start(coordinator); + new ParentAssistantTextCheckpointService(db).appendChunk([{ ...execution, sequence: 1, text: "Partial answer" }]); + workers[0]?.crash(); + expect(await coordinator.waitForRecovery(0)).toEqual({ kind: "recovered", workerIndex: 0 }); + expect(workers).toHaveLength(2); + expect(db.prepare("SELECT terminal_outcome, recovery_incident_id FROM canonical_agent_ingest_checkpoints WHERE execution_id = ?") + .get(execution.executionId)).toMatchObject({ terminal_outcome: "interrupted", + recovery_incident_id: expect.stringMatching(/^worker-loss:[0-9a-f]{64}$/) }); + expect(new MessageRepo(db).listIncludingInternal(execution.threadId)).toContainEqual(expect.objectContaining({ + role: "assistant", content: "Partial answer", outcome: "interrupted", is_internal: false, + })); + expect(published).toContain(`${execution.executionId}:recovery-interrupted`); + expect(coordinator.scheduler.submit({ execution, lease, command: { + kind: "checkpoint", phase: "running", nativeCursor: null, + }, byteLength: 100 })).toEqual({ kind: "stale-execution" }); + expect(coordinator.scheduler.claim({ ...execution, executionId: "new-execution" }, 2).kind).toBe("claimed"); + }); + + it("keeps the slot fenced after a failed writer receipt until explicit retry", async () => { + const lease = await start(coordinator); + db.run(`CREATE TRIGGER fail_loss_receipt BEFORE INSERT ON canonical_writer_operation_receipts + WHEN NEW.kind = 'semantic:worker-lost' BEGIN SELECT RAISE(ABORT, 'loss receipt unavailable'); END`); + workers[0]?.crash(); + expect(await coordinator.waitForRecovery(0)).toMatchObject({ kind: "failed", workerIndex: 0, + unresolved: [{ execution, lease }] }); + expect(workers).toHaveLength(1); + expect(coordinator.scheduler.claim(execution, 2)).toEqual({ kind: "thread-busy" }); + expect(coordinator.scheduler.claim({ threadId: "other", turnId: "other", executionId: "other" }, 2)) + .toEqual({ kind: "worker-unavailable" }); + expect(db.prepare("SELECT terminal_outcome FROM canonical_agent_ingest_checkpoints WHERE execution_id = ?") + .get(execution.executionId)).toEqual({ terminal_outcome: null }); + db.run("DROP TRIGGER fail_loss_receipt"); + expect(await coordinator.retryFailed(0)).toEqual({ kind: "recovered", workerIndex: 0 }); + expect(workers).toHaveLength(2); + expect(recoveryIncidentIds).toHaveLength(2); + expect(recoveryIncidentIds[1]).toBe(recoveryIncidentIds[0]); + expect(db.prepare("SELECT terminal_outcome FROM canonical_agent_ingest_checkpoints WHERE execution_id = ?") + .get(execution.executionId)).toEqual({ terminal_outcome: "interrupted" }); + }); + + it("keeps a shared slot fenced until every lost task has writer evidence", async () => { + const second = { + threadId: "second-lost-thread", turnId: "second-lost-turn", + executionId: "00000000-0000-4000-8000-000000000189", + }; + db.prepare("INSERT INTO threads (id, workspace_id, title, branch, provider, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?)") + .run(second.threadId, "lost-worker-workspace", "Second", "main", "codex", NOW, NOW); + await start(coordinator); + await start(coordinator, second); + db.run(`CREATE TRIGGER fail_second_loss BEFORE INSERT ON canonical_writer_operation_receipts + WHEN NEW.kind = 'semantic:worker-lost' AND NEW.execution_id = '${second.executionId}' + BEGIN SELECT RAISE(ABORT, 'second loss receipt unavailable'); END`); + workers[0]?.crash(); + expect(await coordinator.waitForRecovery(0)).toMatchObject({ kind: "failed", workerIndex: 0, + unresolved: [{ execution: second }] }); + expect(workers).toHaveLength(1); + expect(db.prepare("SELECT terminal_outcome FROM canonical_agent_ingest_checkpoints WHERE execution_id = ?") + .get(execution.executionId)).toEqual({ terminal_outcome: "interrupted" }); + expect(db.prepare("SELECT terminal_outcome FROM canonical_agent_ingest_checkpoints WHERE execution_id = ?") + .get(second.executionId)).toEqual({ terminal_outcome: null }); + expect(coordinator.scheduler.claim(second, 2)).toEqual({ kind: "thread-busy" }); + db.run("DROP TRIGGER fail_second_loss"); + expect(await coordinator.retryFailed(0)).toEqual({ kind: "recovered", workerIndex: 0 }); + expect(workers).toHaveLength(2); + expect(recoveryIncidentIds).toHaveLength(3); + expect(recoveryIncidentIds[2]).toBe(recoveryIncidentIds[1]); + expect(recoveryIncidentIds[0]).not.toBe(recoveryIncidentIds[1]); + expect(db.prepare("SELECT terminal_outcome FROM canonical_agent_ingest_checkpoints WHERE execution_id = ?") + .get(second.executionId)).toEqual({ terminal_outcome: "interrupted" }); + const incidents = db.prepare("SELECT recovery_incident_id FROM canonical_agent_ingest_checkpoints WHERE execution_id IN (?, ?)") + .all(execution.executionId, second.executionId); + expect(new Set(incidents.map((row) => JSON.stringify(row))).size).toBe(2); + }); + + it("replaces an idle crashed slot using the callback's slot index", async () => { + workers[0]?.crash(); + expect(await coordinator.waitForRecovery(0)).toEqual({ kind: "recovered", workerIndex: 0 }); + expect(workers).toHaveLength(2); + }); +}); diff --git a/apps/server/src/features/agents/execution/execution-mailbox-scheduler.ts b/apps/server/src/features/agents/execution/execution-mailbox-scheduler.ts index b72ab6b2e..c896ae584 100644 --- a/apps/server/src/features/agents/execution/execution-mailbox-scheduler.ts +++ b/apps/server/src/features/agents/execution/execution-mailbox-scheduler.ts @@ -79,7 +79,7 @@ export interface ExecutionMailboxOptions ExecutionWorkerPort, Result>; - readonly onWorkerLost: (assignments: readonly ExecutionLostAssignment[]) => void; + readonly onWorkerLost: (assignments: readonly ExecutionLostAssignment[], workerIndex: number) => void; } export type ExecutionClaim = @@ -400,7 +400,7 @@ export class ExecutionMailboxScheduler ({ execution: assignment.execution, lease: assignment.lease })); this.settleSlot(slot, "worker-lost"); - this.onWorkerLost(revoked); + this.onWorkerLost(revoked, slot.index); } private settleSlot(slot: Slot, Result>, kind: "worker-lost" | "shutdown"): void { diff --git a/apps/server/src/features/agents/execution/execution-worker-loss-coordinator.ts b/apps/server/src/features/agents/execution/execution-worker-loss-coordinator.ts new file mode 100644 index 000000000..fb246b6b2 --- /dev/null +++ b/apps/server/src/features/agents/execution/execution-worker-loss-coordinator.ts @@ -0,0 +1,119 @@ +import * as NodeCrypto from "node:crypto"; + +import type { CanonicalExecutionWriterPort } from "../canonical/canonical-execution-writer-port.js"; +import { + ExecutionMailboxScheduler, + type ExecutionLostAssignment, + type ExecutionMailboxOptions, + type ExecutionRecoveryReceipt, +} from "./execution-mailbox-scheduler.js"; +import type { ExecutionWorkCommand, ExecutionWorkerResult, ExecutionWriteReceipt } from "./execution-worker-handler.js"; + +const WORKER_LOSS_REASON = "The provider could not prove that this execution was still active after its worker exited."; + +type Scheduler = ExecutionMailboxScheduler; +type SchedulerOptions = Omit, "onWorkerLost">; + +/** Reconciliation result for a worker slot. A failure leaves that slot unavailable. */ +export type WorkerLossResolution = + | { readonly kind: "recovered"; readonly workerIndex: number } + | { readonly kind: "failed"; readonly workerIndex: number; readonly error: Error; readonly unresolved: readonly ExecutionLostAssignment[] }; + +interface LostSlot { + readonly workerIndex: number; + readonly unresolved: ExecutionLostAssignment[]; + task: Promise | null; + result: WorkerLossResolution | null; +} + +/** Owns the durable handoff between a crashed execution worker and its replacement. */ +export class ExecutionWorkerLossCoordinator { + readonly scheduler: Scheduler; + private readonly slots = new Map(); + + constructor( + private readonly writer: Pick, + options: SchedulerOptions, + ) { + this.scheduler = new ExecutionMailboxScheduler({ + ...options, + onWorkerLost: (assignments, workerIndex) => this.workerLost(assignments, workerIndex), + }); + } + + /** Wait for the current recovery attempt without starting another attempt. */ + waitForRecovery(workerIndex: number): Promise { + const slot = this.slots.get(workerIndex); + if (!slot?.task) throw new Error(`No lost execution worker at slot ${workerIndex}`); + return slot.task; + } + + /** Retry a failed reconciliation explicitly; no provider command is replayed. */ + retryFailed(workerIndex: number): Promise { + const slot = this.slots.get(workerIndex); + if (!slot || slot.result?.kind !== "failed") { + throw new Error(`No failed execution recovery at slot ${workerIndex}`); + } + this.schedule(slot); + if (!slot.task) throw new Error(`No pending execution recovery at slot ${workerIndex}`); + return slot.task; + } + + private workerLost(assignments: readonly ExecutionLostAssignment[], workerIndex: number): void { + const slot: LostSlot = { + workerIndex, + unresolved: [...assignments], + task: null, + result: null, + }; + this.slots.set(workerIndex, slot); + this.schedule(slot); + } + + private schedule(slot: LostSlot): void { + slot.result = null; + slot.task = Promise.resolve().then(() => this.reconcile(slot)).then((result) => { + slot.result = result; + return result; + }); + } + + private async reconcile(slot: LostSlot): Promise { + try { + while (slot.unresolved.length > 0) { + const assignment = slot.unresolved[0]; + if (!assignment) break; + const receipt = await this.writer.interruptWorkerLoss({ + ...assignment, + reason: WORKER_LOSS_REASON, + recoveryIncidentId: incidentId(assignment), + }); + const evidence = recoveryEvidence(receipt); + if (!evidence || !this.scheduler.reconcileLost(assignment.execution, assignment.lease, evidence)) { + throw new Error(`Lost execution has no matching durable recovery evidence: ${assignment.execution.executionId}`); + } + slot.unresolved.shift(); + } + if (!this.scheduler.replaceWorker(slot.workerIndex)) { + throw new Error(`Lost execution worker slot ${slot.workerIndex} could not be replaced`); + } + return { kind: "recovered", workerIndex: slot.workerIndex }; + } catch (error) { + return { kind: "failed", workerIndex: slot.workerIndex, + error: error instanceof Error ? error : new Error(String(error)), + unresolved: [...slot.unresolved] }; + } + } +} + +function recoveryEvidence(receipt: ExecutionWriteReceipt): ExecutionRecoveryReceipt | null { + if (receipt.kind === "committed") return receipt; + if (!receipt.recoveryState) return null; + return { ...receipt, recoveryState: receipt.recoveryState }; +} + +function incidentId(assignment: ExecutionLostAssignment): string { + const hash = NodeCrypto.createHash("sha256") + .update(JSON.stringify([assignment.execution, assignment.lease])).digest("hex"); + return `worker-loss:${hash}`; +} From 0a0decb54825d8c0181922de88845d93d62206a4 Mon Sep 17 00:00:00 2001 From: "(CJ) Chukwudi Nwobodo" <142016413+chuks-qua@users.noreply.github.com> Date: Thu, 24 Sep 2026 21:33:36 +0100 Subject: [PATCH 066/136] refactor(server): bind narrative buffers to one state identity --- .../__tests__/narrative-store.test.ts | 39 +++- .../conversation/narrative/narrative-store.ts | 109 ++++++++- .../narrative/narrative-turn-state.ts | 211 +++++++++++------- 3 files changed, 276 insertions(+), 83 deletions(-) diff --git a/apps/server/src/features/agents/conversation/narrative/__tests__/narrative-store.test.ts b/apps/server/src/features/agents/conversation/narrative/__tests__/narrative-store.test.ts index 7ecbac934..7020f2219 100644 --- a/apps/server/src/features/agents/conversation/narrative/__tests__/narrative-store.test.ts +++ b/apps/server/src/features/agents/conversation/narrative/__tests__/narrative-store.test.ts @@ -640,7 +640,7 @@ describe("NarrativeStore write seam (server-side traps)", () => { } it("emits late Agent identity as data without a repository", () => { - const state = new NarrativeTurnState(); + const state = new NarrativeTurnState(THREAD); state.beginTurn(THREAD); state.resetTurnCounters(THREAD); state.bufferToolCall(THREAD, { @@ -667,6 +667,43 @@ describe("NarrativeStore write seam (server-side traps)", () => { }]); }); + it("keeps two execution states for the same thread independent", () => { + const firstExecution = { threadId: THREAD, turnId: "turn-1", executionId: "execution-1" }; + const secondExecution = { threadId: THREAD, turnId: "turn-2", executionId: "execution-2" }; + const first = new NarrativeTurnState(firstExecution); + const second = new NarrativeTurnState(secondExecution); + first.beginTurn(THREAD); + first.resetTurnCounters(THREAD); + second.beginTurn(THREAD); + second.resetTurnCounters(THREAD); + first.openOrExtendThought(THREAD, "First turn"); + first.bufferToolCall(THREAD, toolUse("read-1", "Read")); + + expect(first.recoverySnapshot(THREAD).map((item) => item.record.sort_order)).toEqual([0, 1]); + expect(first.execution).toEqual(firstExecution); + expect(second.execution).toEqual(secondExecution); + expect(() => first.assertExecution(secondExecution)).toThrow("different execution"); + expect(() => first.assertExecution(firstExecution)).not.toThrow(); + expect(second.recoverySnapshot(THREAD)).toEqual([]); + expect(second.nextSortOrder(THREAD)).toBe(0); + expect(() => first.bufferToolCall("another-thread", toolUse("read-2", "Read"))) + .toThrow("Narrative turn belongs to thread-1"); + expect(first.getBufferedToolCalls(THREAD)).toHaveLength(1); + }); + + it("keeps the live store's thread adapters independent", () => { + const otherThread = "thread-2"; + store.beginTurn(THREAD); + store.resetTurnCounters(THREAD); + store.beginTurn(otherThread); + store.resetTurnCounters(otherThread); + store.bufferToolCall(THREAD, toolUse("read-1", "Read")); + + expect(store.getBufferedToolCalls(otherThread)).toEqual([]); + expect(store.nextSortOrder(otherThread)).toBe(0); + expect(store.nextSortOrder(THREAD)).toBe(1); + }); + it("persists a long active turn in order across bounded transactions", async () => { seedAssistantMessage("m1", "done", 1); store.beginTurn(THREAD); diff --git a/apps/server/src/features/agents/conversation/narrative/narrative-store.ts b/apps/server/src/features/agents/conversation/narrative/narrative-store.ts index 23effb8a3..0cebbc235 100644 --- a/apps/server/src/features/agents/conversation/narrative/narrative-store.ts +++ b/apps/server/src/features/agents/conversation/narrative/narrative-store.ts @@ -138,17 +138,14 @@ interface PersistedNarrativeRows { type RecoveredToolCallItem = Extract; @injectable() -export class NarrativeStore extends NarrativeTurnState { +export class NarrativeStore { constructor( @inject(MessageRepo) _messageRepo: MessageRepo, @inject(ToolCallRecordRepo) private readonly toolCallRecordRepo: ToolCallRecordRepo, @inject(ThoughtSegmentRepo) private readonly thoughtSegmentRepo: ThoughtSegmentRepo, @inject(HookExecutionRepo) private readonly hookExecutionRepo: HookExecutionRepo, @inject("Database") private readonly db?: Database, - ) { - super(); - this.setEffectSink((effect) => this.applyStateEffect(effect)); - } + ) {} private applyStateEffect(effect: NarrativeTurnStateEffect): void { this.toolCallRecordRepo.updateSubagentIdentity( @@ -156,6 +153,108 @@ export class NarrativeStore extends NarrativeTurnState { ); } + private readonly narrativeStates = new Map(); + + private stateFor(threadId: string): NarrativeTurnState { + const existing = this.narrativeStates.get(threadId); + if (existing) return existing; + const state = new NarrativeTurnState(threadId, (effect) => this.applyStateEffect(effect)); + this.narrativeStates.set(threadId, state); + return state; + } + + beginTurn(...args: Parameters): ReturnType { + return this.stateFor(args[0]).beginTurn(...args); + } + + resetTurnCounters(...args: Parameters): ReturnType { + return this.stateFor(args[0]).resetTurnCounters(...args); + } + + nextSortOrder(...args: Parameters): ReturnType { + return this.stateFor(args[0]).nextSortOrder(...args); + } + + openOrExtendThought(...args: Parameters): ReturnType { + return this.stateFor(args[0]).openOrExtendThought(...args); + } + + closeOpenThought(...args: Parameters): ReturnType { + this.narrativeStates.get(args[0])?.closeOpenThought(...args); + } + + dropOpenThought(...args: Parameters): ReturnType { + this.narrativeStates.get(args[0])?.dropOpenThought(...args); + } + + takeOpenThought(...args: Parameters): ReturnType { + return this.narrativeStates.get(args[0])?.takeOpenThought(...args) ?? ""; + } + + getCurrentParentToolCallId(...args: Parameters): ReturnType { + return this.narrativeStates.get(args[0])?.getCurrentParentToolCallId(...args); + } + + bufferToolCall(...args: Parameters): ReturnType { + return this.stateFor(args[0]).bufferToolCall(...args); + } + + updateBufferedToolCallOutput(...args: Parameters): ReturnType { + return this.stateFor(args[0]).updateBufferedToolCallOutput(...args); + } + + clearAgentStackOnMessage(...args: Parameters): ReturnType { + this.narrativeStates.get(args[0])?.clearAgentStackOnMessage(...args); + } + + getBufferedToolCalls(...args: Parameters): ReturnType { + return this.narrativeStates.get(args[0])?.getBufferedToolCalls(...args) ?? []; + } + + recoverySnapshot(...args: Parameters): ReturnType { + return this.narrativeStates.get(args[0])?.recoverySnapshot(...args) ?? []; + } + + stageNarrationSegment(...args: Parameters): ReturnType { + return this.stateFor(args[0]).stageNarrationSegment(...args); + } + + recoverySnapshotWithStagedNarration(...args: Parameters): ReturnType { + return this.stateFor(args[0]).recoverySnapshotWithStagedNarration(...args); + } + + applyStagedNarrationSegment(...args: Parameters): ReturnType { + return this.stateFor(args[0]).applyStagedNarrationSegment(...args); + } + + hasBufferedNarrative(...args: Parameters): ReturnType { + return this.narrativeStates.get(args[0])?.hasBufferedNarrative(...args) ?? false; + } + + openHook(...args: Parameters): ReturnType { + return this.stateFor(args[0]).openHook(...args); + } + + peekOpenHook(...args: Parameters): ReturnType { + return this.narrativeStates.get(args[0])?.peekOpenHook(...args); + } + + removeOpenHook(...args: Parameters): ReturnType { + this.narrativeStates.get(args[0])?.removeOpenHook(...args); + } + + pushClosedHook(...args: Parameters): ReturnType { + return this.stateFor(args[0]).pushClosedHook(...args); + } + + prepareNarrativePersistence(...args: Parameters): ReturnType { + return this.stateFor(args[0]).prepareNarrativePersistence(...args); + } + + clearTurn(...args: Parameters): ReturnType { + this.narrativeStates.get(args[0])?.clearTurn(...args); + } + private ormInstance: BunSQLiteDatabase | null = null; private requireOrm(): BunSQLiteDatabase { diff --git a/apps/server/src/features/agents/conversation/narrative/narrative-turn-state.ts b/apps/server/src/features/agents/conversation/narrative/narrative-turn-state.ts index c913db728..4e02e587e 100644 --- a/apps/server/src/features/agents/conversation/narrative/narrative-turn-state.ts +++ b/apps/server/src/features/agents/conversation/narrative/narrative-turn-state.ts @@ -12,6 +12,7 @@ import type { CreateHookExecutionInput } from "../../events/persistence/hook-exe import type { TurnOutcome } from "../../turns/turn-outcome.js"; import { ACTIVE_TURN_WRITE_BATCH_LIMITS } from "../../../../runtime/persistence/sqlite/bounded-write-batches.js"; import { assertActiveTurnRecoveryRetention } from "../../turns/active-turn-recovery-retention-policy.js"; +import type { ExecutionIdentity } from "../../execution/execution-mailbox-protocol.js"; /** Buffered tool call with raw input preserved for deferred summarization. */ export interface BufferedToolCall extends CreateToolCallRecordInput { @@ -152,20 +153,42 @@ export type NarrativeTurnStateEffect = { readonly identityKey: string; }; -/** Owns per-thread narrative buffers and classification without a database connection. */ +/** Owns one identity's narrative buffers; the live store currently binds by thread. */ export class NarrativeTurnState { - private turnToolCalls = new Map(); - private agentCallStack = new Map(); - private turnSortCounters = new Map(); - private turnOpenThought = new Map(); - private turnThoughts = new Map(); - private turnOpenHooks = new Map>(); - private turnHooks = new Map(); - private effectSink: ((effect: NarrativeTurnStateEffect) => void) | undefined; + private turnToolCalls: BufferedToolCall[] = []; + private agentCallStack: string[] = []; + private turnSortCounters = 0; + private turnOpenThought: OpenThought | null = null; + private turnThoughts: CreateThoughtSegmentInput[] = []; + private turnOpenHooks = new Map(); + private turnHooks: CreateHookExecutionInput[] = []; + private readonly effectSink: ((effect: NarrativeTurnStateEffect) => void) | undefined; private readonly pendingEffects: NarrativeTurnStateEffect[] = []; + readonly threadId: string; + readonly execution: ExecutionIdentity | undefined; + + constructor( + owner: string | ExecutionIdentity, + effectSink?: (effect: NarrativeTurnStateEffect) => void, + ) { + this.threadId = typeof owner === "string" ? owner : owner.threadId; + this.execution = typeof owner === "string" ? undefined : { ...owner }; + this.effectSink = effectSink; + } + + /** Fence a worker command to the exact turn attempt bound to this state. */ + assertExecution(execution: ExecutionIdentity): void { + const owner = this.execution; + if (!owner || owner.threadId !== execution.threadId + || owner.turnId !== execution.turnId || owner.executionId !== execution.executionId) { + throw new Error(`Narrative turn belongs to a different execution: ${execution.executionId}`); + } + } - protected setEffectSink(sink: (effect: NarrativeTurnStateEffect) => void): void { - this.effectSink = sink; + private assertThread(threadId: string): void { + if (threadId !== this.threadId) { + throw new Error(`Narrative turn belongs to ${this.threadId}, received ${threadId}`); + } } /** Return any data-only effects emitted without a live persistence adapter. */ @@ -186,11 +209,12 @@ export class NarrativeTurnState { * prior turn can still increment the old counter. */ beginTurn(threadId: string): void { - this.turnToolCalls.set(threadId, []); - this.turnOpenThought.set(threadId, null); - this.turnThoughts.set(threadId, []); - this.turnOpenHooks.set(threadId, new Map()); - this.turnHooks.set(threadId, []); + this.assertThread(threadId); + this.turnToolCalls = []; + this.turnOpenThought = null; + this.turnThoughts = []; + this.turnOpenHooks = new Map(); + this.turnHooks = []; } /** @@ -200,14 +224,16 @@ export class NarrativeTurnState { * increment the old one (see {@link clearTurn}). */ resetTurnCounters(threadId: string): void { - this.turnSortCounters.set(threadId, 0); - this.agentCallStack.set(threadId, []); + this.assertThread(threadId); + this.turnSortCounters = 0; + this.agentCallStack = []; } /** Allocate the next shared sort order for the thread's current turn. */ nextSortOrder(threadId: string): number { - const sortOrder = this.turnSortCounters.get(threadId) ?? 0; - this.turnSortCounters.set(threadId, sortOrder + 1); + this.assertThread(threadId); + const sortOrder = this.turnSortCounters; + this.turnSortCounters = sortOrder + 1; return sortOrder; } @@ -218,15 +244,16 @@ export class NarrativeTurnState { * the live client builder. Never touches the `agentCallStack` (Trap 2). */ openOrExtendThought(threadId: string, delta: string): void { - const open = this.turnOpenThought.get(threadId); + this.assertThread(threadId); + const open = this.turnOpenThought; if (!open) { const sortOrder = this.nextSortOrder(threadId); - this.turnOpenThought.set(threadId, { + this.turnOpenThought = { id: NodeCrypto.randomUUID(), text: delta, startedAt: new Date().toISOString(), sortOrder, - }); + }; } else { open.text += delta; } @@ -238,9 +265,10 @@ export class NarrativeTurnState { * sorts strictly before the tool) and during turn-end drain. */ closeOpenThought(threadId: string): void { - const open = this.turnOpenThought.get(threadId); + this.assertThread(threadId); + const open = this.turnOpenThought; if (!open) return; - const list = this.turnThoughts.get(threadId) ?? []; + const list = this.turnThoughts; list.push({ id: open.id, messageId: "", @@ -249,8 +277,8 @@ export class NarrativeTurnState { endedAt: new Date().toISOString(), sortOrder: open.sortOrder, }); - this.turnThoughts.set(threadId, list); - this.turnOpenThought.set(threadId, null); + this.turnThoughts = list; + this.turnOpenThought = null; } /** @@ -262,7 +290,8 @@ export class NarrativeTurnState { * would duplicate the body as a ThoughtBlock in the narrative. */ dropOpenThought(threadId: string): void { - this.turnOpenThought.set(threadId, null); + this.assertThread(threadId); + this.turnOpenThought = null; } /** @@ -273,8 +302,9 @@ export class NarrativeTurnState { * TurnFinalizer so the same text is not buffered in both stores. */ takeOpenThought(threadId: string): string { - const open = this.turnOpenThought.get(threadId); - this.turnOpenThought.set(threadId, null); + this.assertThread(threadId); + const open = this.turnOpenThought; + this.turnOpenThought = null; return open?.text ?? ""; } @@ -284,6 +314,7 @@ export class NarrativeTurnState { * {@link bufferToolCall} when the SDK omits `parent_tool_use_id` (Trap 1). */ getCurrentParentToolCallId(threadId: string): string | undefined { + this.assertThread(threadId); return this.getStackDerivedParentFallback(threadId); } @@ -295,10 +326,11 @@ export class NarrativeTurnState { * undefined so tools do not attach under the wrong subagent row. */ private getStackDerivedParentFallback(threadId: string): string | undefined { - const stack = this.agentCallStack.get(threadId) ?? []; + this.assertThread(threadId); + const stack = this.agentCallStack; if (stack.length === 0) return undefined; - const buffer = this.turnToolCalls.get(threadId) ?? []; + const buffer = this.turnToolCalls; const runningAgentIds: string[] = []; for (const agentId of stack) { const row = buffer.find( @@ -320,8 +352,9 @@ export class NarrativeTurnState { * and are pushed onto the `agentCallStack` (Trap 2 push site). */ bufferToolCall(threadId: string, event: BufferToolCallEvent): string | undefined { - const buffer = this.turnToolCalls.get(threadId) ?? []; - const stack = this.agentCallStack.get(threadId) ?? []; + this.assertThread(threadId); + const buffer = this.turnToolCalls; + const stack = this.agentCallStack; const parentToolCallId = this.resolveParentToolCallId(threadId, event); this.logParentToolCallAttribution(threadId, event, parentToolCallId, stack.length); const existing = buffer.find((tc) => tc.toolCallId === event.toolCallId); @@ -341,6 +374,7 @@ export class NarrativeTurnState { threadId: string, event: BufferToolCallEvent, ): string | undefined { + this.assertThread(threadId); if (event.toolName === "Agent") return event.parentToolCallId; return event.parentToolCallId ?? this.getStackDerivedParentFallback(threadId); } @@ -351,6 +385,7 @@ export class NarrativeTurnState { parentToolCallId: string | undefined, stackDepth: number, ): void { + this.assertThread(threadId); if (event.toolName === "Agent" || !parentToolCallId) return; logger.debug("bufferToolCall: parent attribution", { threadId, @@ -456,14 +491,15 @@ export class NarrativeTurnState { event: BufferToolCallEvent, parentToolCallId: string | undefined, ): string | undefined { + this.assertThread(threadId); const sortOrder = this.nextSortOrder(threadId); if (event.toolName === "Agent") { stack.push(event.toolCallId); - this.agentCallStack.set(threadId, stack); + this.agentCallStack = stack; } const presentation = this.subagentPresentation(event); buffer.push(this.createBufferedToolCall(event, presentation, sortOrder, parentToolCallId)); - this.turnToolCalls.set(threadId, buffer); + this.turnToolCalls = buffer; return parentToolCallId; } @@ -524,6 +560,7 @@ export class NarrativeTurnState { }, subagentPresentation?: SubagentPresentation, ): void { + this.assertThread(threadId); this.removeAgentFromStack(threadId, toolCallId); const toolCall = this.latestBufferedToolCall(threadId, toolCallId); if (!toolCall) return; @@ -542,11 +579,12 @@ export class NarrativeTurnState { } private removeAgentFromStack(threadId: string, toolCallId: string): void { - const stack = this.agentCallStack.get(threadId) ?? []; + this.assertThread(threadId); + const stack = this.agentCallStack; const stackIndex = stack.indexOf(toolCallId); if (stackIndex < 0) return; stack.splice(stackIndex, 1); - this.agentCallStack.set(threadId, stack); + this.agentCallStack = stack; logger.debug("updateBufferedToolCallOutput: popped Agent from stack", { threadId, toolCallId, @@ -555,7 +593,8 @@ export class NarrativeTurnState { } private latestBufferedToolCall(threadId: string, toolCallId: string): BufferedToolCall | undefined { - const buffer = this.turnToolCalls.get(threadId) ?? []; + this.assertThread(threadId); + const buffer = this.turnToolCalls; for (let index = buffer.length - 1; index >= 0; index -= 1) { if (buffer[index].toolCallId === toolCallId) return buffer[index]; } @@ -611,15 +650,17 @@ export class NarrativeTurnState { * end-of-turn clear). No-ops when the stack is already empty. */ clearAgentStackOnMessage(threadId: string): void { - const stack = this.agentCallStack.get(threadId); - if (stack && stack.length > 0) { + this.assertThread(threadId); + const stack = this.agentCallStack; + if (stack.length > 0) { stack.length = 0; } } /** Snapshot of the thread's buffered tool calls (read-only inspection). */ getBufferedToolCalls(threadId: string): readonly BufferedToolCall[] { - return this.turnToolCalls.get(threadId) ?? []; + this.assertThread(threadId); + return this.turnToolCalls; } /** @@ -627,10 +668,11 @@ export class NarrativeTurnState { * retaining provider protocol traffic or private raw tool input. */ recoverySnapshot(threadId: string): ParentNarrativeRecoveryItem[] { + this.assertThread(threadId); const snapshot: ParentNarrativeRecoveryItem[] = []; let bytes = 0; bytes = this.appendBufferedToolCallRecoveryItems(snapshot, bytes, threadId); - for (const thought of this.turnThoughts.get(threadId) ?? []) { + for (const thought of this.turnThoughts) { bytes = this.appendRecoverySnapshotItem( snapshot, bytes, @@ -638,7 +680,7 @@ export class NarrativeTurnState { threadId, ); } - const openThought = this.turnOpenThought.get(threadId); + const openThought = this.turnOpenThought; if (openThought) { bytes = this.appendRecoverySnapshotItem( snapshot, @@ -647,7 +689,7 @@ export class NarrativeTurnState { threadId, ); } - for (const hook of this.turnHooks.get(threadId) ?? []) { + for (const hook of this.turnHooks) { bytes = this.appendRecoverySnapshotItem( snapshot, bytes, @@ -655,7 +697,7 @@ export class NarrativeTurnState { threadId, ); } - for (const hook of this.turnOpenHooks.get(threadId)?.values() ?? []) { + for (const hook of this.turnOpenHooks.values()) { bytes = this.appendRecoverySnapshotItem( snapshot, bytes, @@ -671,7 +713,7 @@ export class NarrativeTurnState { bytes: number, threadId: string, ): number { - for (const toolCall of this.turnToolCalls.get(threadId) ?? []) { + for (const toolCall of this.turnToolCalls) { bytes = this.appendRecoverySnapshotItem( snapshot, bytes, @@ -834,7 +876,8 @@ export class NarrativeTurnState { /** Stage narration for recovery without mutating the active turn buffer. */ stageNarrationSegment(threadId: string, text: string): StagedNarrationSegment | null { - const open = this.turnOpenThought.get(threadId); + this.assertThread(threadId); + const open = this.turnOpenThought; const endedAt = new Date().toISOString(); if (open) { return { @@ -852,7 +895,7 @@ export class NarrativeTurnState { text, startedAt: endedAt, endedAt, - sortOrder: this.turnSortCounters.get(threadId) ?? 0, + sortOrder: this.turnSortCounters, }; } @@ -861,6 +904,7 @@ export class NarrativeTurnState { threadId: string, staged: StagedNarrationSegment, ): ParentNarrativeRecoveryItem[] { + this.assertThread(threadId); const snapshot = this.recoverySnapshot(threadId).filter((item) => ( item.kind !== "narrationSegment" || item.record.id !== staged.id )); @@ -889,19 +933,20 @@ export class NarrativeTurnState { /** Apply a narration record only after its recovery projection is durable. */ applyStagedNarrationSegment(threadId: string, staged: StagedNarrationSegment): void { + this.assertThread(threadId); if (staged.openThoughtId) { - const open = this.turnOpenThought.get(threadId); + const open = this.turnOpenThought; if (!open || open.id !== staged.openThoughtId) { throw new Error(`Staged narration no longer matches the open thought: ${staged.id}`); } - this.turnOpenThought.set(threadId, null); + this.turnOpenThought = null; } else { - const nextSortOrder = this.turnSortCounters.get(threadId) ?? 0; + const nextSortOrder = this.turnSortCounters; if (nextSortOrder <= staged.sortOrder) { - this.turnSortCounters.set(threadId, staged.sortOrder + 1); + this.turnSortCounters = staged.sortOrder + 1; } } - const thoughts = this.turnThoughts.get(threadId) ?? []; + const thoughts = this.turnThoughts; thoughts.push({ id: staged.id, messageId: "", @@ -910,7 +955,7 @@ export class NarrativeTurnState { endedAt: staged.endedAt, sortOrder: staged.sortOrder, }); - this.turnThoughts.set(threadId, thoughts); + this.turnThoughts = thoughts; } private requireRecoveryString(value: string | undefined, field: string): string { @@ -931,16 +976,18 @@ export class NarrativeTurnState { * a turn with narrative but no assistant body still earns a persisted row. */ hasBufferedNarrative(threadId: string): boolean { + this.assertThread(threadId); return this.narrativeBufferStates(threadId).some(Boolean); } private narrativeBufferStates(threadId: string): boolean[] { + this.assertThread(threadId); return [ - (this.turnToolCalls.get(threadId)?.length ?? 0) > 0, - Boolean(this.turnOpenThought.get(threadId)), - (this.turnThoughts.get(threadId)?.length ?? 0) > 0, - (this.turnOpenHooks.get(threadId)?.size ?? 0) > 0, - (this.turnHooks.get(threadId)?.length ?? 0) > 0, + this.turnToolCalls.length > 0, + Boolean(this.turnOpenThought), + this.turnThoughts.length > 0, + this.turnOpenHooks.size > 0, + this.turnHooks.length > 0, ]; } @@ -953,7 +1000,8 @@ export class NarrativeTurnState { threadId: string, hook: { hookName: string; toolName: string | null; phase: string; payload: string; sortOrder: number }, ): string { - const map = this.turnOpenHooks.get(threadId) ?? new Map(); + this.assertThread(threadId); + const map = this.turnOpenHooks; const id = NodeCrypto.randomUUID(); map.set(hook.hookName, { id, @@ -964,25 +1012,28 @@ export class NarrativeTurnState { startedAt: new Date().toISOString(), sortOrder: hook.sortOrder, }); - this.turnOpenHooks.set(threadId, map); + this.turnOpenHooks = map; return id; } /** Look up (without removing) an open hook by name. */ peekOpenHook(threadId: string, hookName: string): OpenHook | undefined { - return this.turnOpenHooks.get(threadId)?.get(hookName); + this.assertThread(threadId); + return this.turnOpenHooks.get(hookName); } /** Remove an open hook by name (after it has been completed or flushed). */ removeOpenHook(threadId: string, hookName: string): void { - this.turnOpenHooks.get(threadId)?.delete(hookName); + this.assertThread(threadId); + this.turnOpenHooks.delete(hookName); } /** Push a completed hook execution onto the closed-hooks list for persistence. */ pushClosedHook(threadId: string, hook: CreateHookExecutionInput): void { - const list = this.turnHooks.get(threadId) ?? []; + this.assertThread(threadId); + const list = this.turnHooks; list.push(hook); - this.turnHooks.set(threadId, list); + this.turnHooks = list; } /** Settle one turn's buffered records into data-only persistence rows. */ @@ -992,6 +1043,7 @@ export class NarrativeTurnState { messageContent: string, outcome: TurnOutcome, ): PreparedNarrativePersistence { + this.assertThread(threadId); const toolCalls = this.prepareToolCallsForPersistence(threadId, messageId, outcome); this.closeOpenThought(threadId); this.closeOpenHooksForPersistence(threadId); @@ -1005,7 +1057,8 @@ export class NarrativeTurnState { messageId: string, outcome: TurnOutcome, ): BufferedToolCall[] { - const toolCalls = this.turnToolCalls.get(threadId) ?? []; + this.assertThread(threadId); + const toolCalls = this.turnToolCalls; const settledAt = new Date().toISOString(); for (const toolCall of toolCalls) { toolCall.toolCallId ??= NodeCrypto.randomUUID(); @@ -1061,9 +1114,10 @@ export class NarrativeTurnState { } private closeOpenHooksForPersistence(threadId: string): void { - const openHookMap = this.turnOpenHooks.get(threadId); + this.assertThread(threadId); + const openHookMap = this.turnOpenHooks; if (!openHookMap || openHookMap.size === 0) return; - const list = this.turnHooks.get(threadId) ?? []; + const list = this.turnHooks; const endedAt = new Date().toISOString(); for (const open of openHookMap.values()) { list.push({ @@ -1080,7 +1134,7 @@ export class NarrativeTurnState { sortOrder: open.sortOrder, }); } - this.turnHooks.set(threadId, list); + this.turnHooks = list; openHookMap.clear(); } @@ -1089,7 +1143,8 @@ export class NarrativeTurnState { messageId: string, messageContent: string, ): CreateThoughtSegmentInput[] { - const bufferedThoughts = this.turnThoughts.get(threadId) ?? []; + this.assertThread(threadId); + const bufferedThoughts = this.turnThoughts; for (const thought of bufferedThoughts) thought.id ??= NodeCrypto.randomUUID(); const thoughts = bufferedThoughts.map((thought) => ({ ...thought, messageId })); const message = messageContent.trim(); @@ -1127,7 +1182,8 @@ export class NarrativeTurnState { threadId: string, messageId: string, ): CreateHookExecutionInput[] { - const bufferedHooks = this.turnHooks.get(threadId) ?? []; + this.assertThread(threadId); + const bufferedHooks = this.turnHooks; for (const hook of bufferedHooks) hook.id ??= NodeCrypto.randomUUID(); return bufferedHooks.map((hook) => ({ ...hook, messageId })); } @@ -1139,11 +1195,12 @@ export class NarrativeTurnState { * increment the completed turn's counter (mirrors the old clearTurnState). */ clearTurn(threadId: string): void { - this.turnToolCalls.delete(threadId); - this.turnOpenThought.delete(threadId); - this.turnThoughts.delete(threadId); - this.turnOpenHooks.delete(threadId); - this.turnHooks.delete(threadId); + this.assertThread(threadId); + this.turnToolCalls = []; + this.turnOpenThought = null; + this.turnThoughts = []; + this.turnOpenHooks = new Map(); + this.turnHooks = []; } /** Generate a human-readable summary of tool input. */ From bcf1576631e8173543a023194e33746cfd05d6f8 Mon Sep 17 00:00:00 2001 From: "(CJ) Chukwudi Nwobodo" <142016413+chuks-qua@users.noreply.github.com> Date: Thu, 24 Sep 2026 21:35:27 +0100 Subject: [PATCH 067/136] fix(server): avoid retaining empty narrative adapters --- .../__tests__/narrative-store.test.ts | 29 +++++++++++++++++++ .../conversation/narrative/narrative-store.ts | 11 ++++--- 2 files changed, 36 insertions(+), 4 deletions(-) diff --git a/apps/server/src/features/agents/conversation/narrative/__tests__/narrative-store.test.ts b/apps/server/src/features/agents/conversation/narrative/__tests__/narrative-store.test.ts index 7020f2219..7dc10b0ca 100644 --- a/apps/server/src/features/agents/conversation/narrative/__tests__/narrative-store.test.ts +++ b/apps/server/src/features/agents/conversation/narrative/__tests__/narrative-store.test.ts @@ -704,6 +704,35 @@ describe("NarrativeStore write seam (server-side traps)", () => { expect(store.nextSortOrder(THREAD)).toBe(1); }); + it("does not retain empty adapters for completed threads without narrative state", () => { + for (let index = 0; index < 50; index += 1) { + const threadId = `completed-thread-${index}`; + store.updateBufferedToolCallOutput(threadId, "missing-tool", "done", false); + expect(store.stageNarrationSegment(threadId, "")).toBeNull(); + expect(store.prepareNarrativePersistence(threadId, "m1", "", "completed")) + .toEqual({ toolCalls: [], thoughts: [], hooks: [] }); + store.clearTurn(threadId); + } + + expect(Reflect.get(store, "narrativeStates").size).toBe(0); + store.beginTurn("completed-thread-0"); + store.resetTurnCounters("completed-thread-0"); + store.bufferToolCall("completed-thread-0", toolUse("new-tool", "Read")); + expect(store.getBufferedToolCalls("completed-thread-0")).toHaveLength(1); + }); + + it("keeps the completed turn's sort counter for late hooks, then resets it on the next turn", () => { + store.beginTurn(THREAD); + store.resetTurnCounters(THREAD); + store.bufferToolCall(THREAD, toolUse("read-1", "Read")); + store.clearTurn(THREAD); + + expect(store.nextSortOrder(THREAD)).toBe(1); + store.beginTurn(THREAD); + store.resetTurnCounters(THREAD); + expect(store.nextSortOrder(THREAD)).toBe(0); + }); + it("persists a long active turn in order across bounded transactions", async () => { seedAssistantMessage("m1", "done", 1); store.beginTurn(THREAD); diff --git a/apps/server/src/features/agents/conversation/narrative/narrative-store.ts b/apps/server/src/features/agents/conversation/narrative/narrative-store.ts index 0cebbc235..5ed92a688 100644 --- a/apps/server/src/features/agents/conversation/narrative/narrative-store.ts +++ b/apps/server/src/features/agents/conversation/narrative/narrative-store.ts @@ -200,7 +200,7 @@ export class NarrativeStore { } updateBufferedToolCallOutput(...args: Parameters): ReturnType { - return this.stateFor(args[0]).updateBufferedToolCallOutput(...args); + this.narrativeStates.get(args[0])?.updateBufferedToolCallOutput(...args); } clearAgentStackOnMessage(...args: Parameters): ReturnType { @@ -216,11 +216,13 @@ export class NarrativeStore { } stageNarrationSegment(...args: Parameters): ReturnType { - return this.stateFor(args[0]).stageNarrationSegment(...args); + return (this.narrativeStates.get(args[0]) ?? new NarrativeTurnState(args[0])) + .stageNarrationSegment(...args); } recoverySnapshotWithStagedNarration(...args: Parameters): ReturnType { - return this.stateFor(args[0]).recoverySnapshotWithStagedNarration(...args); + return (this.narrativeStates.get(args[0]) ?? new NarrativeTurnState(args[0])) + .recoverySnapshotWithStagedNarration(...args); } applyStagedNarrationSegment(...args: Parameters): ReturnType { @@ -248,7 +250,8 @@ export class NarrativeStore { } prepareNarrativePersistence(...args: Parameters): ReturnType { - return this.stateFor(args[0]).prepareNarrativePersistence(...args); + return (this.narrativeStates.get(args[0]) ?? new NarrativeTurnState(args[0])) + .prepareNarrativePersistence(...args); } clearTurn(...args: Parameters): ReturnType { From 5131b483eb19dbb44768ddae891f2b58ce189c29 Mon Sep 17 00:00:00 2001 From: "(CJ) Chukwudi Nwobodo" <142016413+chuks-qua@users.noreply.github.com> Date: Thu, 24 Sep 2026 21:37:28 +0100 Subject: [PATCH 068/136] feat(server): checkpoint assistant text in semantic writer --- .../canonical-agent-writer-client.test.ts | 47 +++++++++ ...anonical-execution-semantic-writer.test.ts | 95 +++++++++++++++++++ .../canonical-execution-semantic-writer.ts | 77 ++++++++++++++- .../execution/execution-worker-handler.ts | 38 ++++++-- 4 files changed, 245 insertions(+), 12 deletions(-) diff --git a/apps/server/src/features/agents/canonical/__tests__/canonical-agent-writer-client.test.ts b/apps/server/src/features/agents/canonical/__tests__/canonical-agent-writer-client.test.ts index 92a1ea951..3fdf86ed6 100644 --- a/apps/server/src/features/agents/canonical/__tests__/canonical-agent-writer-client.test.ts +++ b/apps/server/src/features/agents/canonical/__tests__/canonical-agent-writer-client.test.ts @@ -10,6 +10,7 @@ import type { CanonicalAgentEventDraft } from "../canonical-agent-boundary.js"; import { CanonicalAgentWriterClient } from "../canonical-agent-writer-client.js"; import type { CanonicalWriterResponse } from "../canonical-agent-writer-protocol.js"; import { ParentAssistantTextCheckpointService } from "../../turns/parent-assistant-text-checkpoint-service.js"; +import type { ExecutionSemanticOperation } from "../../execution/execution-worker-handler.js"; const THREAD_ID = "writer-thread"; const TURN_ID = "writer-turn"; @@ -162,6 +163,52 @@ describe("canonical SQLite writer", () => { expect(db.prepare("SELECT COUNT(*) AS count FROM canonical_writer_operation_receipts").get()).toEqual({ count: 1 }); }); + it("replays durable semantic assistant text after a worker reply is lost", async () => { + const execution = { threadId: THREAD_ID, turnId: TURN_ID, executionId: EXECUTION_ID }; + const lease = { ownerEpoch: 1, workerIndex: 0, workerGeneration: 1, leaseId: "text-lease" }; + const start: ExecutionSemanticOperation = { + operationId: "text-lease:1", execution, lease, ordinal: 1, + mutation: { + kind: "begin", providerId: "codex", + input: { + thread: { id: THREAD_ID, workspaceId: "writer-workspace", providerId: "codex", createdAt: NOW }, + turnId: TURN_ID, executionId: EXECUTION_ID, + permissionMode: "supervised", providerIdentities: [], + userMessage: { kind: "create", messageId: "text-user", content: "Question", sequence: 1 }, + }, + }, + }; + writer = new CanonicalAgentWriterClient(dbPath); + expect(await writer.transactSemantic(start, () => {})).toMatchObject({ kind: "committed" }); + await writer.close(); + + let created = 0; + writer = new CanonicalAgentWriterClient(dbPath, () => created++ === 0 + ? workerDroppingReply("semantic-transacted") + : new Worker(new URL("../canonical-agent-writer.worker.ts", import.meta.url), { type: "module" })); + const textOperation: ExecutionSemanticOperation = { + operationId: "text-lease:2", execution, lease, ordinal: 2, + mutation: { kind: "append-assistant-text", inputs: [{ ...execution, sequence: 1, text: "Worker durable text" }] }, + }; + const receipt = await writer.transactSemantic(textOperation, () => {}); + expect(receipt).toMatchObject({ kind: "committed", operationId: "text-lease:2", + assistantTextCheckpoint: { outcome: "committed", durableThrough: 1 } }); + expect(created).toBe(2); + expect(new ParentAssistantTextCheckpointService(db).restore(EXECUTION_ID)).toBe("Worker durable text"); + expect(db.prepare("SELECT COUNT(*) AS count FROM parent_assistant_text_checkpoint_chunks WHERE execution_id = ?") + .get(EXECUTION_ID)).toEqual({ count: 1 }); + await expect(NodeFSPromises.stat(NodePath.join(tempDir, "app.sqlite.recovery", "parent-assistant-text"))) + .rejects.toMatchObject({ code: "ENOENT" }); + await writer.close(); + + writer = new CanonicalAgentWriterClient(dbPath); + expect(await writer.transactSemantic(textOperation, () => {})).toEqual(receipt); + expect(await writer.interruptWorkerLoss({ execution, lease, reason: "worker exited", recoveryIncidentId: "text-loss" }, + () => {})).toMatchObject({ kind: "committed", operationId: "text-lease:worker-lost" }); + expect(db.prepare("SELECT content FROM messages WHERE role = 'assistant' AND is_internal = 0").get()) + .toEqual({ content: "Worker durable text" }); + }); + it("rejects a database failure without reporting a durable receipt", async () => { writer = new CanonicalAgentWriterClient(dbPath); await writer.whenReady(); diff --git a/apps/server/src/features/agents/canonical/__tests__/canonical-execution-semantic-writer.test.ts b/apps/server/src/features/agents/canonical/__tests__/canonical-execution-semantic-writer.test.ts index f667b26b4..a13cb53ec 100644 --- a/apps/server/src/features/agents/canonical/__tests__/canonical-execution-semantic-writer.test.ts +++ b/apps/server/src/features/agents/canonical/__tests__/canonical-execution-semantic-writer.test.ts @@ -152,6 +152,101 @@ describe("CanonicalExecutionSemanticWriter through ExecutionWorkerHandler", () = expect(published).toContain(`${EXECUTION_ID}:recovery-interrupted`); }); + it("keeps an assistant-text receipt and text across writer restart and worker loss", async () => { + expect((await send(1, { kind: "start", providerId: "codex", input: startInput() })).kind).toBe("committed"); + const inputs = [{ ...execution, sequence: 1, text: "Durable partial answer" }]; + const textOperation = operation(2, { kind: "append-assistant-text", inputs }); + const appended = await send(2, { kind: "assistant-text", inputs }); + expect(appended).toMatchObject({ + kind: "committed", operationId: "lease-1:2", + assistantTextCheckpoint: { outcome: "committed", durableThrough: 1, committedItems: 1 }, + }); + expect(new ParentAssistantTextCheckpointService(db).restore(EXECUTION_ID)).toBe("Durable partial answer"); + + db.close(true); + db = openDatabase({ dbPath: path }); + writer = new CanonicalExecutionSemanticWriter(db, () => {}); + expect(await writer.transact(textOperation)).toEqual(appended); + expect(new ParentAssistantTextCheckpointService(db).restore(EXECUTION_ID)).toBe("Durable partial answer"); + expect(db.prepare("SELECT COUNT(*) AS count FROM parent_assistant_text_checkpoint_chunks WHERE execution_id = ?") + .get(EXECUTION_ID)).toEqual({ count: 1 }); + expect(writer.interruptWorkerLoss(loss).kind).toBe("committed"); + expect(new MessageRepo(db).listIncludingInternal(THREAD_ID)).toContainEqual(expect.objectContaining({ + role: "assistant", content: "Durable partial answer", outcome: "interrupted", + })); + }); + + it("cancels after a text checkpoint and retires it only after terminal commit", async () => { + expect((await send(1, { kind: "start", providerId: "codex", input: startInput() })).kind).toBe("committed"); + expect((await send(2, { kind: "assistant-text", inputs: [ + { ...execution, sequence: 1, text: "Saved before cancellation" }, + ] })).kind).toBe("committed"); + expect((await handler.handle({ requestId: 3, execution, lease, ordinal: 3, stopWatermark: 2, + command: { kind: "stop", requestId: "cancel-1" } })).result.kind).toBe("committed"); + expect((await send(4, { kind: "provider-outcome", outcome: "cancelled" })).kind).toBe("committed"); + expect((await send(5, { kind: "stage-terminal", input: { + threadId: THREAD_ID, executionId: EXECUTION_ID, outcome: "cancelled", endedAt: NOW, + assistant: { content: "Saved before cancellation", model: null, attachments: [] }, narrative: [], + } })).kind).toBe("committed"); + expect(new ParentAssistantTextCheckpointService(db).restore(EXECUTION_ID)).toBe("Saved before cancellation"); + const finish = operation(6, { kind: "finish", outcome: "cancelled", input: { + threadId: THREAD_ID, turnId: TURN_ID, executionId: EXECUTION_ID, + providerId: "codex", providerIdentities: [], outcome: "cancelled", projection: { kind: "writer-staged" }, + } }); + db.run("CREATE TRIGGER fail_text_retirement BEFORE DELETE ON parent_assistant_text_checkpoints BEGIN SELECT RAISE(ABORT, 'retirement unavailable'); END"); + await expect(writer.transact(finish)).rejects.toThrow("retirement unavailable"); + expect(db.prepare("SELECT terminal_outcome FROM canonical_agent_ingest_checkpoints WHERE execution_id = ?") + .get(EXECUTION_ID)).toEqual({ terminal_outcome: "cancelled" }); + expect(db.prepare("SELECT COUNT(*) AS count FROM parent_assistant_text_checkpoints WHERE execution_id = ?") + .get(EXECUTION_ID)).toEqual({ count: 1 }); + db.run("DROP TRIGGER fail_text_retirement"); + expect((await writer.transact(finish)).kind).toBe("committed"); + expect(db.prepare("SELECT COUNT(*) AS count FROM parent_assistant_text_checkpoints WHERE execution_id = ?") + .get(EXECUTION_ID)).toEqual({ count: 0 }); + expect(new MessageRepo(db).listIncludingInternal(THREAD_ID)).toContainEqual(expect.objectContaining({ + role: "assistant", content: "Saved before cancellation", outcome: "cancelled", is_internal: false, + })); + expect((await writer.transact(finish)).kind).toBe("committed"); + }); + + it("fences assistant-text routing, lease, ordinal, input size, and conflicting replay", async () => { + expect((await send(1, { kind: "start", providerId: "codex", input: startInput() })).kind).toBe("committed"); + const inputs = [{ ...execution, sequence: 1, text: "Saved text" }]; + const appendText = operation(2, { kind: "append-assistant-text", inputs }); + expect(await writer.transact({ ...appendText, lease: { ...lease, ownerEpoch: 2, leaseId: "lease-2" }, + operationId: "lease-2:2" })).toEqual({ kind: "conflict", operationId: "lease-2:2" }); + expect(await writer.transact({ ...appendText, ordinal: 3, operationId: "lease-1:3" })) + .toEqual({ kind: "conflict", operationId: "lease-1:3" }); + expect(await writer.transact(operation(2, { kind: "append-assistant-text", + inputs: [{ ...inputs[0]!, threadId: "wrong-thread" }], + }))).toEqual({ kind: "conflict", operationId: "lease-1:2" }); + expect(await writer.transact(operation(2, { kind: "append-assistant-text", + inputs: [{ ...inputs[0]!, text: "x".repeat(16 * 1024 + 1) }], + }))).toEqual({ kind: "conflict", operationId: "lease-1:2" }); + expect(await writer.transact(appendText)).toMatchObject({ kind: "committed", operationId: "lease-1:2" }); + expect(await writer.transact(operation(2, { kind: "append-assistant-text", + inputs: [{ ...inputs[0]!, text: "Changed text" }], + }))).toEqual({ kind: "conflict", operationId: "lease-1:2" }); + expect(await writer.transact(operation(3, { kind: "append-assistant-text", inputs }))) + .toEqual({ kind: "conflict", operationId: "lease-1:3" }); + expect(new ParentAssistantTextCheckpointService(db).restore(EXECUTION_ID)).toBe("Saved text"); + }); + + it("rolls back assistant text if its semantic receipt cannot commit", async () => { + expect((await send(1, { kind: "start", providerId: "codex", input: startInput() })).kind).toBe("committed"); + db.run("CREATE TRIGGER fail_text_receipt BEFORE INSERT ON canonical_writer_operation_receipts WHEN NEW.kind = 'semantic:append-assistant-text' BEGIN SELECT RAISE(ABORT, 'text receipt unavailable'); END"); + const input = operation(2, { kind: "append-assistant-text", + inputs: [{ ...execution, sequence: 1, text: "Keep me out until receipt" }], + }); + await expect(writer.transact(input)).rejects.toThrow("text receipt unavailable"); + expect(db.prepare("SELECT COUNT(*) AS count FROM parent_assistant_text_checkpoint_chunks").get()) + .toEqual({ count: 0 }); + expect(db.prepare("SELECT receipt_json FROM canonical_writer_operation_receipts WHERE execution_id = ? AND operation_id = ?") + .get(EXECUTION_ID, "semantic:head")).toMatchObject({ receipt_json: expect.stringContaining('"ordinal":1') }); + db.run("DROP TRIGGER fail_text_receipt"); + expect(await writer.transact(input)).toMatchObject({ kind: "committed", operationId: "lease-1:2" }); + }); + it("rejects stale worker-loss leases without changing the unfinished turn", async () => { expect((await send(1, { kind: "start", providerId: "codex", input: startInput() })).kind).toBe("committed"); expect(writer.interruptWorkerLoss({ ...loss, lease: { ...lease, ownerEpoch: 2 } })) diff --git a/apps/server/src/features/agents/canonical/canonical-execution-semantic-writer.ts b/apps/server/src/features/agents/canonical/canonical-execution-semantic-writer.ts index 9463e23ed..71e010c98 100644 --- a/apps/server/src/features/agents/canonical/canonical-execution-semantic-writer.ts +++ b/apps/server/src/features/agents/canonical/canonical-execution-semantic-writer.ts @@ -21,6 +21,12 @@ import type { CanonicalAgentEventPublisher } from "./canonical-agent-boundary.js import { CanonicalAgentBoundary } from "./canonical-agent-boundary.js"; import { CanonicalCommittedProviderProjector } from "./canonical-committed-provider-projector.js"; import { CanonicalParentTurnWrite } from "./canonical-parent-turn-write.js"; +import { + ParentAssistantTextCheckpointService, + PARENT_ASSISTANT_TEXT_QUEUE_POLICY, + type ParentAssistantTextCheckpointInput, + type ParentAssistantTextCheckpointResult, +} from "../turns/parent-assistant-text-checkpoint-service.js"; const HEAD_ID = "semantic:head"; const HEAD_KIND = "semantic-head"; @@ -76,6 +82,12 @@ const storedReceiptSchema = z.object({ eventCount: z.number().int().nonnegative(), }).optional(), providerEvents: z.array(projectedProviderEventSchema).optional(), + assistantTextCheckpoint: z.object({ + outcome: z.literal("committed"), + durableThrough: z.number().int().nonnegative(), + committedItems: z.number().int().positive(), + committedBytes: z.number().int().positive(), + }).optional(), }); const storedOperationSchema = z.object({ kind: z.string(), @@ -115,6 +127,7 @@ export interface LostExecutionInterruption { export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter { private readonly turns: CanonicalParentTurnWrite; private readonly providerProjector: CanonicalCommittedProviderProjector; + private readonly assistantText: ParentAssistantTextCheckpointService; private readonly findOperation: ReturnType; private readonly listPublicationChunks: ReturnType; private readonly findPublishedEvent: ReturnType; @@ -137,6 +150,7 @@ export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter this.bufferPublication(events); }); this.providerProjector = new CanonicalCommittedProviderProjector(codexBoundary); + this.assistantText = new ParentAssistantTextCheckpointService(db); this.findOperation = db.prepare("SELECT kind, input_hash, receipt_json FROM canonical_writer_operation_receipts WHERE execution_id = ? AND operation_id = ?"); this.listPublicationChunks = db.prepare("SELECT operation_id, kind, input_hash, receipt_json FROM canonical_writer_operation_receipts WHERE execution_id = ? AND operation_id > ? AND operation_id < ? ORDER BY operation_id LIMIT ?"); this.findPublishedEvent = db.prepare("SELECT envelope_json FROM canonical_agent_events WHERE execution_id = ? AND accepted_sequence = ?"); @@ -153,7 +167,12 @@ export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter if (!validOperation(operation)) return conflict(operation); const hash = fingerprint(operation); const existing = this.existingReceipt(operation, hash); - if (existing) return existing; + if (existing) { + if (existing.kind === "committed" && operation.mutation.kind === "finish") { + this.assistantText.retire(operation.execution.executionId); + } + return existing; + } try { return await this.applySupported(operation, hash); } catch (error) { @@ -212,6 +231,7 @@ export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter switch (operation.mutation.kind) { case "begin": return this.begin(operation, hash); case "append-events": return this.append(operation, hash); + case "append-assistant-text": return this.appendAssistantText(operation, hash); case "checkpoint": case "stop-requested": case "provider-outcome": return this.control(operation, hash); @@ -299,6 +319,21 @@ export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter })()); } + private appendAssistantText(operation: ExecutionSemanticOperation, hash: string): ExecutionWriteReceipt { + const mutation = operation.mutation; + if (mutation.kind !== "append-assistant-text") return conflict(operation); + return this.db.transaction(() => { + const head = this.requireNextHead(operation); + const result = this.assistantText.appendChunk(mutation.inputs); + if (result.outcome !== "committed") throw new SemanticConflict(); + // Assistant text has its own durable sequence; it does not advance the canonical event revision. + const receipt = committed(operation, head.durableRevision, undefined, undefined, result); + this.storeHead({ ...head, ordinal: operation.ordinal }); + this.storeReceipt(operation, hash, receipt); + return receipt; + })(); + } + private async finish(operation: ExecutionSemanticOperation, hash: string): Promise { const mutation = operation.mutation; if (mutation.kind !== "finish" || !validFinish(operation, mutation)) return conflict(operation); @@ -319,7 +354,10 @@ export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter }); if (result.outcome !== "committed") return conflict(operation); const stored = this.loadOperation(operation.execution.executionId, operation.operationId); - return stored ? this.readReceipt(operation, hash, stored) : conflict(operation); + if (!stored) return conflict(operation); + const receipt = this.readReceipt(operation, hash, stored); + if (receipt.kind === "committed") this.assistantText.retire(operation.execution.executionId); + return receipt; } private stageTerminal(operation: ExecutionSemanticOperation, hash: string): ExecutionWriteReceipt { @@ -480,7 +518,8 @@ export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter if (stored.kind !== `semantic:${operation.mutation.kind}` || stored.input_hash !== hash) return conflict(operation); const receipt = storedReceiptSchema.parse(JSON.parse(stored.receipt_json)); return receipt.operationId === operation.operationId && Number.isSafeInteger(receipt.durableRevision) - ? committed(operation, receipt.durableRevision, receipt.providerCommit, receipt.providerEvents) : conflict(operation); + ? committed(operation, receipt.durableRevision, receipt.providerCommit, receipt.providerEvents, + receipt.assistantTextCheckpoint) : conflict(operation); } private publishStoredEvents(executionId: string, hash: string): void { @@ -539,7 +578,35 @@ function validOperation(operation: ExecutionSemanticOperation): boolean { return operation.operationId === `${operation.lease.leaseId}:${operation.ordinal}` && operation.operationId !== HEAD_ID && operation.operationId.length <= 256 && Number.isSafeInteger(operation.ordinal) && operation.ordinal > 0 - && validIdentity(operation.execution) && validLease(operation.lease); + && validIdentity(operation.execution) && validLease(operation.lease) + && (operation.mutation.kind !== "append-assistant-text" + || validAssistantTextInput(operation.mutation.inputs, operation.execution)); +} + +function validAssistantTextInput( + inputs: readonly ParentAssistantTextCheckpointInput[], + execution: ExecutionIdentity, +): boolean { + if (!Array.isArray(inputs) || inputs.length === 0 + || inputs.length > PARENT_ASSISTANT_TEXT_QUEUE_POLICY.maxQueuedEvents) return false; + let bytes = 0; + const first = inputs[0]; + if (!first || !validAssistantTextEntry(first, execution)) return false; + const firstSequence = first.sequence; + for (const [index, input] of inputs.entries()) { + if (!validAssistantTextEntry(input, execution) || input.sequence !== firstSequence + index) return false; + bytes += Buffer.byteLength(input.text, "utf8"); + if (bytes > PARENT_ASSISTANT_TEXT_QUEUE_POLICY.maxChunkBytes) return false; + } + return bytes > 0; +} + +function validAssistantTextEntry(input: ParentAssistantTextCheckpointInput | undefined, execution: ExecutionIdentity): boolean { + if (!input) return false; + return input.executionId === execution.executionId && input.threadId === execution.threadId + && input.turnId === execution.turnId && typeof input.text === "string" + && Buffer.byteLength(input.text, "utf8") > 0 + && Number.isSafeInteger(input.sequence) && input.sequence > 0; } function validIdentity(identity: ExecutionIdentity): boolean { @@ -598,11 +665,13 @@ function committed( durableRevision: number, providerCommit?: ExecutionProviderCommitReceipt, providerEvents?: readonly ProjectedCommittedProviderEvent[], + assistantTextCheckpoint?: ParentAssistantTextCheckpointResult, ): Extract { return { kind: "committed", operationId: operation.operationId, durableRevision, ...(providerCommit ? { providerCommit } : {}), ...(providerEvents ? { providerEvents } : {}), + ...(assistantTextCheckpoint ? { assistantTextCheckpoint } : {}), }; } diff --git a/apps/server/src/features/agents/execution/execution-worker-handler.ts b/apps/server/src/features/agents/execution/execution-worker-handler.ts index 511c88bc7..075089564 100644 --- a/apps/server/src/features/agents/execution/execution-worker-handler.ts +++ b/apps/server/src/features/agents/execution/execution-worker-handler.ts @@ -8,6 +8,10 @@ import type { } from "../canonical/canonical-parent-turn-write.js"; import type { CanonicalAgentCommitResult } from "../canonical/canonical-agent-boundary.js"; import type { ProviderEventIngressEvent } from "../../providers/composition/provider-event-ingress.js"; +import type { + ParentAssistantTextCheckpointInput, + ParentAssistantTextCheckpointResult, +} from "../turns/parent-assistant-text-checkpoint-service.js"; import type { ExecutionIdentity, ExecutionLease, @@ -21,6 +25,7 @@ export type ExecutionWorkCommand = | { readonly kind: "start"; readonly providerId: string; readonly input: DataOnlyParentTurnStartInput } | { readonly kind: "resume"; readonly providerId: string; readonly checkpointId: string } | { readonly kind: "event"; readonly phase: string; readonly nativeCursor: unknown | null; readonly events: readonly ProviderEventDraft[] } + | { readonly kind: "assistant-text"; readonly inputs: readonly ParentAssistantTextCheckpointInput[] } | { readonly kind: "checkpoint"; readonly phase: string; readonly nativeCursor: unknown | null } | { readonly kind: "effect-result"; readonly effectId: string; readonly settled: boolean } | { readonly kind: "provider-outcome"; readonly outcome: TurnOutcome } @@ -38,6 +43,7 @@ export interface ExecutionSemanticOperation { | { readonly kind: "begin"; readonly providerId: string; readonly input: DataOnlyParentTurnStartInput } | { readonly kind: "resume"; readonly providerId: string; readonly checkpointId: string } | { readonly kind: "append-events"; readonly phase: string; readonly nativeCursor: unknown | null; readonly events: readonly ProviderEventDraft[] } + | { readonly kind: "append-assistant-text"; readonly inputs: readonly ParentAssistantTextCheckpointInput[] } | { readonly kind: "checkpoint"; readonly phase: string; readonly nativeCursor: unknown | null } | { readonly kind: "stop-requested"; readonly requestId: string; readonly lastAdmittedOrdinal: number } | { readonly kind: "effect-result"; readonly effectId: string; readonly settled: boolean } @@ -64,7 +70,7 @@ export type ProjectedCommittedProviderEvent = Omit< /** A writer reply is valid only after the semantic operation commits durably. */ export type ExecutionWriteReceipt = - | { readonly kind: "committed"; readonly operationId: string; readonly durableRevision: number; readonly providerCommit?: ExecutionProviderCommitReceipt; readonly providerEvents?: readonly ProjectedCommittedProviderEvent[] } + | { readonly kind: "committed"; readonly operationId: string; readonly durableRevision: number; readonly providerCommit?: ExecutionProviderCommitReceipt; readonly providerEvents?: readonly ProjectedCommittedProviderEvent[]; readonly assistantTextCheckpoint?: ParentAssistantTextCheckpointResult } | { readonly kind: "conflict"; readonly operationId: string; readonly recoveryState?: "not-started" | "already-terminal" }; /** @@ -78,9 +84,9 @@ export interface ExecutionSemanticWriter { /** A command result that never calls an uncommitted mutation successful. */ export type ExecutionWorkerResult = - | { readonly kind: "committed"; readonly operationId: string; readonly durableRevision: number; readonly providerCommit?: ExecutionProviderCommitReceipt; readonly providerEvents?: readonly ProjectedCommittedProviderEvent[] } + | { readonly kind: "committed"; readonly operationId: string; readonly durableRevision: number; readonly providerCommit?: ExecutionProviderCommitReceipt; readonly providerEvents?: readonly ProjectedCommittedProviderEvent[]; readonly assistantTextCheckpoint?: ParentAssistantTextCheckpointResult } | { readonly kind: "released" } - | { readonly kind: "rejected"; readonly reason: "no-execution" | "stale-execution" | "out-of-order" | "invalid-transition" | "invalid-event-routing" | "invalid-stop-watermark" | "writer-conflict" }; + | { readonly kind: "rejected"; readonly reason: "no-execution" | "stale-execution" | "out-of-order" | "invalid-transition" | "invalid-event-routing" | "invalid-text-routing" | "invalid-stop-watermark" | "writer-conflict" }; type WorkerCommand = ExecutionMailboxCommand; const METADATA_MUTATIONS: ReadonlySet = new Set([ @@ -189,7 +195,6 @@ function mutationFor( state: ExecutionState, ): ExecutionSemanticOperation["mutation"] | undefined { const command = request.command; - if (isMetadataMutation(command)) return command; switch (command.kind) { case "event": return eventMutation(command, request.execution, state); @@ -201,13 +206,20 @@ function mutationFor( case "resume": case "release": return undefined; - default: { - const exhaustive: never = command; - return exhaustive; - } + default: return metadataOrTextMutation(command, request.execution, state); } } +function metadataOrTextMutation( + command: Extract, + execution: ExecutionIdentity, + state: ExecutionState, +): ExecutionSemanticOperation["mutation"] | undefined { + if (isMetadataMutation(command)) return command; + return state.phase === "running" && validTextRouting(command.inputs, execution) + ? { kind: "append-assistant-text", inputs: command.inputs } : undefined; +} + function isMetadataMutation(command: WorkerCommand): command is Extract< WorkerCommand, { readonly kind: "checkpoint" | "effect-result" | "provider-outcome" | "stage-terminal" } @@ -256,6 +268,9 @@ function invalidReason(request: ExecutionWorkerRequest): Extract< if (request.command.kind === "event" && !validEventRouting(request.command.events, request.execution)) { return "invalid-event-routing"; } + if (request.command.kind === "assistant-text" && !validTextRouting(request.command.inputs, request.execution)) { + return "invalid-text-routing"; + } return "invalid-transition"; } @@ -264,6 +279,12 @@ function validEventRouting(events: readonly ProviderEventDraft[], execution: Exe && event.routing.turnId === execution.turnId && event.routing.executionId === execution.executionId); } +function validTextRouting(inputs: readonly ParentAssistantTextCheckpointInput[], execution: ExecutionIdentity): boolean { + return Array.isArray(inputs) && inputs.length > 0 && inputs.every((input) => input + && input.threadId === execution.threadId + && input.turnId === execution.turnId && input.executionId === execution.executionId); +} + function validStartInput( command: Extract, execution: ExecutionIdentity, @@ -317,6 +338,7 @@ function committedResult(receipt: Extract Date: Thu, 24 Sep 2026 21:37:46 +0100 Subject: [PATCH 069/136] feat(server): hand off file turn identity to execution tracker --- .../turns/__tests__/turn-file-tracker.test.ts | 47 +++++++ .../agents/turns/turn-file-tracker.ts | 119 +++++++++++++++--- 2 files changed, 151 insertions(+), 15 deletions(-) diff --git a/apps/server/src/features/agents/turns/__tests__/turn-file-tracker.test.ts b/apps/server/src/features/agents/turns/__tests__/turn-file-tracker.test.ts index c03f86950..025fa80b0 100644 --- a/apps/server/src/features/agents/turns/__tests__/turn-file-tracker.test.ts +++ b/apps/server/src/features/agents/turns/__tests__/turn-file-tracker.test.ts @@ -108,6 +108,53 @@ describe("TurnFileTracker", () => { expect(updates.at(-1)).toEqual(summary); }); + it("transfers one execution's pre-edit capture to a separate tracker after a real edit", async () => { + const root = await tempDir("mcode-transferred-file-observation-"); + const path = NodePath.join(root, "tracked.txt"); + await NodeFSPromises.writeFile(path, "before\n"); + const baseline = async () => ({ kind: "unavailable" as const }); + const host = new TurnFileTracker(baseline, () => {}, TEST_PLATFORM); + const worker = new TurnFileTracker(baseline, () => {}, TEST_PLATFORM); + const expected = { threadId: "t", executionId: "execution-1", cwd: root }; + const handoff = host.beginExecutionTurn({ ...expected, baselineRef: null }); + const event = { threadId: "t", toolCallId: "edit", toolName: "Edit", toolInput: { file_path: "tracked.txt" } }; + const captured = host.captureToolUseObservation(event.threadId, event.toolCallId, event.toolName, event.toolInput); + if (!captured) throw new Error("Expected a pre-edit capture"); + + await NodeFSPromises.writeFile(path, "after\nextra\n"); + expect(worker.beginTurnFromHandoff(structuredClone(handoff), expected)).toBe(true); + expect(worker.beginTurnFromHandoff({ ...handoff, baselineRef: "other-ref" }, expected)).toBe(false); + await worker.setBaselineRef("t", handoff.generation, "later-ref"); + expect(worker.beginTurnFromHandoff(structuredClone(handoff), expected)).toBe(true); + expect(await worker.observeCapturedToolUse(event, { ...captured, executionId: "other-execution" })).toBe(false); + expect(await worker.observeCapturedToolUse(event, structuredClone(captured))).toBe(true); + await worker.observeToolResult("t", "edit"); + expect(await worker.finalizeTurn("t")).toMatchObject({ fileCount: 1, additions: 2, deletions: 1 }); + }); + + it("rejects a handoff or capture for another execution or root", async () => { + const root = await tempDir("mcode-transferred-file-stale-"); + const otherRoot = await tempDir("mcode-transferred-file-other-root-"); + const baseline = async () => ({ kind: "unavailable" as const }); + const oldHost = new TurnFileTracker(baseline, () => {}, TEST_PLATFORM); + const currentHost = new TurnFileTracker(baseline, () => {}, TEST_PLATFORM); + const worker = new TurnFileTracker(baseline, () => {}, TEST_PLATFORM); + const event = { threadId: "t", toolCallId: "edit", toolName: "Edit", toolInput: { file_path: "tracked.txt" } }; + oldHost.beginExecutionTurn({ threadId: "t", executionId: "old-execution", cwd: root, baselineRef: null }); + const oldCapture = oldHost.captureToolUseObservation(event.threadId, event.toolCallId, event.toolName, event.toolInput); + if (!oldCapture) throw new Error("Expected an old capture"); + const expected = { threadId: "t", executionId: "current-execution", cwd: root }; + const handoff = currentHost.beginExecutionTurn({ ...expected, baselineRef: null }); + + expect(worker.beginTurnFromHandoff(handoff, { ...expected, executionId: "old-execution" })).toBe(false); + expect(worker.beginTurnFromHandoff(handoff, { ...expected, cwd: otherRoot })).toBe(false); + expect(worker.beginTurnFromHandoff({ ...handoff, unrelatedThread: "secret" }, expected)).toBe(false); + expect(worker.beginTurnFromHandoff(handoff, expected)).toBe(true); + expect(await worker.observeCapturedToolUse(event, oldCapture)).toBe(false); + expect(await worker.observeCapturedToolUse(event, { ...oldCapture, executionId: expected.executionId })).toBe(false); + expect(await worker.finalizeTurn("t")).toMatchObject({ fileCount: 0 }); + }); + it("rejects a captured baseline from another tool, root, or turn generation", async () => { const root = await tempDir("mcode-stale-observation-"); const path = NodePath.join(root, "tracked.txt"); diff --git a/apps/server/src/features/agents/turns/turn-file-tracker.ts b/apps/server/src/features/agents/turns/turn-file-tracker.ts index 1c701825d..dcfdd25d8 100644 --- a/apps/server/src/features/agents/turns/turn-file-tracker.ts +++ b/apps/server/src/features/agents/turns/turn-file-tracker.ts @@ -5,6 +5,7 @@ import * as NodePath from "node:path"; import { createTextPatch } from "@mcode/shared"; import type { FileEffect, TurnFileEffectSummary } from "@mcode/contracts"; import { MAX_TURN_FILE_EFFECTS } from "@mcode/contracts"; +import { z } from "zod"; import { normalizeFilesystemPath } from "../../../shared/filesystem/path-identity.js"; const MAX_FILE_BYTES = 1_048_576; @@ -41,6 +42,7 @@ interface CandidateObservation { /** Plain, bounded pre-edit evidence that can cross a worker message boundary. */ export interface CapturedToolUseObservation { readonly threadId: string; + readonly executionId: string | null; readonly toolCallId: string; readonly generation: number; readonly generationToken: string; @@ -83,11 +85,13 @@ interface TrackedPath { interface TurnState { reconstructionRejected?: boolean; + executionId: string | null; generation: number; generationToken: string; cwd: string; canonicalRoot: string; baselineRef: string | null; + initialBaselineRef: string | null; revision: number; tracked: Map; summary: TurnFileEffectSummary; @@ -111,6 +115,27 @@ export type TurnBaselineReader = ( /** Injection token for the composed live file-effect tracker. */ export const TURN_FILE_TRACKER = "TurnFileTracker"; +const fileTurnHandoffSchema = z.object({ + threadId: z.string().min(1), + executionId: z.string().min(1), + cwd: z.string().min(1).max(4096), + canonicalRoot: z.string().min(1).max(4096), + baselineRef: z.string().nullable(), + generation: z.number().int().positive().max(Number.MAX_SAFE_INTEGER - 1), + generationToken: z.string().uuid(), +}).strict(); + +/** One execution's cloneable file-tracker start state, with no other turn data. */ +export type FileTurnHandoff = Readonly>; +type FileTurnStart = Omit & { readonly executionId: string | null }; + +/** Identity supplied by the scheduler, independently of the handoff payload. */ +export interface ExpectedFileExecution { + readonly threadId: string; + readonly executionId: string; + readonly cwd: string; +} + /** * Tracks bounded, explicit agent file mutations and reduces them to net turn effects. * Git supplies the immutable in-project baseline; external paths use the state observed @@ -131,28 +156,60 @@ export class TurnFileTracker { /** Start a fresh tracker scope for an agent turn. */ beginTurn(threadId: string, cwd: string, baselineRef: string | null): number { + return this.openTurn(threadId, cwd, baselineRef, null).generation; + } + + /** Start one execution and return only the identity needed by its worker tracker. */ + beginExecutionTurn(input: ExpectedFileExecution & { readonly baselineRef: string | null }): FileTurnHandoff { + if (!input.threadId || !input.executionId) throw new Error("File turn execution identity is required"); + const turn = this.openTurn(input.threadId, input.cwd, input.baselineRef, input.executionId); + return Object.freeze({ + threadId: input.threadId, executionId: input.executionId, cwd: turn.cwd, + canonicalRoot: turn.canonicalRoot, baselineRef: turn.baselineRef, + generation: turn.generation, generationToken: turn.generationToken, + }); + } + + /** Open the same execution in a separate tracker after checking its scheduled identity and root. */ + beginTurnFromHandoff(value: unknown, expected: ExpectedFileExecution): boolean { + const parsed = fileTurnHandoffSchema.safeParse(value); + if (!parsed.success) return false; + const handoff = parsed.data; + if (handoff.threadId !== expected.threadId || handoff.executionId !== expected.executionId) return false; + const canonicalRoot = canonicalWorkingRoot(expected.cwd, this.platform); + if (!canonicalRoot || canonicalRoot !== handoff.canonicalRoot + || canonicalWorkingRoot(handoff.cwd, this.platform) !== canonicalRoot) return false; + const current = this.getTurn(handoff.threadId); + if (current) return sameFileTurnHandoff(current, handoff); + if (this.turns.get(handoff.threadId)?.has(handoff.generation)) return false; + this.installTurn(handoff.threadId, newTurnState(handoff)); + this.nextGeneration = Math.max(this.nextGeneration, handoff.generation + 1); + return true; + } + + private openTurn(threadId: string, cwd: string, baselineRef: string | null, executionId: string | null): TurnState { const canonicalRoot = normalizeFilesystemPath(NodeFS.realpathSync.native(cwd), this.platform); - const generation = this.nextGeneration++; + const start = { + threadId, executionId, cwd, canonicalRoot, baselineRef, + generation: this.nextGeneration, generationToken: NodeCrypto.randomUUID(), + }; + if (executionId !== null) fileTurnHandoffSchema.parse(start); + this.nextGeneration += 1; + const turn = newTurnState(start); + this.installTurn(threadId, turn); + return turn; + } + + private installTurn(threadId: string, turn: TurnState): void { const generations = this.turns.get(threadId) ?? new Map(); - generations.set(generation, { - generation, - generationToken: NodeCrypto.randomUUID(), - cwd, - canonicalRoot, - baselineRef, - revision: 0, - tracked: new Map(), - summary: EMPTY_SUMMARY, - chain: Promise.resolve(), - }); + generations.set(turn.generation, turn); while (generations.size > 4) { const oldest = generations.keys().next().value as number | undefined; if (oldest === undefined) break; generations.delete(oldest); } this.turns.set(threadId, generations); - this.currentGeneration.set(threadId, generation); - return generation; + this.currentGeneration.set(threadId, turn.generation); } /** Return the authoritative tracker generation for the active turn. */ @@ -194,7 +251,8 @@ export class TurnFileTracker { if (candidates.length === 0) return null; const observations = this.synchronousObservations(turn, candidates).map(freezeCandidateObservation); return Object.freeze({ - threadId, toolCallId, generation, generationToken: turn.generationToken, + threadId, toolCallId, executionId: turn.executionId, + generation, generationToken: turn.generationToken, canonicalRoot: turn.canonicalRoot, candidateIdentity: mutationCandidateIdentity(toolName, candidates), observations: Object.freeze(observations), @@ -724,6 +782,36 @@ function boundedMutationCandidates(toolName: string, toolInput: Record Date: Thu, 24 Sep 2026 21:38:21 +0100 Subject: [PATCH 070/136] refactor(server): isolate narrative recovery delta --- .../narrative-recovery-delta.test.ts | 56 +++++++++++++++++++ .../agents/turns/narrative-recovery-delta.ts | 47 ++++++++++++++++ .../parent-narrative-recovery-coordinator.ts | 48 +++++----------- 3 files changed, 118 insertions(+), 33 deletions(-) create mode 100644 apps/server/src/features/agents/turns/__tests__/narrative-recovery-delta.test.ts create mode 100644 apps/server/src/features/agents/turns/narrative-recovery-delta.ts diff --git a/apps/server/src/features/agents/turns/__tests__/narrative-recovery-delta.test.ts b/apps/server/src/features/agents/turns/__tests__/narrative-recovery-delta.test.ts new file mode 100644 index 000000000..c379b5042 --- /dev/null +++ b/apps/server/src/features/agents/turns/__tests__/narrative-recovery-delta.test.ts @@ -0,0 +1,56 @@ +import { describe, expect, it } from "vitest"; +import type { ParentNarrativeRecoveryItem } from "@mcode/contracts"; + +import { NarrativeRecoveryDelta } from "../narrative-recovery-delta.js"; + +const thought = { + kind: "narrationSegment", + record: { + id: "thought-1", message_id: "message-1", text: "First thought", + started_at: "2026-09-24T10:00:00.000Z", ended_at: null, sort_order: 1, + }, +} satisfies ParentNarrativeRecoveryItem; +const revisedThought = { ...thought, record: { ...thought.record, text: "Revised thought" } }; + +describe("NarrativeRecoveryDelta", () => { + it("returns changed items, then skips an unchanged acknowledged snapshot", () => { + const delta = new NarrativeRecoveryDelta(); + const first = delta.prepare([thought]); + expect(first).toMatchObject({ items: [thought], discardedItemIds: [] }); + first?.acknowledge(); + expect(delta.prepare([thought])).toBeNull(); + + const revised = delta.prepare([revisedThought]); + expect(revised).toMatchObject({ items: [revisedThought], discardedItemIds: [] }); + revised?.acknowledge(); + expect(delta.prepare([revisedThought])).toBeNull(); + }); + + it("reports an item removed from the complete snapshot", () => { + const delta = new NarrativeRecoveryDelta(); + delta.prepare([thought])?.acknowledge(); + const discard = delta.prepare([]); + expect(discard).toMatchObject({ items: [], discardedItemIds: ["narrationSegment:thought-1"] }); + discard?.acknowledge(); + expect(delta.prepare([])).toBeNull(); + }); + + it("retries the same delta when persistence failed before acknowledgement", () => { + const delta = new NarrativeRecoveryDelta(); + const failed = delta.prepare([thought]); + const retry = delta.prepare([thought]); + expect(retry).toMatchObject({ items: [thought], discardedItemIds: [] }); + retry?.acknowledge(); + expect(delta.prepare([thought])).toBeNull(); + expect(() => failed?.acknowledge()).toThrow("already superseded"); + }); + + it("keeps separate executions' fingerprints independent", () => { + const firstExecution = new NarrativeRecoveryDelta(); + const secondExecution = new NarrativeRecoveryDelta(); + firstExecution.prepare([thought])?.acknowledge(); + expect(firstExecution.prepare([thought])).toBeNull(); + expect(secondExecution.prepare([thought])).toMatchObject({ items: [thought] }); + expect(secondExecution.prepare([])).toBeNull(); + }); +}); diff --git a/apps/server/src/features/agents/turns/narrative-recovery-delta.ts b/apps/server/src/features/agents/turns/narrative-recovery-delta.ts new file mode 100644 index 000000000..11fe41ab5 --- /dev/null +++ b/apps/server/src/features/agents/turns/narrative-recovery-delta.ts @@ -0,0 +1,47 @@ +import type { ParentNarrativeRecoveryItem } from "@mcode/contracts"; + +/** One full-snapshot difference awaiting confirmation of its durable write. */ +export interface PreparedNarrativeRecoveryDelta { + readonly items: readonly ParentNarrativeRecoveryItem[]; + readonly discardedItemIds: readonly string[]; + acknowledge(): void; +} + +/** Tracks one execution's committed narrative snapshot without owning persistence. */ +export class NarrativeRecoveryDelta { + private fingerprints = new Map(); + private revision = 0; + + /** Compare a complete snapshot to the last acknowledged one. */ + prepare(snapshot: readonly ParentNarrativeRecoveryItem[]): PreparedNarrativeRecoveryDelta | null { + const next = new Map(); + const items: ParentNarrativeRecoveryItem[] = []; + for (const item of snapshot) { + const id = `${item.kind}:${item.record.id}`; + const fingerprint = JSON.stringify(item); + next.set(id, fingerprint); + if (this.fingerprints.get(id) !== fingerprint) items.push(item); + } + const discardedItemIds = [...this.fingerprints.keys()] + .filter((id) => !next.has(id)).map(canonicalItemId); + if (items.length === 0 && discardedItemIds.length === 0) return null; + const preparedAt = this.revision; + return { + items, + discardedItemIds, + acknowledge: () => { + if (this.revision !== preparedAt) throw new Error("Narrative recovery delta was already superseded"); + this.fingerprints = next; + this.revision += 1; + }, + }; + } +} + +function canonicalItemId(key: string): string { + const separator = key.indexOf(":"); + if (separator <= 0 || separator === key.length - 1) { + throw new Error(`Invalid narrative recovery identity: ${key}`); + } + return key; +} diff --git a/apps/server/src/features/agents/turns/parent-narrative-recovery-coordinator.ts b/apps/server/src/features/agents/turns/parent-narrative-recovery-coordinator.ts index 2b0c2b6bf..b297e1bd0 100644 --- a/apps/server/src/features/agents/turns/parent-narrative-recovery-coordinator.ts +++ b/apps/server/src/features/agents/turns/parent-narrative-recovery-coordinator.ts @@ -6,6 +6,7 @@ import { import type { ParentTurnDurability } from "./parent-turn-durability.js"; import type { NarrativeStore } from "../conversation/narrative/narrative-store.js"; import { serverWorkTrace } from "../diagnostics/server-work-trace.js"; +import { NarrativeRecoveryDelta } from "./narrative-recovery-delta.js"; interface ParentNarrativeRecoveryCheckpoint { persist(): void; @@ -14,7 +15,7 @@ interface ParentNarrativeRecoveryCheckpoint { /** Commits changed structured narrative records before AgentService publishes their source event. */ export class ParentNarrativeRecoveryCoordinator { - private readonly fingerprintsByExecution = new Map>(); + private readonly deltasByExecution = new Map(); constructor( private readonly canonicalSink: ParentTurnDurability, @@ -44,42 +45,32 @@ export class ParentNarrativeRecoveryCoordinator { event: AgentEvent, snapshot?: readonly ParentNarrativeRecoveryItem[], ): ParentNarrativeRecoveryCheckpoint | null { - if (!this.requiresStructuredRecovery(event) || !event.turnExecutionId) return null; - if (!this.canonicalSink.loadTurnByExecution(event.turnExecutionId)) return null; + const executionId = event.turnExecutionId; + if (!this.requiresStructuredRecovery(event) || !executionId) return null; + if (!this.canonicalSink.loadTurnByExecution(executionId)) return null; const snapshots = snapshot ?? this.narrativeStore.recoverySnapshot(event.threadId); - - const fingerprints = this.fingerprintsByExecution.get(event.turnExecutionId) ?? new Map(); - const nextFingerprints = new Map(); - const currentKeys = new Set(snapshots.map((item) => `${item.kind}:${item.record.id}`)); - const changed = snapshots.filter((item) => { - const key = `${item.kind}:${item.record.id}`; - const fingerprint = JSON.stringify(item); - nextFingerprints.set(key, fingerprint); - if (fingerprints.get(key) === fingerprint) return false; - return true; - }); - const discardedItemIds = [...fingerprints.keys()] - .filter((key) => !currentKeys.has(key)) - .map((key) => this.canonicalItemId(key)); - if (changed.length === 0 && discardedItemIds.length === 0) return null; + const delta = this.deltasByExecution.get(executionId) ?? new NarrativeRecoveryDelta(); + const prepared = delta.prepare(snapshots); + if (!prepared) return null; + this.deltasByExecution.set(executionId, delta); return { persist: () => { const committed = this.canonicalSink.recordParentNarrativeRecovery({ - executionId: event.turnExecutionId!, - items: changed, - discardedItemIds, + executionId, + items: prepared.items, + discardedItemIds: prepared.discardedItemIds, }); if (!committed) { - throw new Error(`Canonical parent turn was not found: ${event.turnExecutionId}`); + throw new Error(`Canonical parent turn was not found: ${executionId}`); } }, - confirm: () => this.fingerprintsByExecution.set(event.turnExecutionId!, nextFingerprints), + confirm: () => prepared.acknowledge(), }; } /** Forget volatile dedupe state after a terminal turn releases its buffers. */ clear(executionId: string | undefined): void { - if (executionId) this.fingerprintsByExecution.delete(executionId); + if (executionId) this.deltasByExecution.delete(executionId); } private requiresStructuredRecovery(event: AgentEvent): boolean { @@ -90,13 +81,4 @@ export class ParentNarrativeRecoveryCoordinator { || event.type === AgentEventType.HookStarted || event.type === AgentEventType.HookCompleted; } - - private canonicalItemId(key: string): string { - const separator = key.indexOf(":"); - const kind = separator < 0 ? "" : key.slice(0, separator); - const id = separator < 0 ? "" : key.slice(separator + 1); - if (!kind || !id) throw new Error(`Invalid narrative recovery identity: ${key}`); - return kind === "toolCall" ? `toolCall:${id}` : `${kind}:${id}`; - } - } From d1732fdebcc133a0628a2272df1432e22d14219a Mon Sep 17 00:00:00 2001 From: "(CJ) Chukwudi Nwobodo" <142016413+chuks-qua@users.noreply.github.com> Date: Thu, 24 Sep 2026 21:38:25 +0100 Subject: [PATCH 071/136] refactor(server): isolate plan execution parsing state --- .../__tests__/plan-execution-state.test.ts | 81 ++++++++++++ .../plan-turn-service-output.test.ts | 68 +++++++++++ .../agents/planning/plan-execution-state.ts | 115 ++++++++++++++++++ .../agents/planning/plan-turn-service.ts | 108 ++++------------ 4 files changed, 285 insertions(+), 87 deletions(-) create mode 100644 apps/server/src/features/agents/planning/__tests__/plan-execution-state.test.ts create mode 100644 apps/server/src/features/agents/planning/__tests__/plan-turn-service-output.test.ts create mode 100644 apps/server/src/features/agents/planning/plan-execution-state.ts diff --git a/apps/server/src/features/agents/planning/__tests__/plan-execution-state.test.ts b/apps/server/src/features/agents/planning/__tests__/plan-execution-state.test.ts new file mode 100644 index 000000000..79397e0d9 --- /dev/null +++ b/apps/server/src/features/agents/planning/__tests__/plan-execution-state.test.ts @@ -0,0 +1,81 @@ +import { describe, expect, it } from "vitest"; + +import { PlanExecutionState } from "../plan-execution-state.js"; + +const question = { + id: "q1", + category: "AUTH", + question: "Which login?", + options: [ + { id: "o1", title: "Passkey", description: "Use passkeys.", recommended: true }, + { id: "o2", title: "Password", description: "Use passwords." }, + ], +}; + +const structuredPlan = { + title: "Login plan", + changeSummary: "Use passkeys", + sections: [{ id: "s1", title: "Implementation", level: 1, content: "Add passkey login." }], +}; + +describe("PlanExecutionState", () => { + it("publishes one plain question outcome from chunked text", () => { + const state = new PlanExecutionState(); + state.beginQuestionGeneration(); + const block = `\`\`\`plan-questions\n${JSON.stringify([question])}\n\`\`\``; + + expect(state.feedText(block.slice(0, 23))).toBeNull(); + const ready = state.feedText(block.slice(23)); + expect(ready).toEqual({ questions: [question] }); + expect(structuredClone(ready)).toEqual(ready); + expect(state.feedText(block)).toBeNull(); + }); + + it("uses the parsed plan at the assistant message boundary", () => { + const state = new PlanExecutionState(); + state.beginOutputGeneration(); + const block = `\`\`\`plan-output\n${JSON.stringify(structuredPlan)}\n\`\`\``; + + expect(state.needsAssistantMaterialization()).toBe(true); + expect(state.feedText(block.slice(0, 19))).toBeNull(); + expect(state.feedText(block.slice(19))).toBeNull(); + expect(state.consumeAssistantMessage("provider prose")).toEqual({ + title: "Login plan", + contentMd: "## Implementation\n\nAdd passkey login.", + sectionsJson: '[{"id":"s1","title":"Implementation","level":1}]', + changeSummary: "Use passkeys", + }); + expect(state.needsAssistantMaterialization()).toBe(false); + expect(state.consumeAssistantMessage("# Another\n## Section")).toBeNull(); + }); + + it("uses assistant markdown when no structured block completes", () => { + const state = new PlanExecutionState(); + state.beginOutputGeneration(); + + expect(state.consumeAssistantMessage("# Fallback\n## Step\nDo it.")).toEqual({ + title: "Fallback", + contentMd: "# Fallback\n## Step\nDo it.", + sectionsJson: '[{"id":"s1","title":"Step","level":2}]', + changeSummary: null, + }); + expect(state.needsAssistantMaterialization()).toBe(false); + }); + + it("lets native exit markdown replace a pending streamed plan", () => { + const state = new PlanExecutionState(); + state.beginOutputGeneration(); + state.feedText(`\`\`\`plan-output\n${JSON.stringify(structuredPlan)}\n\`\`\``); + state.handleNativeExit("# Native plan\n## Deploy\nShip it."); + + expect(state.consumeAssistantMessage("ignored")).toEqual({ + title: "Native plan", + contentMd: "# Native plan\n## Deploy\nShip it.", + sectionsJson: '[{"id":"s1","title":"Deploy","level":2}]', + changeSummary: null, + }); + state.markPlanPersisted(); + state.handleNativeExit("# Later\n## Ignored"); + expect(state.consumeAssistantMessage("ignored")).toBeNull(); + }); +}); diff --git a/apps/server/src/features/agents/planning/__tests__/plan-turn-service-output.test.ts b/apps/server/src/features/agents/planning/__tests__/plan-turn-service-output.test.ts new file mode 100644 index 000000000..3d25a9c58 --- /dev/null +++ b/apps/server/src/features/agents/planning/__tests__/plan-turn-service-output.test.ts @@ -0,0 +1,68 @@ +import "reflect-metadata"; +import { AgentEventType, type AgentEvent } from "@mcode/contracts"; +import { describe, expect, it, vi } from "vitest"; + +import { openMemoryDatabase } from "../../../../runtime/persistence/sqlite/database.js"; +import { WorkspaceRepo } from "../../../projects/persistence/workspace-repo.js"; +import { ProviderRegistry } from "../../../providers/composition/provider-registry.js"; +import { ThreadRepo } from "../../../thread-control/persistence/thread-repo.js"; +import { MessageRepo } from "../../conversation/persistence/message-repo.js"; +import { AgentRuntimeCommandPort, AgentTurnCommandPort } from "../../orchestration/agent-turn-command-port.js"; +import { PlanQuestionAnswersRepo } from "../persistence/plan-question-answers-repo.js"; +import { PlanRepo } from "../persistence/plan-repo.js"; +import { PlanQuestionService } from "../plan-question-service.js"; +import { PlanTurnService } from "../plan-turn-service.js"; + +vi.mock("../../../../application/transport/push.js", () => ({ broadcast: vi.fn() })); +import { broadcast } from "../../../../application/transport/push.js"; + +describe("PlanTurnService output", () => { + it("publishes parsed questions and persists the plan at its assistant message", () => { + const db = openMemoryDatabase(); + const workspace = new WorkspaceRepo(db).create("plans", process.cwd(), false); + const threads = new ThreadRepo(db); + const thread = threads.create(workspace.id, "plan", "direct", "main", false, "codex"); + const messages = new MessageRepo(db); + const plans = new PlanRepo(db); + const questions = new PlanQuestionService(messages, new PlanQuestionAnswersRepo(db)); + const service = new PlanTurnService( + threads, + new ProviderRegistry([]), + questions, + plans, + new AgentTurnCommandPort(new AgentRuntimeCommandPort()), + ); + const question = { + id: "q1", category: "AUTH", question: "Which login?", + options: [ + { id: "o1", title: "Passkey", description: "Use passkeys." }, + { id: "o2", title: "Password", description: "Use passwords." }, + ], + }; + + service.beginQuestionGeneration(thread.id); + service.onTextDelta(thread.id, `\`\`\`plan-questions\n${JSON.stringify([question])}\n\`\`\``); + expect(broadcast).toHaveBeenCalledWith("plan.questions", { threadId: thread.id, questions: [question] }); + + service.beginOutputGeneration(thread.id); + const output = { title: "Login plan", sections: [{ id: "s1", title: "Implement", level: 1, content: "Add passkeys." }] }; + const block = `\`\`\`plan-output\n${JSON.stringify(output)}\n\`\`\``; + service.onTextDelta(thread.id, block.slice(0, 24)); + service.onTextDelta(thread.id, block.slice(24)); + const assistant = messages.create(thread.id, "assistant", "Plan response", 1); + const event: Extract = { + type: AgentEventType.Message, threadId: thread.id, messageId: assistant.id, content: assistant.content, tokens: null, + }; + + expect(service.needsAssistantMaterialization(event)).toBe(true); + service.persistAssistantMessage(event); + expect(plans.getLatestForThread(thread.id)).toMatchObject({ + messageId: assistant.id, + title: "Login plan", + contentMd: "## Implement\n\nAdd passkeys.", + sectionsJson: [{ id: "s1", title: "Implement", level: 1 }], + }); + expect(service.needsAssistantMaterialization(event)).toBe(false); + db.close(true); + }); +}); diff --git a/apps/server/src/features/agents/planning/plan-execution-state.ts b/apps/server/src/features/agents/planning/plan-execution-state.ts new file mode 100644 index 000000000..cc03f8939 --- /dev/null +++ b/apps/server/src/features/agents/planning/plan-execution-state.ts @@ -0,0 +1,115 @@ +import type { PlanOutput, PlanQuestion } from "@mcode/contracts"; + +import { PlanOutputParser } from "./plan-output-parser.js"; +import { PlanQuestionParser } from "./plan-question-parser.js"; + +/** Parsed questions ready for the service to publish. */ +export interface PlanQuestionsReady { + questions: PlanQuestion[]; +} + +/** Plan data ready for the service to persist against an assistant message. */ +export interface PlanPersistenceReady { + title: string; + contentMd: string; + sectionsJson: string; + changeSummary: string | null; +} + +/** Parsing and output decisions for one plan execution, without database or transport dependencies. */ +export class PlanExecutionState { + private questionParser: PlanQuestionParser | undefined; + private outputParser: PlanOutputParser | undefined; + private pendingOutput: PlanOutput | undefined; + private pendingExitMarkdown: string | undefined; + private captured = false; + + beginQuestionGeneration(): void { + this.questionParser = new PlanQuestionParser(); + } + + beginOutputGeneration(): void { + this.outputParser = new PlanOutputParser(); + this.captured = false; + } + + feedText(delta: string): PlanQuestionsReady | null { + const questions = this.questionParser?.feed(delta); + if (questions) this.questionParser = undefined; + const output = this.outputParser?.feed(delta); + if (output) { + this.outputParser = undefined; + this.pendingOutput = output; + } + return questions ? { questions } : null; + } + + handleNativeExit(markdown: string): void { + if (this.captured) return; + this.outputParser = undefined; + this.pendingOutput = undefined; + this.pendingExitMarkdown = markdown; + } + + needsAssistantMaterialization(): boolean { + return this.pendingOutput !== undefined + || this.pendingExitMarkdown !== undefined + || this.outputParser !== undefined; + } + + consumeAssistantMessage(content: string): PlanPersistenceReady | null { + const output = this.pendingOutput; + if (output) { + this.pendingOutput = undefined; + this.outputParser = undefined; + this.pendingExitMarkdown = undefined; + const contentMd = output.sections.map((section) => ( + `${"#".repeat(section.level + 1)} ${section.title}\n\n${section.content}` + )).join("\n\n"); + const sectionsJson = JSON.stringify(output.sections.map((section) => ({ + id: section.id, + title: section.title, + level: section.level, + }))); + return { title: output.title, contentMd, sectionsJson, changeSummary: output.changeSummary ?? null }; + } + const markdown = this.pendingExitMarkdown; + if (markdown) { + this.pendingExitMarkdown = undefined; + this.outputParser = undefined; + return extractMarkdown(markdown); + } + if (!this.outputParser || !content) return null; + this.outputParser = undefined; + return extractMarkdown(content); + } + + hasPersistedPlan(): boolean { + return this.captured; + } + + markPlanPersisted(): void { + this.captured = true; + } +} + +function extractMarkdown(content: string): PlanPersistenceReady | null { + let title: string | null = null; + let nextId = 0; + const sections: Array<{ id: string; title: string; level: number }> = []; + for (const line of content.split("\n")) { + const match = /^(#{1,3})\s+(.+)/.exec(line); + if (!match) continue; + const level = match[1].length; + const heading = match[2].trim(); + if (!title) { + title = heading; + continue; + } + nextId += 1; + sections.push({ id: `s${nextId}`, title: heading, level }); + } + return title && sections.length > 0 + ? { title, contentMd: content, sectionsJson: JSON.stringify(sections), changeSummary: null } + : null; +} diff --git a/apps/server/src/features/agents/planning/plan-turn-service.ts b/apps/server/src/features/agents/planning/plan-turn-service.ts index 8505f5a31..e60162a30 100644 --- a/apps/server/src/features/agents/planning/plan-turn-service.ts +++ b/apps/server/src/features/agents/planning/plan-turn-service.ts @@ -5,7 +5,6 @@ import type { ContextWindowMode, IProviderRegistry, PermissionMode, - PlanOutput, ProviderId, ReasoningLevel, } from "@mcode/contracts"; @@ -15,8 +14,7 @@ import { AGENT_TURN_COMMAND_PORT, type AgentTurnCommandPort, } from "../orchestration/agent-turn-command-port.js"; -import { PlanOutputParser } from "./plan-output-parser.js"; -import { PlanQuestionParser } from "./plan-question-parser.js"; +import { PlanExecutionState, type PlanPersistenceReady } from "./plan-execution-state.js"; import { PlanQuestionService, type PlanAnswerInput } from "./plan-question-service.js"; import { PlanRepo } from "./persistence/plan-repo.js"; @@ -29,11 +27,7 @@ type ClaudePlanAnswerModeProvider = { /** Owns plan-question turns and durable plan-output materialization. */ @injectable() export class PlanTurnService { - private readonly questionParsers = new Map(); - private readonly outputParsers = new Map(); - private readonly pendingOutputs = new Map(); - private readonly pendingExitMarkdown = new Map(); - private readonly capturedThreads = new Set(); + private readonly executionByThread = new Map(); constructor( @inject(ThreadRepo) private readonly threadRepo: ThreadRepo, @@ -45,13 +39,12 @@ export class PlanTurnService { /** Start parsing one plan-question generation turn. */ beginQuestionGeneration(threadId: string): void { - this.questionParsers.set(threadId, new PlanQuestionParser()); + this.execution(threadId).beginQuestionGeneration(); } /** Start parsing one structured plan-output turn and arm its native provider mode. */ beginOutputGeneration(threadId: string): void { - this.outputParsers.set(threadId, new PlanOutputParser()); - this.capturedThreads.delete(threadId); + this.execution(threadId).beginOutputGeneration(); this.armNativeOutputMode(threadId); } @@ -94,65 +87,27 @@ ${userMessage}`; /** Consume one visible text delta while a plan turn is active. */ onTextDelta(threadId: string, delta: string): void { - const questions = this.questionParsers.get(threadId)?.feed(delta); - if (questions) { - this.questionParsers.delete(threadId); - broadcast("plan.questions", { threadId, questions }); - } - const output = this.outputParsers.get(threadId)?.feed(delta); - if (output) { - this.outputParsers.delete(threadId); - this.pendingOutputs.set(threadId, output); - } + const ready = this.executionByThread.get(threadId)?.feedText(delta); + if (ready) broadcast("plan.questions", { threadId, questions: ready.questions }); } /** Capture native plan markdown until its assistant message receives a durable identity. */ handleExitPlanMode(threadId: string, planMarkdown: string): void { - if (this.capturedThreads.has(threadId)) return; - this.outputParsers.delete(threadId); - this.pendingOutputs.delete(threadId); - this.pendingExitMarkdown.set(threadId, planMarkdown); + this.execution(threadId).handleNativeExit(planMarkdown); } /** Return whether a message needs early durable materialization for a plan record. */ needsAssistantMaterialization(event: PlanMessage): boolean { if (!event.messageId) return false; - return this.pendingOutputs.has(event.threadId) - || this.pendingExitMarkdown.has(event.threadId) - || this.outputParsers.has(event.threadId); + return this.executionByThread.get(event.threadId)?.needsAssistantMaterialization() ?? false; } /** Persist the one plan record that an assistant message can materialize. */ persistAssistantMessage(event: PlanMessage): void { if (!event.messageId) return; - const output = this.pendingOutputs.get(event.threadId); - if (output) { - this.pendingOutputs.delete(event.threadId); - this.outputParsers.delete(event.threadId); - this.pendingExitMarkdown.delete(event.threadId); - const content = output.sections.map((section) => ( - `${"#".repeat(section.level + 1)} ${section.title}\n\n${section.content}` - )).join("\n\n"); - const sections = JSON.stringify(output.sections.map((section) => ({ - id: section.id, - title: section.title, - level: section.level, - }))); - this.persistPlan(event.threadId, event.messageId, output.title, content, sections, output.changeSummary ?? null); - return; - } - const markdown = this.pendingExitMarkdown.get(event.threadId); - if (markdown) { - this.pendingExitMarkdown.delete(event.threadId); - this.outputParsers.delete(event.threadId); - const extracted = this.extractMarkdown(markdown); - if (extracted) this.persistPlan(event.threadId, event.messageId, extracted.title, extracted.contentMd, extracted.sectionsJson, null); - return; - } - if (!this.outputParsers.has(event.threadId) || !event.content) return; - this.outputParsers.delete(event.threadId); - const extracted = this.extractMarkdown(event.content); - if (extracted) this.persistPlan(event.threadId, event.messageId, extracted.title, extracted.contentMd, extracted.sectionsJson, null); + const execution = this.executionByThread.get(event.threadId); + const ready = execution?.consumeAssistantMessage(event.content); + if (execution && ready) this.persistPlan(event.threadId, event.messageId, ready, execution); } /** Submit answers and dispatch the complete answer turn through the command facade. */ @@ -190,11 +145,7 @@ ${userMessage}`; /** Clear volatile plan state once a turn reaches its terminal lifecycle. */ clearTurn(threadId: string): void { - this.questionParsers.delete(threadId); - this.outputParsers.delete(threadId); - this.pendingOutputs.delete(threadId); - this.pendingExitMarkdown.delete(threadId); - this.capturedThreads.delete(threadId); + this.executionByThread.delete(threadId); } private armNativeOutputMode(threadId: string): void { @@ -207,15 +158,13 @@ ${userMessage}`; private persistPlan( threadId: string, messageId: string, - title: string, - contentMd: string, - sectionsJson: string, - changeSummary: string | null, + ready: PlanPersistenceReady, + execution: PlanExecutionState, ): void { - if (this.capturedThreads.has(threadId)) return; + if (execution.hasPersistedPlan()) return; try { - const plan = this.planRepo.create(threadId, messageId, title, contentMd, sectionsJson, changeSummary); - this.capturedThreads.add(threadId); + const plan = this.planRepo.create(threadId, messageId, ready.title, ready.contentMd, ready.sectionsJson, ready.changeSummary); + execution.markPlanPersisted(); broadcast("plan.generated", { threadId, plan }); } catch (error) { logger.error("Failed to persist plan output", { @@ -225,24 +174,9 @@ ${userMessage}`; } } - private extractMarkdown(content: string): { title: string; contentMd: string; sectionsJson: string } | null { - let title: string | null = null; - let nextId = 0; - const sections: Array<{ id: string; title: string; level: number }> = []; - for (const line of content.split("\n")) { - const match = /^(#{1,3})\s+(.+)/.exec(line); - if (!match) continue; - const level = match[1].length; - const heading = match[2].trim(); - if (!title) { - title = heading; - continue; - } - nextId += 1; - sections.push({ id: `s${nextId}`, title: heading, level }); - } - return title && sections.length > 0 - ? { title, contentMd: content, sectionsJson: JSON.stringify(sections) } - : null; + private execution(threadId: string): PlanExecutionState { + const state = this.executionByThread.get(threadId) ?? new PlanExecutionState(); + this.executionByThread.set(threadId, state); + return state; } } From 252194fca5dfaf2548d0caa4084942a090a642b4 Mon Sep 17 00:00:00 2001 From: "(CJ) Chukwudi Nwobodo" <142016413+chuks-qua@users.noreply.github.com> Date: Thu, 24 Sep 2026 21:42:12 +0100 Subject: [PATCH 072/136] feat(server): reduce live Codex events per execution --- .../codex-live-event-reducer.test.ts | 106 +++++++ .../execution/codex-live-event-reducer.ts | 298 ++++++++++++++++++ 2 files changed, 404 insertions(+) create mode 100644 apps/server/src/features/agents/execution/__tests__/codex-live-event-reducer.test.ts create mode 100644 apps/server/src/features/agents/execution/codex-live-event-reducer.ts diff --git a/apps/server/src/features/agents/execution/__tests__/codex-live-event-reducer.test.ts b/apps/server/src/features/agents/execution/__tests__/codex-live-event-reducer.test.ts new file mode 100644 index 000000000..7b380eaa2 --- /dev/null +++ b/apps/server/src/features/agents/execution/__tests__/codex-live-event-reducer.test.ts @@ -0,0 +1,106 @@ +import { describe, expect, it } from "vitest"; +import { AgentEventSchema, type AgentEvent } from "@mcode/contracts"; +import { CodexEventMapper } from "../../../../../../../packages/providers/src/private/codex/codex-event-mapper.js"; +import { CodexLiveEventReducer } from "../codex-live-event-reducer.js"; + +const execution = { + threadId: "test-thread", + turnId: "test-turn", + executionId: "11111111-1111-4111-8111-111111111111", +}; + +function event(type: AgentEvent["type"], fields: Record = {}): AgentEvent { + return AgentEventSchema().parse({ type, threadId: execution.threadId, ...fields }); +} + +describe("CodexLiveEventReducer", () => { + it("reduces a Codex notification sequence through tool narration and final answer", () => { + const mapper = new CodexEventMapper(execution.threadId); + const reducer = new CodexLiveEventReducer(execution); + const reductions = [reducer.reduce(event("turnStarted"))]; + const notifications = [ + { method: "item/agentMessage/delta", params: { itemId: "thought", delta: "I will check." } }, + { method: "item/completed", params: { item: { type: "agentMessage", id: "thought" } } }, + { method: "item/started", params: { item: { type: "commandExecution", id: "cmd", command: "pwd" } } }, + { method: "item/completed", params: { item: { type: "commandExecution", id: "cmd", command: "pwd", output: "/repo", exitCode: 0 } } }, + { method: "item/agentMessage/delta", params: { itemId: "answer", delta: "Done." } }, + { method: "item/completed", params: { item: { type: "agentMessage", id: "answer" } } }, + { method: "turn/completed", params: { turn: { status: "completed" } } }, + ]; + for (const notification of notifications) { + for (const runtimeEvent of mapper.mapNotification({ jsonrpc: "2.0", ...notification })) { + reductions.push(reducer.reduce(runtimeEvent.event)); + } + } + + expect(reductions.every((result) => result.kind === "reduced")).toBe(true); + const reduced = reductions.filter((result) => result.kind === "reduced"); + expect(reduced.map((result) => result.publication.event.type)).toEqual([ + "turnStarted", "textDelta", "assistantMessageBoundary", "toolUse", "toolResult", + "textDelta", "assistantMessageBoundary", "message", "turnComplete", + ]); + const toolUse = reduced.find((result) => result.publication.event.type === "toolUse"); + expect(toolUse?.writer).toContainEqual(expect.objectContaining({ kind: "tool-use" })); + const message = reduced.find((result) => result.publication.event.type === "message"); + expect(message?.writer).toContainEqual(expect.objectContaining({ + kind: "assistant-body", content: "Done.", attachments: [], + })); + const terminal = reduced.find((result) => result.publication.event.type === "turnComplete"); + expect(terminal?.writer).toContainEqual(expect.objectContaining({ + kind: "terminal-projection", source: "turnComplete", outcome: "completed", + assistant: expect.objectContaining({ content: "Done." }), + })); + expect(terminal?.publication.after).toBe("terminal"); + expect(structuredClone(reduced)).toEqual(reduced); + }); + + it("reclassifies unknown text as narration and carries attachments and late hooks", () => { + const reducer = new CodexLiveEventReducer(execution); + expect(reducer.reduce(event("turnStarted")).kind).toBe("reduced"); + const delta = reducer.reduce(event("textDelta", { delta: "working" })); + expect(delta.kind).toBe("reduced"); + if (delta.kind !== "reduced") return; + expect(delta.writer).toContainEqual({ kind: "assistant-text-delta", delta: "working", classification: "unknown" }); + const boundary = reducer.reduce(event("assistantMessageBoundary", { isFinalResponse: false })); + expect(boundary.kind).toBe("reduced"); + if (boundary.kind !== "reduced") return; + expect(boundary.writer).toContainEqual({ kind: "assistant-text-reclassify", text: "working", classification: "narration" }); + expect(boundary.writer).toContainEqual(expect.objectContaining({ + kind: "narrative-recovery", + items: [expect.objectContaining({ kind: "narrationSegment", record: expect.objectContaining({ text: "working" }) })], + })); + + const attachment = { id: "image", name: "image.png", mimeType: "image/png", sizeBytes: 12 }; + expect(reducer.reduce(event("generatedAttachment", { attachment })).kind).toBe("reduced"); + const message = reducer.reduce(event("message", { content: "Answer", tokens: null })); + expect(message.kind).toBe("reduced"); + if (message.kind !== "reduced") return; + expect(message.writer).toContainEqual(expect.objectContaining({ kind: "assistant-body", attachments: [attachment] })); + + expect(reducer.reduce(event("turnComplete", { reason: "completed", costUsd: null, tokensIn: 2, tokensOut: 1 })).kind).toBe("reduced"); + const started = reducer.reduce(event("hookStarted", { hookName: "stop", hookType: "stop" })); + expect(started.kind).toBe("reduced"); + if (started.kind !== "reduced") return; + expect(started.writer).toContainEqual(expect.objectContaining({ kind: "hook-started", late: true })); + const completed = reducer.reduce(event("hookCompleted", { hookName: "stop", exitCode: 0, durationMs: 10, didBlock: false })); + expect(completed.kind).toBe("reduced"); + if (completed.kind !== "reduced") return; + expect(completed.writer).toContainEqual(expect.objectContaining({ kind: "hook-completed", late: true })); + expect(completed.publication.after).toBe("terminal"); + }); + + it("rejects another execution and unowned events without changing the active execution", () => { + const reducer = new CodexLiveEventReducer(execution); + expect(reducer.reduce(event("turnStarted")).kind).toBe("reduced"); + expect(reducer.reduce(event("textDelta", { delta: "wrong", turnExecutionId: "22222222-2222-4222-8222-222222222222" }))).toEqual({ + kind: "unsupported", eventType: "textDelta", reason: "different execution", + }); + expect(reducer.reduce(event("goalCleared", { reason: "cleared" }))).toEqual({ + kind: "unsupported", eventType: "goalCleared", reason: "event needs a separate feature owner", + }); + const final = reducer.reduce(event("textDelta", { delta: "right", isFinalResponse: true })); + expect(final.kind).toBe("reduced"); + if (final.kind !== "reduced") return; + expect(final.writer).toContainEqual({ kind: "assistant-text-delta", delta: "right", classification: "final" }); + }); +}); diff --git a/apps/server/src/features/agents/execution/codex-live-event-reducer.ts b/apps/server/src/features/agents/execution/codex-live-event-reducer.ts new file mode 100644 index 000000000..a5830d72b --- /dev/null +++ b/apps/server/src/features/agents/execution/codex-live-event-reducer.ts @@ -0,0 +1,298 @@ +import type { AgentEvent, ParentNarrativeRecoveryItem, StoredAttachment } from "@mcode/contracts"; +import { NarrativeTurnState, type NarrativeTurnStateEffect } from "../conversation/narrative/narrative-turn-state.js"; +import { AssistantExecutionState, type AssistantMaterializationInput } from "../turns/assistant-execution-state.js"; +import type { ExecutionIdentity } from "./execution-mailbox-protocol.js"; + +type ToolUseEvent = Extract; +type ToolResultEvent = Extract; +type MessageEvent = Extract; +type TextDeltaEvent = Extract; + +/** Data to commit for one provider event before releasing its publication. */ +export type CodexLiveWriterIntent = + | { readonly kind: "turn-started" } + | { readonly kind: "assistant-text-delta"; readonly delta: string; readonly classification: "final" | "unknown" } + | { readonly kind: "assistant-text-reclassify"; readonly text: string; readonly classification: "narration" } + | { readonly kind: "assistant-text-promote"; readonly text: string } + | { readonly kind: "assistant-body"; readonly content: string; readonly model: string | null; readonly attachments: StoredAttachment[]; readonly tokens: number | null } + | { readonly kind: "generated-attachment"; readonly attachment: StoredAttachment } + | { readonly kind: "tool-use"; readonly event: ToolUseEvent } + | { readonly kind: "tool-result"; readonly event: ToolResultEvent } + | { readonly kind: "hook-started"; readonly hookId: string; readonly late: boolean } + | { readonly kind: "hook-completed"; readonly hookId: string; readonly late: boolean; readonly exitCode: number; readonly durationMs: number; readonly didBlock: boolean } + | { readonly kind: "narrative-recovery"; readonly items: ParentNarrativeRecoveryItem[] } + | { readonly kind: "narrative-effect"; readonly effect: NarrativeTurnStateEffect } + | { readonly kind: "feature-event"; readonly feature: "plan-text" | "assistant-message" | "task-tool" | "goal-refresh"; readonly event: AgentEvent } + | { readonly kind: "terminal-projection"; readonly source: "turnComplete" | "ended"; readonly outcome: "completed" | "errored" | "interrupted"; readonly assistant: AssistantMaterializationInput; readonly narrative: ParentNarrativeRecoveryItem[] } + | { readonly kind: "turn-ended" }; + +/** A publication is released only after the writer accepts every intent for the event. */ +export interface CodexLivePublicationIntent { + readonly event: AgentEvent; + readonly after: "writer" | "terminal"; +} + +/** Unsupported means no reducer state was changed and no publication may be emitted. */ +export type CodexLiveReduction = + | { readonly kind: "reduced"; readonly execution: ExecutionIdentity; readonly writer: CodexLiveWriterIntent[]; readonly publication: CodexLivePublicationIntent } + | { readonly kind: "unsupported"; readonly eventType: AgentEvent["type"]; readonly reason: string }; + +/** + * Reduces one Codex execution's live AgentEvents without database or transport calls. + * The caller must discard this instance if its writer operation fails, because + * the next event may only observe state whose preceding intents were committed. + */ +export class CodexLiveEventReducer { + private readonly narrative: NarrativeTurnState; + private readonly assistant = new AssistantExecutionState(); + private phase: "awaiting-start" | "active" | "completed" | "ended" = "awaiting-start"; + private unknownText = ""; + private knownFinalText = false; + + constructor(readonly execution: ExecutionIdentity) { + this.narrative = new NarrativeTurnState(execution); + } + + reduce(input: AgentEvent): CodexLiveReduction { + const rejection = this.identityRejection(input) ?? this.phaseRejection(input) ?? this.textRejection(input); + if (rejection) return this.unsupported(input, rejection); + let event: AgentEvent; + try { + event = structuredClone({ ...input, turnExecutionId: this.execution.executionId }); + } catch { + return this.unsupported(input, "event is not cloneable"); + } + + const writer = this.apply(event); + if (!writer) return this.unsupported(input, "event needs a separate feature owner"); + for (const effect of this.narrative.takeEffects()) writer.push({ kind: "narrative-effect", effect }); + return structuredClone({ + kind: "reduced", + execution: this.execution, + writer, + publication: { + event: this.publicationEvent(event), + after: this.publicationBarrier(event), + }, + }); + } + + private identityRejection(event: AgentEvent): string | undefined { + if (event.threadId !== this.execution.threadId) return "different thread"; + if (event.turnExecutionId && event.turnExecutionId !== this.execution.executionId) return "different execution"; + return undefined; + } + + private phaseRejection(event: AgentEvent): string | undefined { + if (this.phase === "ended") return "execution already ended"; + if (this.phase === "awaiting-start" && event.type !== "turnStarted") return "turn has not started"; + if (this.phase !== "awaiting-start" && event.type === "turnStarted") return "turn already started"; + if (this.phase === "completed" && event.type !== "ended" && event.type !== "hookStarted" && event.type !== "hookCompleted") { + return "event after turn completion requires another owner"; + } + return undefined; + } + + private textRejection(event: AgentEvent): string | undefined { + switch (event.type) { + case "assistantMessageBoundary": + return !event.isFinalResponse && this.knownFinalText + ? "mixed final and unknown assistant text" : undefined; + case "textDelta": return this.deltaRejection(event); + case "turnComplete": + case "ended": + return this.unknownText ? "assistant text still lacks a boundary" : undefined; + default: return undefined; + } + } + + private deltaRejection(event: TextDeltaEvent): string | undefined { + if (event.isFinalResponse === false && this.knownFinalText) return "mixed final and nonfinal assistant text"; + if (event.isFinalResponse !== undefined && this.unknownText) return "mixed classified and unknown assistant text"; + return undefined; + } + + private publicationEvent(event: AgentEvent): AgentEvent { + return event.type === "ended" && event.outcome === "cancelled" + ? { ...event, outcome: "interrupted" } + : event; + } + + private publicationBarrier(event: AgentEvent): "writer" | "terminal" { + return this.phase === "completed" || (event.type === "ended" && event.outcome !== undefined) + ? "terminal" : "writer"; + } + + private apply(event: AgentEvent): CodexLiveWriterIntent[] | undefined { + return this.applyNarrative(event) ?? this.applyLifecycle(event); + } + + private applyNarrative(event: AgentEvent): CodexLiveWriterIntent[] | undefined { + switch (event.type) { + case "textDelta": return this.textDelta(event); + case "assistantMessageBoundary": return this.boundary(event); + case "message": return this.message(event); + case "generatedAttachment": return this.attachment(event); + case "toolUse": return this.toolUse(event); + case "toolResult": return this.toolResult(event); + case "hookStarted": return this.hookStarted(event); + case "hookCompleted": return this.hookCompleted(event); + default: return undefined; + } + } + + private applyLifecycle(event: AgentEvent): CodexLiveWriterIntent[] | undefined { + switch (event.type) { + case "turnStarted": return this.start(event); + case "turnComplete": return this.turnComplete(event); + case "ended": return this.ended(event); + default: return undefined; + } + } + + private start(event: Extract): CodexLiveWriterIntent[] { + this.narrative.beginTurn(event.threadId); + this.narrative.resetTurnCounters(event.threadId); + this.phase = "active"; + return [{ kind: "turn-started" }]; + } + + private textDelta(event: TextDeltaEvent): CodexLiveWriterIntent[] { + const writer: CodexLiveWriterIntent[] = [{ kind: "feature-event", feature: "plan-text", event }]; + if (event.isFinalResponse === false) { + this.narrative.openOrExtendThought(event.threadId, event.delta); + writer.push(this.recovery()); + } else { + this.assistant.appendStreamingText(event.delta); + if (event.isFinalResponse === undefined) this.unknownText += event.delta; + else this.knownFinalText = true; + writer.push({ kind: "assistant-text-delta", delta: event.delta, classification: event.isFinalResponse ? "final" : "unknown" }); + } + return writer; + } + + private boundary(event: Extract): CodexLiveWriterIntent[] { + const writer: CodexLiveWriterIntent[] = []; + if (event.isFinalResponse) { + const text = this.narrative.takeOpenThought(event.threadId); + if (text) { + this.assistant.appendStreamingText(text); + writer.push({ kind: "assistant-text-promote", text }); + } + this.unknownText = ""; + } else { + if (this.unknownText) { + const staged = this.narrative.stageNarrationSegment(event.threadId, this.unknownText); + if (staged) this.narrative.applyStagedNarrationSegment(event.threadId, staged); + writer.push({ kind: "assistant-text-reclassify", text: this.unknownText, classification: "narration" }); + this.assistant.resetStreamingText(); + this.unknownText = ""; + } + this.narrative.closeOpenThought(event.threadId); + } + this.knownFinalText = false; + writer.push(this.recovery()); + return writer; + } + + private message(event: MessageEvent): CodexLiveWriterIntent[] { + this.assistant.bufferBody(event.content, event.model ?? null, event.attachments ?? []); + const body = this.assistant.materializationInput(event.model ?? null); + this.assistant.resetStreamingText(); + this.narrative.clearAgentStackOnMessage(event.threadId); + this.knownFinalText = false; + this.unknownText = ""; + return [ + { kind: "assistant-body", content: body.content, model: body.model, attachments: body.attachments, tokens: event.tokens }, + { kind: "feature-event", feature: "assistant-message", event }, + ]; + } + + private attachment(event: Extract): CodexLiveWriterIntent[] { + this.assistant.bufferAttachments([event.attachment]); + return [{ kind: "generated-attachment", attachment: event.attachment }]; + } + + private toolUse(event: ToolUseEvent): CodexLiveWriterIntent[] { + this.narrative.closeOpenThought(event.threadId); + const parentToolCallId = this.narrative.bufferToolCall(event.threadId, event); + const attributed = { ...event, parentToolCallId }; + return [ + { kind: "tool-use", event: attributed }, + { kind: "feature-event", feature: "task-tool", event: attributed }, + this.recovery(), + ]; + } + + private toolResult(event: ToolResultEvent): CodexLiveWriterIntent[] { + this.narrative.updateBufferedToolCallOutput(event.threadId, event.toolCallId, event.output, event.isError, + event.toolInput, { + exitCode: event.exitCode, + outputTruncated: event.outputTruncated, + outputTotalBytes: event.outputTotalBytes, + outputArtifactPath: event.outputArtifactPath, + }, event.subagentPresentation); + return [ + { kind: "tool-result", event }, + { kind: "feature-event", feature: "task-tool", event }, + this.recovery(), + ]; + } + + private hookStarted(event: Extract): CodexLiveWriterIntent[] { + const late = this.phase === "completed"; + if (!late) this.narrative.closeOpenThought(event.threadId); + const hookId = this.narrative.openHook(event.threadId, { + hookName: event.hookName, + toolName: late ? null : event.toolName ?? null, + phase: late ? "stop" : event.hookType, + payload: JSON.stringify({ hookType: late ? "stop" : event.hookType, toolName: late ? null : event.toolName ?? null }), + sortOrder: this.narrative.nextSortOrder(event.threadId), + }); + return [{ kind: "hook-started", hookId, late }, this.recovery()]; + } + + private hookCompleted(event: Extract): CodexLiveWriterIntent[] { + const open = this.narrative.peekOpenHook(event.threadId, event.hookName); + if (!open) return []; + this.narrative.pushClosedHook(event.threadId, { + ...open, + messageId: "", + durationMs: event.durationMs, + didBlock: event.didBlock, + endedAt: new Date().toISOString(), + }); + this.narrative.removeOpenHook(event.threadId, event.hookName); + return [{ kind: "hook-completed", hookId: open.id, late: this.phase === "completed", + exitCode: event.exitCode, durationMs: event.durationMs, didBlock: event.didBlock }, this.recovery()]; + } + + private turnComplete(event: Extract): CodexLiveWriterIntent[] { + this.phase = "completed"; + return [ + { kind: "terminal-projection", source: "turnComplete", outcome: "completed", + assistant: this.assistant.materializationInput(null), narrative: this.narrative.recoverySnapshot(event.threadId) }, + { kind: "feature-event", feature: "goal-refresh", event }, + ]; + } + + private ended(event: Extract): CodexLiveWriterIntent[] { + const writer: CodexLiveWriterIntent[] = []; + if (event.outcome && this.phase !== "completed") { + writer.push({ kind: "terminal-projection", source: "ended", + outcome: event.outcome === "cancelled" ? "interrupted" : event.outcome, + assistant: this.assistant.materializationInput(null), narrative: this.narrative.recoverySnapshot(event.threadId) }); + } + writer.push({ kind: "turn-ended" }); + this.phase = "ended"; + return writer; + } + + private recovery(): CodexLiveWriterIntent { + return { kind: "narrative-recovery", items: this.narrative.recoverySnapshot(this.execution.threadId) }; + } + + private unsupported(event: AgentEvent, reason: string): CodexLiveReduction { + return { kind: "unsupported", eventType: event.type, reason }; + } +} From 24542671e841d5d8c5b343b333c335b2ff7a46a7 Mon Sep 17 00:00:00 2001 From: "(CJ) Chukwudi Nwobodo" <142016413+chuks-qua@users.noreply.github.com> Date: Thu, 24 Sep 2026 21:45:02 +0100 Subject: [PATCH 073/136] feat(server): route narrative deltas through execution worker --- .../execution-worker-handler.test.ts | 56 ++++++++++++++++++- .../execution/execution-worker-handler.ts | 19 ++++++- 2 files changed, 72 insertions(+), 3 deletions(-) diff --git a/apps/server/src/features/agents/execution/__tests__/execution-worker-handler.test.ts b/apps/server/src/features/agents/execution/__tests__/execution-worker-handler.test.ts index 21fee6fb8..8a89f64a8 100644 --- a/apps/server/src/features/agents/execution/__tests__/execution-worker-handler.test.ts +++ b/apps/server/src/features/agents/execution/__tests__/execution-worker-handler.test.ts @@ -5,6 +5,7 @@ import type { DataOnlyParentTurnFinishInput, DataOnlyParentTurnStartInput, } from "../../canonical/canonical-parent-turn-write.js"; +import type { ParentNarrativeRecoveryCommit } from "../../turns/parent-turn-durability.js"; import { ExecutionMailboxScheduler, type ExecutionMailboxCommand, @@ -54,6 +55,17 @@ const FINISH_INPUT: DataOnlyParentTurnFinishInput = { projection: { message: null, narrative: [] }, }; +const NARRATIVE_INPUT: ParentNarrativeRecoveryCommit = { + executionId: EXECUTION.executionId, + items: [{ + kind: "narrationSegment", + record: { + id: "thought-1", message_id: "message-1", text: "Working", + started_at: "2026-09-24T12:00:00.000Z", ended_at: null, sort_order: 1, + }, + }], +}; + const LIMITS: ExecutionMailboxLimits = { maxPending: 12, maxPendingBytes: 12_000, @@ -103,12 +115,15 @@ class InlineWorker implements ExecutionWorkerPort({ workerCount: 1, limits: LIMITS, createWorker: () => { - const worker = new InlineWorker(new ExecutionWorkerHandler(writer)); + const handler = new ExecutionWorkerHandler(writer); + const worker = new InlineWorker(handler); + handlers.push(handler); workers.push(worker); return worker; }, @@ -116,7 +131,7 @@ function fixture(writer = new RecordingWriter()) { }); const claim = scheduler.claim(EXECUTION, 1); if (claim.kind !== "claimed") throw new Error(`Claim failed: ${claim.kind}`); - return { scheduler, worker: workers[0]!, writer, lost, lease: claim.lease }; + return { scheduler, worker: workers[0]!, handler: handlers[0]!, writer, lost, lease: claim.lease }; } function submit( @@ -148,6 +163,43 @@ async function committed(admission: ReturnType, revision: number) } describe("ExecutionWorkerHandler through its scheduler", () => { + it("maps a running narrative delta to one fenced semantic operation", async () => { + const { scheduler, writer, lease } = fixture(); + await committed(submit(scheduler, lease, { kind: "start", providerId: "codex", input: START_INPUT }), 1); + await committed(submit(scheduler, lease, { kind: "narrative-delta", input: NARRATIVE_INPUT }), 2); + + expect(writer.operations[1]).toMatchObject({ + operationId: `${lease.leaseId}:2`, execution: EXECUTION, lease, ordinal: 2, + mutation: { kind: "narrative-delta", input: NARRATIVE_INPUT }, + }); + scheduler.shutdown(); + }); + + it("rejects a narrative delta for another execution before it reaches the writer", async () => { + const { scheduler, writer, lease } = fixture(); + await committed(submit(scheduler, lease, { kind: "start", providerId: "codex", input: START_INPUT }), 1); + await expect(submit(scheduler, lease, { + kind: "narrative-delta", input: { ...NARRATIVE_INPUT, executionId: "another-execution" }, + }).completion).resolves.toEqual({ + kind: "reply", result: { kind: "rejected", reason: "invalid-narrative-routing" }, + }); + expect(writer.operations.map((operation) => operation.mutation.kind)).toEqual(["begin"]); + scheduler.shutdown(); + }); + + it("rejects a narrative delta once the execution is stopping", async () => { + const { scheduler, handler, writer, lease } = fixture(); + await committed(submit(scheduler, lease, { kind: "start", providerId: "codex", input: START_INPUT }), 1); + await committed(submit(scheduler, lease, { kind: "stop", requestId: "stop-1" }), 2); + const reply = await handler.handle({ + requestId: 3, execution: EXECUTION, lease, ordinal: 3, + command: { kind: "narrative-delta", input: NARRATIVE_INPUT }, + }); + expect(reply.result).toEqual({ kind: "rejected", reason: "invalid-transition" }); + expect(writer.operations.map((operation) => operation.mutation.kind)).toEqual(["begin", "stop-requested"]); + scheduler.shutdown(); + }); + it("runs one synthetic turn from start through Stop, checkpoint, and finalization", async () => { const { scheduler, worker, writer, lease } = fixture(); await committed(submit(scheduler, lease, { kind: "start", providerId: "codex", input: START_INPUT }), 1); diff --git a/apps/server/src/features/agents/execution/execution-worker-handler.ts b/apps/server/src/features/agents/execution/execution-worker-handler.ts index 075089564..aa9fee940 100644 --- a/apps/server/src/features/agents/execution/execution-worker-handler.ts +++ b/apps/server/src/features/agents/execution/execution-worker-handler.ts @@ -12,6 +12,7 @@ import type { ParentAssistantTextCheckpointInput, ParentAssistantTextCheckpointResult, } from "../turns/parent-assistant-text-checkpoint-service.js"; +import type { ParentNarrativeRecoveryCommit } from "../turns/parent-turn-durability.js"; import type { ExecutionIdentity, ExecutionLease, @@ -26,6 +27,7 @@ export type ExecutionWorkCommand = | { readonly kind: "resume"; readonly providerId: string; readonly checkpointId: string } | { readonly kind: "event"; readonly phase: string; readonly nativeCursor: unknown | null; readonly events: readonly ProviderEventDraft[] } | { readonly kind: "assistant-text"; readonly inputs: readonly ParentAssistantTextCheckpointInput[] } + | { readonly kind: "narrative-delta"; readonly input: ParentNarrativeRecoveryCommit } | { readonly kind: "checkpoint"; readonly phase: string; readonly nativeCursor: unknown | null } | { readonly kind: "effect-result"; readonly effectId: string; readonly settled: boolean } | { readonly kind: "provider-outcome"; readonly outcome: TurnOutcome } @@ -44,6 +46,7 @@ export interface ExecutionSemanticOperation { | { readonly kind: "resume"; readonly providerId: string; readonly checkpointId: string } | { readonly kind: "append-events"; readonly phase: string; readonly nativeCursor: unknown | null; readonly events: readonly ProviderEventDraft[] } | { readonly kind: "append-assistant-text"; readonly inputs: readonly ParentAssistantTextCheckpointInput[] } + | { readonly kind: "narrative-delta"; readonly input: ParentNarrativeRecoveryCommit } | { readonly kind: "checkpoint"; readonly phase: string; readonly nativeCursor: unknown | null } | { readonly kind: "stop-requested"; readonly requestId: string; readonly lastAdmittedOrdinal: number } | { readonly kind: "effect-result"; readonly effectId: string; readonly settled: boolean } @@ -86,7 +89,7 @@ export interface ExecutionSemanticWriter { export type ExecutionWorkerResult = | { readonly kind: "committed"; readonly operationId: string; readonly durableRevision: number; readonly providerCommit?: ExecutionProviderCommitReceipt; readonly providerEvents?: readonly ProjectedCommittedProviderEvent[]; readonly assistantTextCheckpoint?: ParentAssistantTextCheckpointResult } | { readonly kind: "released" } - | { readonly kind: "rejected"; readonly reason: "no-execution" | "stale-execution" | "out-of-order" | "invalid-transition" | "invalid-event-routing" | "invalid-text-routing" | "invalid-stop-watermark" | "writer-conflict" }; + | { readonly kind: "rejected"; readonly reason: "no-execution" | "stale-execution" | "out-of-order" | "invalid-transition" | "invalid-event-routing" | "invalid-text-routing" | "invalid-narrative-routing" | "invalid-stop-watermark" | "writer-conflict" }; type WorkerCommand = ExecutionMailboxCommand; const METADATA_MUTATIONS: ReadonlySet = new Set([ @@ -198,6 +201,8 @@ function mutationFor( switch (command.kind) { case "event": return eventMutation(command, request.execution, state); + case "narrative-delta": + return narrativeMutation(command, request.execution, state); case "stop": return stopMutation(command, request, state); case "finalize": @@ -236,6 +241,15 @@ function eventMutation( return { kind: "append-events", phase: command.phase, nativeCursor: command.nativeCursor, events: command.events }; } +function narrativeMutation( + command: Extract, + execution: ExecutionIdentity, + state: ExecutionState, +): ExecutionSemanticOperation["mutation"] | undefined { + if (state.phase !== "running" || command.input?.executionId !== execution.executionId) return undefined; + return { kind: "narrative-delta", input: command.input }; +} + function stopMutation( command: Extract, request: ExecutionWorkerRequest, @@ -271,6 +285,9 @@ function invalidReason(request: ExecutionWorkerRequest): Extract< if (request.command.kind === "assistant-text" && !validTextRouting(request.command.inputs, request.execution)) { return "invalid-text-routing"; } + if (request.command.kind === "narrative-delta" && request.command.input?.executionId !== request.execution.executionId) { + return "invalid-narrative-routing"; + } return "invalid-transition"; } From 635ddd5132f5438ac8b43e86bae865639b67c916 Mon Sep 17 00:00:00 2001 From: "(CJ) Chukwudi Nwobodo" <142016413+chuks-qua@users.noreply.github.com> Date: Thu, 24 Sep 2026 21:47:41 +0100 Subject: [PATCH 074/136] feat(server): persist narrative deltas in semantic writer --- .../canonical-agent-writer-client.test.ts | 42 +++++++++++ ...anonical-execution-semantic-writer.test.ts | 72 +++++++++++++++++++ .../canonical-execution-semantic-writer.ts | 54 +++++++++++++- 3 files changed, 166 insertions(+), 2 deletions(-) diff --git a/apps/server/src/features/agents/canonical/__tests__/canonical-agent-writer-client.test.ts b/apps/server/src/features/agents/canonical/__tests__/canonical-agent-writer-client.test.ts index 3fdf86ed6..c835efed0 100644 --- a/apps/server/src/features/agents/canonical/__tests__/canonical-agent-writer-client.test.ts +++ b/apps/server/src/features/agents/canonical/__tests__/canonical-agent-writer-client.test.ts @@ -209,6 +209,48 @@ describe("canonical SQLite writer", () => { .toEqual({ content: "Worker durable text" }); }); + it("replays a narrative delta after the worker commits but loses its reply", async () => { + const execution = { threadId: THREAD_ID, turnId: TURN_ID, executionId: EXECUTION_ID }; + const lease = { ownerEpoch: 1, workerIndex: 0, workerGeneration: 1, leaseId: "narrative-lease" }; + const begin: ExecutionSemanticOperation = { + operationId: "narrative-lease:1", execution, lease, ordinal: 1, + mutation: { kind: "begin", providerId: "codex", input: { + thread: { id: THREAD_ID, workspaceId: "writer-workspace", providerId: "codex", createdAt: NOW }, + turnId: TURN_ID, executionId: EXECUTION_ID, permissionMode: "supervised", providerIdentities: [], + userMessage: { kind: "create", messageId: "narrative-user", content: "Question", sequence: 1 }, + } }, + }; + writer = new CanonicalAgentWriterClient(dbPath); + expect((await writer.transactSemantic(begin, () => {})).kind).toBe("committed"); + await writer.close(); + + let created = 0; + const published: string[] = []; + writer = new CanonicalAgentWriterClient(dbPath, () => created++ === 0 + ? workerDroppingReply("semantic-transacted") + : new Worker(new URL("../canonical-agent-writer.worker.ts", import.meta.url), { type: "module" })); + const delta: ExecutionSemanticOperation = { + operationId: "narrative-lease:2", execution, lease, ordinal: 2, + mutation: { kind: "narrative-delta", input: { + executionId: EXECUTION_ID, items: [recoveryToolCall()], discardedItemIds: [], + } }, + }; + const receipt = await writer.transactSemantic(delta, (events) => published.push(...events.map((event) => event.eventId))); + expect(receipt).toMatchObject({ kind: "committed", operationId: "narrative-lease:2" }); + expect(created).toBe(2); + expect(published).toEqual([]); + expect(db.prepare("SELECT COUNT(*) AS count FROM canonical_agent_items WHERE id = ?") + .get("toolCall:writer-recovery-tool")).toEqual({ count: 1 }); + await writer.close(); + + writer = new CanonicalAgentWriterClient(dbPath); + expect(await writer.transactSemantic(delta, () => {})).toEqual(receipt); + expect(await writer.interruptWorkerLoss({ execution, lease, reason: "worker exited", recoveryIncidentId: "narrative-loss" }, + () => {})).toMatchObject({ kind: "committed", operationId: "narrative-lease:worker-lost" }); + expect(db.prepare("SELECT id, status FROM tool_call_records WHERE id = ?") + .get("writer-recovery-tool")).toEqual({ id: "writer-recovery-tool", status: "completed" }); + }); + it("rejects a database failure without reporting a durable receipt", async () => { writer = new CanonicalAgentWriterClient(dbPath); await writer.whenReady(); diff --git a/apps/server/src/features/agents/canonical/__tests__/canonical-execution-semantic-writer.test.ts b/apps/server/src/features/agents/canonical/__tests__/canonical-execution-semantic-writer.test.ts index a13cb53ec..69790807d 100644 --- a/apps/server/src/features/agents/canonical/__tests__/canonical-execution-semantic-writer.test.ts +++ b/apps/server/src/features/agents/canonical/__tests__/canonical-execution-semantic-writer.test.ts @@ -10,6 +10,8 @@ import { MessageRepo } from "../../conversation/persistence/message-repo.js"; import type { ExecutionSemanticOperation } from "../../execution/execution-worker-handler.js"; import { ExecutionWorkerHandler } from "../../execution/execution-worker-handler.js"; import { ParentAssistantTextCheckpointService } from "../../turns/parent-assistant-text-checkpoint-service.js"; +import { NarrativeRecoveryDelta } from "../../turns/narrative-recovery-delta.js"; +import { CanonicalAgentBoundary } from "../canonical-agent-boundary.js"; import { CanonicalExecutionSemanticWriter } from "../canonical-execution-semantic-writer.js"; import type { DataOnlyParentTurnStartInput } from "../canonical-parent-turn-write.js"; @@ -247,6 +249,76 @@ describe("CanonicalExecutionSemanticWriter through ExecutionWorkerHandler", () = expect(await writer.transact(input)).toMatchObject({ kind: "committed", operationId: "lease-1:2" }); }); + it("replays narrative changes and discards, then recovers only the saved narrative after worker loss", async () => { + expect((await send(1, { kind: "start", providerId: "codex", input: startInput() })).kind).toBe("committed"); + const canonical = new CanonicalAgentBoundary(db, () => {}); + const reducer = new NarrativeRecoveryDelta(); + const publicationCount = published.length; + const initialDelta = reducer.prepare(toolNarrative("", 1)); + if (!initialDelta) throw new Error("Expected an initial narrative delta"); + const first = operation(2, { kind: "narrative-delta", input: { + executionId: EXECUTION_ID, items: initialDelta.items, discardedItemIds: initialDelta.discardedItemIds, + } }); + expect((await writer.transact(first)).kind).toBe("committed"); + initialDelta.acknowledge(); + expect(canonical.loadParentNarrativeRecovery(TURN_ID)).toHaveLength(1); + expect(published).toHaveLength(publicationCount); + const discardedDelta = reducer.prepare([]); + if (!discardedDelta) throw new Error("Expected a discarded narrative delta"); + const discard = operation(3, { kind: "narrative-delta", input: { + executionId: EXECUTION_ID, items: discardedDelta.items, discardedItemIds: discardedDelta.discardedItemIds, + } }); + expect((await writer.transact(discard)).kind).toBe("committed"); + discardedDelta.acknowledge(); + expect(canonical.loadParentNarrativeRecovery(TURN_ID)).toEqual([]); + const latestDelta = reducer.prepare([toolNarrative("", 2)[1]!]); + if (!latestDelta) throw new Error("Expected a new narrative delta"); + const latest = operation(4, { kind: "narrative-delta", input: { + executionId: EXECUTION_ID, items: latestDelta.items, discardedItemIds: latestDelta.discardedItemIds, + } }); + const receipt = await writer.transact(latest); + expect(receipt).toMatchObject({ kind: "committed", operationId: "lease-1:4" }); + latestDelta.acknowledge(); + expect(published).toHaveLength(publicationCount); + + db.close(true); + db = openDatabase({ dbPath: path }); + writer = new CanonicalExecutionSemanticWriter(db, (events) => published.push(...events.map((item) => item.eventId))); + expect(await writer.transact(first)).toMatchObject({ kind: "committed", operationId: "lease-1:2" }); + expect(await writer.transact(discard)).toMatchObject({ kind: "committed", operationId: "lease-1:3" }); + expect(await writer.transact(latest)).toEqual(receipt); + expect(db.prepare("SELECT COUNT(*) AS count FROM canonical_agent_items WHERE id = ?").get("toolCall:tool-1")) + .toEqual({ count: 1 }); + expect(db.prepare("SELECT COUNT(*) AS count FROM canonical_agent_items WHERE id = ?").get("toolCall:tool-0")) + .toEqual({ count: 0 }); + expect(writer.interruptWorkerLoss(loss).kind).toBe("committed"); + expect(db.prepare("SELECT id, status FROM tool_call_records WHERE id = ?").get("tool-1")) + .toEqual({ id: "tool-1", status: "completed" }); + expect(db.prepare("SELECT COUNT(*) AS count FROM tool_call_records WHERE id = ?").get("tool-0")) + .toEqual({ count: 0 }); + }); + + it("rolls back narrative changes when the semantic receipt cannot commit", async () => { + expect((await send(1, { kind: "start", providerId: "codex", input: startInput() })).kind).toBe("committed"); + const delta = operation(2, { kind: "narrative-delta", input: { + executionId: EXECUTION_ID, items: toolNarrative("", 1), discardedItemIds: [], + } }); + expect(await writer.transact(operation(2, { kind: "narrative-delta", input: { + executionId: "wrong-execution", items: toolNarrative("", 1), discardedItemIds: [], + } }))).toEqual({ kind: "conflict", operationId: "lease-1:2" }); + expect(await writer.transact(operation(2, { kind: "narrative-delta", input: { + executionId: EXECUTION_ID, items: [{ ...toolNarrative("", 1)[0]!, record: { + ...toolNarrative("", 1)[0]!.record, input_summary: "x".repeat(256 * 1024 + 1), + } }], discardedItemIds: [], + } }))).toEqual({ kind: "conflict", operationId: "lease-1:2" }); + db.run("CREATE TRIGGER fail_narrative_receipt BEFORE INSERT ON canonical_writer_operation_receipts WHEN NEW.kind = 'semantic:narrative-delta' BEGIN SELECT RAISE(ABORT, 'narrative receipt unavailable'); END"); + await expect(writer.transact(delta)).rejects.toThrow("narrative receipt unavailable"); + expect(db.prepare("SELECT COUNT(*) AS count FROM canonical_agent_items WHERE id = ?").get("toolCall:tool-0")) + .toEqual({ count: 0 }); + db.run("DROP TRIGGER fail_narrative_receipt"); + expect((await writer.transact(delta)).kind).toBe("committed"); + }); + it("rejects stale worker-loss leases without changing the unfinished turn", async () => { expect((await send(1, { kind: "start", providerId: "codex", input: startInput() })).kind).toBe("committed"); expect(writer.interruptWorkerLoss({ ...loss, lease: { ...lease, ownerEpoch: 2 } })) diff --git a/apps/server/src/features/agents/canonical/canonical-execution-semantic-writer.ts b/apps/server/src/features/agents/canonical/canonical-execution-semantic-writer.ts index 71e010c98..e99e7d6c7 100644 --- a/apps/server/src/features/agents/canonical/canonical-execution-semantic-writer.ts +++ b/apps/server/src/features/agents/canonical/canonical-execution-semantic-writer.ts @@ -3,12 +3,14 @@ import * as NodeCrypto from "node:crypto"; import { AgentEventSchema, CanonicalAgentEventEnvelopeSchema, + ParentNarrativeRecoveryItemSchema, ProviderIdSchema, TurnOutcomeSchema, type TurnOutcome, } from "@mcode/contracts"; import { z } from "zod"; +import { ACTIVE_TURN_WRITE_BATCH_LIMITS } from "../../../runtime/persistence/sqlite/bounded-write-batches.js"; import type { ExecutionIdentity, ExecutionLease } from "../execution/execution-mailbox-protocol.js"; import type { ExecutionProviderCommitReceipt, @@ -35,6 +37,11 @@ const PENDING_FINISH_KIND = "semantic:finish-pending"; const PUBLICATION_CHUNK_SIZE = 64; const PUBLICATION_CHUNK_PAGE_SIZE = 16; const MAX_BUFFERED_PUBLICATION_EVENTS = 2_048; +const narrativeDeltaSchema = z.object({ + executionId: z.string(), + items: z.array(ParentNarrativeRecoveryItemSchema()), + discardedItemIds: z.array(z.string().regex(/^(toolCall|narrationSegment|hook):.+$/)).optional(), +}); const storedPublicationSequencesSchema = z.array(z.union([ z.number().int().positive().max(Number.MAX_SAFE_INTEGER), z.object({ executionId: z.string(), sequence: z.number().int().positive().max(Number.MAX_SAFE_INTEGER) }), @@ -128,6 +135,7 @@ export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter private readonly turns: CanonicalParentTurnWrite; private readonly providerProjector: CanonicalCommittedProviderProjector; private readonly assistantText: ParentAssistantTextCheckpointService; + private readonly canonical: CanonicalAgentBoundary; private readonly findOperation: ReturnType; private readonly listPublicationChunks: ReturnType; private readonly findPublishedEvent: ReturnType; @@ -149,6 +157,7 @@ export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter if (!this.bufferedPublication) throw new Error("Codex projection requires a semantic transaction"); this.bufferPublication(events); }); + this.canonical = codexBoundary; this.providerProjector = new CanonicalCommittedProviderProjector(codexBoundary); this.assistantText = new ParentAssistantTextCheckpointService(db); this.findOperation = db.prepare("SELECT kind, input_hash, receipt_json FROM canonical_writer_operation_receipts WHERE execution_id = ? AND operation_id = ?"); @@ -232,6 +241,7 @@ export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter case "begin": return this.begin(operation, hash); case "append-events": return this.append(operation, hash); case "append-assistant-text": return this.appendAssistantText(operation, hash); + case "narrative-delta": return this.recordNarrativeDelta(operation, hash); case "checkpoint": case "stop-requested": case "provider-outcome": return this.control(operation, hash); @@ -334,6 +344,24 @@ export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter })(); } + private recordNarrativeDelta(operation: ExecutionSemanticOperation, hash: string): ExecutionWriteReceipt { + const mutation = operation.mutation; + if (mutation.kind !== "narrative-delta") return conflict(operation); + return this.db.transaction(() => { + const head = this.requireNextHead(operation); + const checkpoint = this.canonical.loadCheckpoint(operation.execution.executionId); + if (!checkpoint || checkpoint.threadId !== operation.execution.threadId + || checkpoint.turnId !== operation.execution.turnId || checkpoint.terminalOutcome !== null) { + throw new SemanticConflict(); + } + if (!this.canonical.recordParentNarrativeRecovery(mutation.input)) throw new SemanticConflict(); + const receipt = committed(operation, head.durableRevision); + this.storeHead({ ...head, ordinal: operation.ordinal }); + this.storeReceipt(operation, hash, receipt); + return receipt; + })(); + } + private async finish(operation: ExecutionSemanticOperation, hash: string): Promise { const mutation = operation.mutation; if (mutation.kind !== "finish" || !validFinish(operation, mutation)) return conflict(operation); @@ -579,8 +607,30 @@ function validOperation(operation: ExecutionSemanticOperation): boolean { && operation.operationId !== HEAD_ID && operation.operationId.length <= 256 && Number.isSafeInteger(operation.ordinal) && operation.ordinal > 0 && validIdentity(operation.execution) && validLease(operation.lease) - && (operation.mutation.kind !== "append-assistant-text" - || validAssistantTextInput(operation.mutation.inputs, operation.execution)); + && validSemanticMutationInput(operation); +} + +function validSemanticMutationInput(operation: ExecutionSemanticOperation): boolean { + if (operation.mutation.kind === "append-assistant-text") { + return validAssistantTextInput(operation.mutation.inputs, operation.execution); + } + if (operation.mutation.kind === "narrative-delta") { + return validNarrativeDeltaInput(operation.mutation.input, operation.execution); + } + return true; +} + +function validNarrativeDeltaInput( + input: Extract["input"], + execution: ExecutionIdentity, +): boolean { + if (!input || input.executionId !== execution.executionId || !Array.isArray(input.items)) return false; + const discarded = input.discardedItemIds ?? []; + if (!Array.isArray(discarded)) return false; + const count = input.items.length + discarded.length; + if (count === 0 || count > ACTIVE_TURN_WRITE_BATCH_LIMITS.maxRows - 2) return false; + return Buffer.byteLength(JSON.stringify(input), "utf8") <= ACTIVE_TURN_WRITE_BATCH_LIMITS.maxBytes + && narrativeDeltaSchema.safeParse(input).success; } function validAssistantTextInput( From fa60335fec583370f338d0c29833bb1354020907 Mon Sep 17 00:00:00 2001 From: "(CJ) Chukwudi Nwobodo" <142016413+chuks-qua@users.noreply.github.com> Date: Thu, 24 Sep 2026 21:53:38 +0100 Subject: [PATCH 075/136] feat(server): bound narrative delta writer commands --- .../narrative-recovery-delta.test.ts | 36 ++++++++++++++++++- .../agents/turns/narrative-recovery-delta.ts | 36 +++++++++++++++++++ 2 files changed, 71 insertions(+), 1 deletion(-) diff --git a/apps/server/src/features/agents/turns/__tests__/narrative-recovery-delta.test.ts b/apps/server/src/features/agents/turns/__tests__/narrative-recovery-delta.test.ts index c379b5042..74664a98a 100644 --- a/apps/server/src/features/agents/turns/__tests__/narrative-recovery-delta.test.ts +++ b/apps/server/src/features/agents/turns/__tests__/narrative-recovery-delta.test.ts @@ -1,7 +1,7 @@ import { describe, expect, it } from "vitest"; import type { ParentNarrativeRecoveryItem } from "@mcode/contracts"; -import { NarrativeRecoveryDelta } from "../narrative-recovery-delta.js"; +import { NarrativeRecoveryDelta, splitNarrativeRecoveryDelta } from "../narrative-recovery-delta.js"; const thought = { kind: "narrationSegment", @@ -53,4 +53,38 @@ describe("NarrativeRecoveryDelta", () => { expect(secondExecution.prepare([thought])).toMatchObject({ items: [thought] }); expect(secondExecution.prepare([])).toBeNull(); }); + + it("splits a large delta without acknowledging any chunk early", () => { + const delta = new NarrativeRecoveryDelta(); + const snapshot = Array.from({ length: 63 }, (_, index) => ({ + ...thought, + record: { ...thought.record, id: `thought-${index}` }, + })); + const prepared = delta.prepare(snapshot); + expect(prepared).not.toBeNull(); + if (!prepared) return; + + const chunks = splitNarrativeRecoveryDelta("execution-1", prepared); + expect(chunks.map((chunk) => chunk.items.length)).toEqual([62, 1]); + expect(chunks.flatMap((chunk) => chunk.items)).toEqual(snapshot); + expect(delta.prepare(snapshot)?.items).toHaveLength(63); + prepared.acknowledge(); + expect(delta.prepare(snapshot)).toBeNull(); + }); + + it("splits by serialized bytes and rejects an item larger than one command", () => { + const delta = new NarrativeRecoveryDelta(); + const snapshot = ["first", "second"].map((id) => ({ + ...thought, + record: { ...thought.record, id, text: "x".repeat(150_000) }, + })); + const prepared = delta.prepare(snapshot); + expect(prepared).not.toBeNull(); + if (!prepared) return; + expect(splitNarrativeRecoveryDelta("execution-1", prepared).map((chunk) => chunk.items.length)).toEqual([1, 1]); + const oversized = delta.prepare([{ ...thought, record: { ...thought.record, text: "x".repeat(300_000) } }]); + expect(oversized).not.toBeNull(); + if (!oversized) return; + expect(() => splitNarrativeRecoveryDelta("execution-1", oversized)).toThrow("exceeds one writer command"); + }); }); diff --git a/apps/server/src/features/agents/turns/narrative-recovery-delta.ts b/apps/server/src/features/agents/turns/narrative-recovery-delta.ts index 11fe41ab5..6e9e60d86 100644 --- a/apps/server/src/features/agents/turns/narrative-recovery-delta.ts +++ b/apps/server/src/features/agents/turns/narrative-recovery-delta.ts @@ -1,4 +1,6 @@ import type { ParentNarrativeRecoveryItem } from "@mcode/contracts"; +import { ACTIVE_TURN_WRITE_BATCH_LIMITS } from "../../../runtime/persistence/sqlite/bounded-write-batches.js"; +import type { ParentNarrativeRecoveryCommit } from "./parent-turn-durability.js"; /** One full-snapshot difference awaiting confirmation of its durable write. */ export interface PreparedNarrativeRecoveryDelta { @@ -38,6 +40,40 @@ export class NarrativeRecoveryDelta { } } +/** Split one prepared delta into writer-sized commands; acknowledge only after every command commits. */ +export function splitNarrativeRecoveryDelta( + executionId: string, + prepared: PreparedNarrativeRecoveryDelta, +): ParentNarrativeRecoveryCommit[] { + if (!executionId) throw new Error("Narrative recovery requires an execution ID"); + const chunks: ParentNarrativeRecoveryCommit[] = []; + let items: ParentNarrativeRecoveryItem[] = []; + let discardedItemIds: string[] = []; + const flush = (): void => { + if (items.length === 0 && discardedItemIds.length === 0) return; + chunks.push({ executionId, items, discardedItemIds }); + items = []; + discardedItemIds = []; + }; + const fits = (nextItems: ParentNarrativeRecoveryItem[], nextDiscarded: string[]): boolean => { + if (nextItems.length + nextDiscarded.length > ACTIVE_TURN_WRITE_BATCH_LIMITS.maxRows - 2) return false; + return Buffer.byteLength(JSON.stringify({ executionId, items: nextItems, discardedItemIds: nextDiscarded }), "utf8") + <= ACTIVE_TURN_WRITE_BATCH_LIMITS.maxBytes; + }; + for (const item of prepared.items) { + if (!fits([...items, item], discardedItemIds)) flush(); + if (!fits([item], [])) throw new Error("Narrative recovery item exceeds one writer command"); + items.push(item); + } + for (const itemId of prepared.discardedItemIds) { + if (!fits(items, [...discardedItemIds, itemId])) flush(); + if (!fits([], [itemId])) throw new Error("Narrative recovery discard exceeds one writer command"); + discardedItemIds.push(itemId); + } + flush(); + return chunks; +} + function canonicalItemId(key: string): string { const separator = key.indexOf(":"); if (separator <= 0 || separator === key.length - 1) { From 5577bf11bfd65693da3e2cf83b0c2678df1ded60 Mon Sep 17 00:00:00 2001 From: "(CJ) Chukwudi Nwobodo" <142016413+chuks-qua@users.noreply.github.com> Date: Thu, 24 Sep 2026 22:07:32 +0100 Subject: [PATCH 076/136] feat(server): preserve assigned assistant ID in writer terminal projection --- .../canonical-parent-turn-write.test.ts | 26 +++++++++++++++++++ .../canonical/canonical-parent-turn-write.ts | 22 +++++++++++----- 2 files changed, 42 insertions(+), 6 deletions(-) diff --git a/apps/server/src/features/agents/canonical/__tests__/canonical-parent-turn-write.test.ts b/apps/server/src/features/agents/canonical/__tests__/canonical-parent-turn-write.test.ts index 6b6be5b13..c10d690f6 100644 --- a/apps/server/src/features/agents/canonical/__tests__/canonical-parent-turn-write.test.ts +++ b/apps/server/src/features/agents/canonical/__tests__/canonical-parent-turn-write.test.ts @@ -10,6 +10,7 @@ import { openDatabase } from "../../../../runtime/persistence/sqlite/database.js import { MessageRepo } from "../../conversation/persistence/message-repo.js"; import { PlanQuestionAnswersRepo } from "../../planning/persistence/plan-question-answers-repo.js"; import { ToolCallRecordRepo } from "../../tools/persistence/tool-call-record-repo.js"; +import { deriveTurnAssistantMessageId } from "../../turns/turn-assistant-message-id.js"; import { CanonicalParentTurnWrite, type DataOnlyParentTerminalProjectionInput, @@ -249,6 +250,31 @@ describe("CanonicalParentTurnWrite", () => { expect(writer.stageTerminalProjection(input).messageId).toBe(first.messageId); }); + it("keeps an assigned public assistant identity through writer staging and reload", async () => { + writer.start(startInput()); + const messageId = deriveTurnAssistantMessageId(THREAD_ID, "user-1"); + const input = terminalProjectionInput(); + input.assistant.messageId = messageId; + const staged = writer.stageTerminalProjection(input); + expect(staged.messageId).toBe(messageId); + + db.close(true); + db = openDatabase({ dbPath: path }); + writer = new CanonicalParentTurnWrite(db, (events) => { + published.push(events.map((event) => event.eventId)); + }); + expect(writer.stageTerminalProjection(input).messageId).toBe(messageId); + const finish = { ...finishInput(staged.projection.message!), projection: { kind: "writer-staged" as const, messageId } }; + expect(() => writer.finish({ ...finish, projection: { kind: "writer-staged", messageId: "0".repeat(64) } })) + .toThrow("Staged assistant projection not found"); + expect(db.prepare("SELECT terminal_outcome FROM canonical_agent_ingest_checkpoints WHERE execution_id = ?").get(EXECUTION_ID)) + .toEqual({ terminal_outcome: null }); + expect((await writer.finish(finish)).outcome).toBe("committed"); + expect(new MessageRepo(db).findByIdInThread(THREAD_ID, messageId)).toMatchObject({ + is_internal: false, outcome: "completed", content: input.assistant.content, + }); + }); + it("rolls back all staged rows when a narrative write fails", async () => { writer.start(startInput()); const input = terminalProjectionInput(); diff --git a/apps/server/src/features/agents/canonical/canonical-parent-turn-write.ts b/apps/server/src/features/agents/canonical/canonical-parent-turn-write.ts index b0b5723ab..f9fd685b4 100644 --- a/apps/server/src/features/agents/canonical/canonical-parent-turn-write.ts +++ b/apps/server/src/features/agents/canonical/canonical-parent-turn-write.ts @@ -92,7 +92,7 @@ export interface DataOnlyParentEventInput extends Omit { - projection: ParentTurnProjection | { readonly kind: "writer-staged" }; + projection: ParentTurnProjection | { readonly kind: "writer-staged"; readonly messageId?: string }; } /** Cloneable terminal data whose compatibility rows are staged on the writer connection. */ @@ -101,7 +101,7 @@ export interface DataOnlyParentTerminalProjectionInput { executionId: string; outcome: TurnOutcome; endedAt: string; - assistant: { content: string; model: string | null; attachments: readonly StoredAttachment[] }; + assistant: { content: string; model: string | null; attachments: readonly StoredAttachment[]; messageId?: string }; narrative: readonly ParentNarrativeRecoveryItem[]; } @@ -230,6 +230,9 @@ export class CanonicalParentTurnWrite { if (!input.threadId || !input.executionId || !Number.isFinite(Date.parse(input.endedAt))) { throw new Error("Invalid parent terminal projection identity or end time"); } + if (input.assistant.messageId !== undefined && !/^[0-9a-f]{64}$/.test(input.assistant.messageId)) { + throw new Error("Invalid parent assistant message identity"); + } const narrative = input.narrative.map((item) => ParentNarrativeRecoveryItemSchema().parse(item)); return this.db.transaction(() => this.stageTerminalProjectionInTransaction(input, narrative))(); } @@ -263,6 +266,9 @@ export class CanonicalParentTurnWrite { private persistedTerminalProjection(input: DataOnlyParentTerminalProjectionInput, turnId: string): StagedParentTerminalProjection { const persisted = this.canonical.loadTerminalProjection(turnId); + if (input.assistant.messageId && persisted.message?.id !== input.assistant.messageId) { + throw new Error(`Canonical terminal projection has a different assistant identity: ${input.executionId}`); + } if (!persisted.message) { if (input.assistant.content.trim() || input.assistant.attachments.length > 0 || input.narrative.length > 0) { throw new Error(`Canonical terminal projection is empty: ${input.executionId}`); @@ -274,7 +280,7 @@ export class CanonicalParentTurnWrite { } private stageAssistant(input: DataOnlyParentTerminalProjectionInput) { - const id = deriveTurnAssistantMessageId(input.threadId, `execution:${input.executionId}`); + const id = input.assistant.messageId ?? deriveTurnAssistantMessageId(input.threadId, `execution:${input.executionId}`); const existing = this.messages.findByIdInThreadIncludingInternal(input.threadId, id); if (existing) { this.assertAssistantMatches(existing, input); @@ -318,7 +324,7 @@ export class CanonicalParentTurnWrite { onBatchWrite?: (batch: CanonicalTerminalBatchWrite) => void, ): Promise { const staged = "kind" in input.projection - ? this.loadStagedTerminalProjection(input.threadId, input.executionId) + ? this.loadStagedTerminalProjection(input.threadId, input.executionId, input.projection.messageId) : input.projection; this.assertStagedAssistant(input.threadId, staged); const projection: ParentTurnProjection = { @@ -343,9 +349,13 @@ export class CanonicalParentTurnWrite { }, onBatchWrite); } - private loadStagedTerminalProjection(threadId: string, executionId: string): ParentTurnProjection { - const id = deriveTurnAssistantMessageId(threadId, `execution:${executionId}`); + private loadStagedTerminalProjection(threadId: string, executionId: string, messageId?: string): ParentTurnProjection { + if (messageId !== undefined && !/^[0-9a-f]{64}$/.test(messageId)) { + throw new Error("Invalid staged assistant message identity"); + } + const id = messageId ?? deriveTurnAssistantMessageId(threadId, `execution:${executionId}`); const message = this.messages.findByIdInThreadIncludingInternal(threadId, id); + if (!message && messageId) throw new Error(`Staged assistant projection not found: ${messageId}`); if (!message) return { message: null, narrative: [] }; if (message.role !== "assistant" || !message.is_internal) { throw new Error(`Staged assistant projection is not private: ${id}`); From a8d04bc128b49e7408ad0ec0ef9158e9c13ac190 Mon Sep 17 00:00:00 2001 From: "(CJ) Chukwudi Nwobodo" <142016413+chuks-qua@users.noreply.github.com> Date: Thu, 24 Sep 2026 22:08:29 +0100 Subject: [PATCH 077/136] fix(server): reject public assistant rows during writer staging --- .../__tests__/canonical-parent-turn-write.test.ts | 15 +++++++++++++++ .../canonical/canonical-parent-turn-write.ts | 1 + 2 files changed, 16 insertions(+) diff --git a/apps/server/src/features/agents/canonical/__tests__/canonical-parent-turn-write.test.ts b/apps/server/src/features/agents/canonical/__tests__/canonical-parent-turn-write.test.ts index c10d690f6..2d26450cd 100644 --- a/apps/server/src/features/agents/canonical/__tests__/canonical-parent-turn-write.test.ts +++ b/apps/server/src/features/agents/canonical/__tests__/canonical-parent-turn-write.test.ts @@ -275,6 +275,21 @@ describe("CanonicalParentTurnWrite", () => { }); }); + it("refuses to reuse a public assistant row as a staged turn projection", () => { + writer.start(startInput()); + const input = terminalProjectionInput(); + const messageId = deriveTurnAssistantMessageId(THREAD_ID, "user-1"); + input.assistant.messageId = messageId; + new MessageRepo(db).createAssistantIdempotent({ + id: messageId, threadId: THREAD_ID, content: input.assistant.content, + sequence: 2, model: input.assistant.model, attachments: [...input.assistant.attachments], + }); + + expect(() => writer.stageTerminalProjection(input)).toThrow("already public"); + expect(db.prepare("SELECT terminal_outcome FROM canonical_agent_ingest_checkpoints WHERE execution_id = ?").get(EXECUTION_ID)) + .toEqual({ terminal_outcome: null }); + }); + it("rolls back all staged rows when a narrative write fails", async () => { writer.start(startInput()); const input = terminalProjectionInput(); diff --git a/apps/server/src/features/agents/canonical/canonical-parent-turn-write.ts b/apps/server/src/features/agents/canonical/canonical-parent-turn-write.ts index f9fd685b4..74d9389d3 100644 --- a/apps/server/src/features/agents/canonical/canonical-parent-turn-write.ts +++ b/apps/server/src/features/agents/canonical/canonical-parent-turn-write.ts @@ -283,6 +283,7 @@ export class CanonicalParentTurnWrite { const id = input.assistant.messageId ?? deriveTurnAssistantMessageId(input.threadId, `execution:${input.executionId}`); const existing = this.messages.findByIdInThreadIncludingInternal(input.threadId, id); if (existing) { + if (!existing.is_internal) throw new Error(`Staged assistant projection is already public: ${id}`); this.assertAssistantMatches(existing, input); return existing; } From e6c3e5647d29a01a83c7083e19e7dc958cba2f29 Mon Sep 17 00:00:00 2001 From: "(CJ) Chukwudi Nwobodo" <142016413+chuks-qua@users.noreply.github.com> Date: Thu, 24 Sep 2026 22:08:40 +0100 Subject: [PATCH 078/136] feat(server): project Codex parent assistant messages as data --- .../codex-parent-message-projection.test.ts | 149 ++++++++++++++++++ .../turns/codex-parent-message-projection.ts | 117 ++++++++++++++ 2 files changed, 266 insertions(+) create mode 100644 apps/server/src/features/agents/turns/__tests__/codex-parent-message-projection.test.ts create mode 100644 apps/server/src/features/agents/turns/codex-parent-message-projection.ts diff --git a/apps/server/src/features/agents/turns/__tests__/codex-parent-message-projection.test.ts b/apps/server/src/features/agents/turns/__tests__/codex-parent-message-projection.test.ts new file mode 100644 index 000000000..c9751b8bd --- /dev/null +++ b/apps/server/src/features/agents/turns/__tests__/codex-parent-message-projection.test.ts @@ -0,0 +1,149 @@ +import "reflect-metadata"; +import * as NodeFS from "node:fs"; +import * as NodeOS from "node:os"; +import * as NodePath from "node:path"; +import type { Database } from "bun:sqlite"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; + +import { openDatabase } from "../../../../runtime/persistence/sqlite/database.js"; +import { MessageRepo } from "../../conversation/persistence/message-repo.js"; +import { NarrativeStore } from "../../conversation/narrative/narrative-store.js"; +import { ThoughtSegmentRepo } from "../../conversation/narrative/persistence/thought-segment-repo.js"; +import { HookExecutionRepo } from "../../events/persistence/hook-execution-repo.js"; +import { ToolCallRecordRepo } from "../../tools/persistence/tool-call-record-repo.js"; +import { ThreadRepo } from "../../../thread-control/persistence/thread-repo.js"; +import { SnapshotService } from "../../../projects/diffs/snapshots/snapshot-service.js"; +import { RealGitExecutor } from "../../../projects/git/execution/real-git-executor.js"; +import { TurnSnapshotRepo } from "../persistence/turn-snapshot-repo.js"; +import { TurnFinalizer } from "../turn-finalizer.js"; +import { CodexParentMessageProjection } from "../codex-parent-message-projection.js"; +import { deriveTurnAssistantMessageId } from "../turn-assistant-message-id.js"; + +const execution = { threadId: "thread-1", turnId: "turn-1", executionId: "execution-1" } as const; +const endedAt = "2026-09-24T10:00:00.000Z"; +const image = { id: "image-1", name: "image.png", mimeType: "image/png", sizeBytes: 12 }; + +describe("CodexParentMessageProjection", () => { + let directory: string; + let path: string; + let db: Database; + let messages: MessageRepo; + + beforeEach(() => { + directory = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "mcode-codex-message-projection-")); + path = NodePath.join(directory, "mcode.db"); + db = openDatabase({ dbPath: path }); + messages = new MessageRepo(db); + db.prepare("INSERT INTO workspaces (id, name, path, created_at, updated_at) VALUES (?, ?, ?, ?, ?)") + .run("workspace-1", "Workspace", "C:/fixture", endedAt, endedAt); + db.prepare("INSERT INTO threads (id, workspace_id, title, branch, provider, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?)") + .run(execution.threadId, "workspace-1", "Thread", "main", "codex", endedAt, endedAt); + messages.create(execution.threadId, "user", "Question", 1, undefined, undefined, undefined, + undefined, undefined, undefined, undefined, undefined, "user-1"); + }); + + afterEach(() => { + db.close(true); + NodeFS.rmSync(directory, { recursive: true, force: true }); + }); + + function finalizer(): TurnFinalizer { + return new TurnFinalizer( + messages, + new ThreadRepo(db), + new NarrativeStore(messages, new ToolCallRecordRepo(db), new ThoughtSegmentRepo(db), new HookExecutionRepo(db)), + new SnapshotService(new RealGitExecutor()), + new TurnSnapshotRepo(db), + db, + ); + } + + function projection(): CodexParentMessageProjection { + return new CodexParentMessageProjection({ execution, turnKind: "ordinary" }); + } + + it("matches the live finalizer after an intervening notice and survives repeat and reload", () => { + const live = finalizer(); + const worker = projection(); + live.bufferAssistantAttachments(execution.threadId, [image]); + worker.bufferGeneratedAttachment(execution, image); + messages.create(execution.threadId, "system", "Notice", 2, undefined, undefined, undefined, + undefined, undefined, undefined, undefined, undefined, "notice-1"); + + const publicMessage = worker.projectMessage({ + execution, content: "Answer", model: "codex-model", precedingMessageId: "notice-1", + postTurnGoalReceipt: false, + }); + const liveMessageId = live.bufferAssistantBody(execution.threadId, "Answer", "codex-model"); + expect(publicMessage).toEqual({ messageId: liveMessageId, model: "codex-model", attachments: [image] }); + expect(liveMessageId).toBe(deriveTurnAssistantMessageId(execution.threadId, "notice-1")); + + const terminal = worker.projectTerminal({ execution, outcome: "completed", endedAt, + fallbackModel: "later-model", narrative: [] }); + expect(structuredClone(terminal)).toEqual(terminal); + expect(terminal).toMatchObject({ fromProvider: true, + assistant: { messageId: publicMessage.messageId, content: "Answer", model: "codex-model", attachments: [image] } }); + expect(worker.projectTerminal({ execution, outcome: "completed", endedAt, + fallbackModel: "later-model", narrative: [] })).toEqual(terminal); + + messages.createAssistantIdempotent({ id: terminal.assistant.messageId, threadId: execution.threadId, + content: terminal.assistant.content, model: terminal.assistant.model, + attachments: [...terminal.assistant.attachments], sequence: 3 }); + db.close(true); + db = openDatabase({ dbPath: path }); + messages = new MessageRepo(db); + expect(messages.listByThread(execution.threadId, 10).messages.filter((message) => message.role === "assistant")) + .toEqual([expect.objectContaining({ id: publicMessage.messageId, model: "codex-model", attachments: [image] })]); + + const replayWorker = projection(); + replayWorker.bufferGeneratedAttachment(execution, image); + const replay = replayWorker.projectMessage({ execution, content: "Answer", model: "codex-model", + precedingMessageId: "notice-1", postTurnGoalReceipt: false }); + expect(replay).toEqual(publicMessage); + messages.createAssistantIdempotent({ id: replay.messageId, threadId: execution.threadId, + content: "Answer", model: "codex-model", sequence: 3 }); + expect(messages.listByThread(execution.threadId, 10).messages.filter((message) => message.role === "assistant")) + .toHaveLength(1); + }); + + it("keeps the published ID when a later notice changes the last visible row", () => { + const worker = projection(); + const publicMessage = worker.projectMessage({ execution, content: "Answer", model: null, + precedingMessageId: "user-1", postTurnGoalReceipt: false }); + messages.create(execution.threadId, "system", "Later notice", 2); + + const terminal = worker.projectTerminal({ execution, outcome: "completed", endedAt, + precedingMessageId: messages.listByThread(execution.threadId, 1).messages[0]?.id, + fallbackModel: "new-model", narrative: [] }); + expect(terminal.assistant.messageId).toBe(publicMessage.messageId); + expect(terminal.assistant.model).toBeNull(); + }); + + it("derives a text-only fallback ID at terminal time and fences a prior execution", () => { + const worker = projection(); + const stale = { ...execution, executionId: "old-execution" }; + expect(() => worker.appendFinalResponseText(stale, "wrong")).toThrow(/Stale Codex/); + expect(() => worker.bufferGeneratedAttachment(stale, image)).toThrow(/Stale Codex/); + expect(() => worker.projectMessage({ execution: stale, content: "wrong", model: null, + precedingMessageId: "user-1", postTurnGoalReceipt: false })).toThrow(/Stale Codex/); + worker.appendFinalResponseText(execution, " partial answer "); + const terminal = worker.projectTerminal({ execution, outcome: "cancelled", endedAt, + precedingMessageId: "user-1", fallbackModel: "codex-fallback", narrative: [] }); + expect(terminal).toMatchObject({ fromProvider: false, + assistant: { messageId: deriveTurnAssistantMessageId(execution.threadId, "user-1"), + content: "partial answer", model: "codex-fallback", attachments: [] } }); + expect(() => worker.projectTerminal({ execution: { ...execution, turnId: "later-turn" }, + outcome: "completed", endedAt, precedingMessageId: "user-1", fallbackModel: null, narrative: [] })) + .toThrow(/Stale Codex/); + }); + + it("rejects plan turns and post-turn goal receipts without mutating the ordinary message", () => { + expect(() => new CodexParentMessageProjection({ execution, turnKind: "plan" })) + .toThrow(/plan output needs early assistant materialization/); + const worker = projection(); + expect(() => worker.projectMessage({ execution, content: "Goal achieved in 3s.", model: null, + precedingMessageId: "user-1", postTurnGoalReceipt: true })).toThrow(/Post-turn goal receipts/); + expect(worker.projectTerminal({ execution, outcome: "completed", endedAt, + precedingMessageId: "user-1", fallbackModel: null, narrative: [] }).assistant.content).toBe(""); + }); +}); diff --git a/apps/server/src/features/agents/turns/codex-parent-message-projection.ts b/apps/server/src/features/agents/turns/codex-parent-message-projection.ts new file mode 100644 index 000000000..31272bb24 --- /dev/null +++ b/apps/server/src/features/agents/turns/codex-parent-message-projection.ts @@ -0,0 +1,117 @@ +import type { ParentNarrativeRecoveryItem, StoredAttachment, TurnOutcome } from "@mcode/contracts"; + +import type { DataOnlyParentTerminalProjectionInput } from "../canonical/canonical-parent-turn-write.js"; +import type { ExecutionIdentity } from "../execution/execution-mailbox-protocol.js"; +import { AssistantExecutionState } from "./assistant-execution-state.js"; +import { deriveTurnAssistantMessageId } from "./turn-assistant-message-id.js"; + +/** The mailbox identifies one turn; the current message anchor arrives at projection time. */ +export interface CodexParentMessageProjectionInput { + readonly execution: ExecutionIdentity; + readonly turnKind: "ordinary" | "plan"; +} + +/** Public fields attached to a provider message before it reaches the renderer. */ +export interface ProjectedCodexParentMessage { + readonly messageId: string; + readonly model: string | null; + readonly attachments?: readonly StoredAttachment[]; +} + +/** Cloneable terminal data, including the public ID the writer must persist. */ +export interface ProjectedCodexParentTerminal extends DataOnlyParentTerminalProjectionInput { + readonly assistant: DataOnlyParentTerminalProjectionInput["assistant"] & { readonly messageId: string }; + readonly fromProvider: boolean; +} + +/** Keeps the assistant projection for one Codex parent execution off the server loop. */ +export class CodexParentMessageProjection { + private readonly execution: ExecutionIdentity; + private messageId: string | undefined; + private readonly assistant = new AssistantExecutionState(); + + constructor(input: CodexParentMessageProjectionInput) { + if (input.turnKind === "plan") { + throw new Error("Codex plan output needs early assistant materialization and is not supported by this projection"); + } + this.execution = structuredClone(input.execution); + } + + /** Record final response text that can become an assistant body if no provider message arrives. */ + appendFinalResponseText(execution: ExecutionIdentity, delta: string): void { + this.assertExecution(execution); + this.assistant.appendStreamingText(delta); + } + + /** Buffer a generated attachment with the same ID replacement order as the live finalizer. */ + bufferGeneratedAttachment(execution: ExecutionIdentity, attachment: StoredAttachment): void { + this.assertExecution(execution); + this.assistant.bufferAttachments([structuredClone(attachment)]); + } + + /** Buffer a normal provider body and return exactly the public fields of the live message path. */ + projectMessage(input: { + readonly execution: ExecutionIdentity; + readonly content: string; + readonly model: string | null; + readonly precedingMessageId: string; + readonly postTurnGoalReceipt: boolean; + }): ProjectedCodexParentMessage { + this.assertExecution(input.execution); + if (input.postTurnGoalReceipt) { + throw new Error("Post-turn goal receipts need a separate persisted message and are not supported by this projection"); + } + const messageId = this.deriveMessageId(input.precedingMessageId); + const attachments = structuredClone(this.assistant.getBufferedAttachments()); + this.assistant.bufferBody(input.content, input.model, attachments); + this.assistant.resetStreamingText(); + this.messageId = messageId; + return { + messageId, + model: input.model, + ...(attachments.length > 0 ? { attachments } : {}), + }; + } + + /** Produce terminal data for a future single-writer mutation without reading or writing SQLite. */ + projectTerminal(input: { + readonly execution: ExecutionIdentity; + readonly outcome: TurnOutcome; + readonly endedAt: string; + readonly fallbackModel: string | null; + readonly precedingMessageId?: string; + readonly narrative: readonly ParentNarrativeRecoveryItem[]; + }): ProjectedCodexParentTerminal { + this.assertExecution(input.execution); + // A notice can change the last visible row after publication. Keep the + // published ID; only a text-only fallback derives its ID at terminal time. + const messageId = this.messageId ?? this.deriveMessageId(input.precedingMessageId); + const assistant = this.assistant.materializationInput(input.fallbackModel); + return { + threadId: this.execution.threadId, + executionId: this.execution.executionId, + outcome: input.outcome, + endedAt: input.endedAt, + fromProvider: assistant.fromProvider, + assistant: { + messageId, + content: assistant.content, + model: assistant.model, + attachments: structuredClone(assistant.attachments), + }, + narrative: structuredClone(input.narrative), + }; + } + + private assertExecution(execution: ExecutionIdentity): void { + if (execution.threadId !== this.execution.threadId || execution.turnId !== this.execution.turnId + || execution.executionId !== this.execution.executionId) { + throw new Error(`Stale Codex parent message projection command: ${execution.executionId}`); + } + } + + private deriveMessageId(precedingMessageId: string | undefined): string { + if (!precedingMessageId) throw new Error("Codex parent projection needs the current preceding message ID"); + return deriveTurnAssistantMessageId(this.execution.threadId, precedingMessageId); + } +} From e630cee3033209aa011e45d5cb4ad9a69a3a61b2 Mon Sep 17 00:00:00 2001 From: "(CJ) Chukwudi Nwobodo" <142016413+chuks-qua@users.noreply.github.com> Date: Thu, 24 Sep 2026 22:13:58 +0100 Subject: [PATCH 079/136] feat(server): persist bounded live publication receipts --- .../canonical-agent-writer-client.test.ts | 37 +++++++- ...anonical-execution-semantic-writer.test.ts | 85 ++++++++++++++++++ .../canonical-execution-semantic-writer.ts | 88 +++++++++++++++++-- .../execution/execution-worker-handler.ts | 20 ++++- 4 files changed, 221 insertions(+), 9 deletions(-) diff --git a/apps/server/src/features/agents/canonical/__tests__/canonical-agent-writer-client.test.ts b/apps/server/src/features/agents/canonical/__tests__/canonical-agent-writer-client.test.ts index c835efed0..f0676fff5 100644 --- a/apps/server/src/features/agents/canonical/__tests__/canonical-agent-writer-client.test.ts +++ b/apps/server/src/features/agents/canonical/__tests__/canonical-agent-writer-client.test.ts @@ -4,7 +4,7 @@ import * as NodeFSPromises from "node:fs/promises"; import * as NodeOS from "node:os"; import * as NodePath from "node:path"; import { afterEach, beforeEach, describe, expect, it } from "vitest"; -import type { ParentNarrativeRecoveryItem } from "@mcode/contracts"; +import { AgentEventType, type ParentNarrativeRecoveryItem } from "@mcode/contracts"; import { openDatabase } from "../../../../runtime/persistence/sqlite/database.js"; import type { CanonicalAgentEventDraft } from "../canonical-agent-boundary.js"; import { CanonicalAgentWriterClient } from "../canonical-agent-writer-client.js"; @@ -163,6 +163,41 @@ describe("canonical SQLite writer", () => { expect(db.prepare("SELECT COUNT(*) AS count FROM canonical_writer_operation_receipts").get()).toEqual({ count: 1 }); }); + it("recovers one durable live publication identity after a file-backed writer loses its reply", async () => { + const execution = { threadId: THREAD_ID, turnId: TURN_ID, executionId: EXECUTION_ID }; + const lease = { ownerEpoch: 1, workerIndex: 0, workerGeneration: 1, leaseId: "publication-lease" }; + const operation: ExecutionSemanticOperation = { + operationId: "publication-lease:1", execution, lease, ordinal: 1, + mutation: { kind: "begin", providerId: "codex", input: { + thread: { id: THREAD_ID, workspaceId: "writer-workspace", providerId: "codex", createdAt: NOW }, + turnId: TURN_ID, executionId: EXECUTION_ID, permissionMode: "supervised", providerIdentities: [], + userMessage: { kind: "create", messageId: "publication-user", content: "Question", sequence: 1 }, + } }, + livePublication: [{ after: "writer", event: { + type: AgentEventType.TurnStarted, threadId: THREAD_ID, turnExecutionId: EXECUTION_ID, + } }], + }; + let created = 0; + writer = new CanonicalAgentWriterClient(dbPath, () => created++ === 0 + ? workerDroppingReply("semantic-transacted") + : new Worker(new URL("../canonical-agent-writer.worker.ts", import.meta.url), { type: "module" })); + + const receipt = await writer.transactSemantic(operation, () => {}); + expect(created).toBe(2); + expect(receipt).toMatchObject({ kind: "committed", livePublication: [{ + publicationId: "publication-lease:1:0", after: "writer", + event: { type: AgentEventType.TurnStarted, turnExecutionId: EXECUTION_ID }, + }] }); + expect(db.prepare("SELECT COUNT(*) AS count FROM messages WHERE id = ?").get("publication-user")) + .toEqual({ count: 1 }); + expect(db.prepare("SELECT COUNT(*) AS count FROM canonical_writer_operation_receipts WHERE execution_id = ? AND operation_id = ?") + .get(EXECUTION_ID, operation.operationId)).toEqual({ count: 1 }); + + await writer.close(); + writer = new CanonicalAgentWriterClient(dbPath); + expect(await writer.transactSemantic(operation, () => {})).toEqual(receipt); + }); + it("replays durable semantic assistant text after a worker reply is lost", async () => { const execution = { threadId: THREAD_ID, turnId: TURN_ID, executionId: EXECUTION_ID }; const lease = { ownerEpoch: 1, workerIndex: 0, workerGeneration: 1, leaseId: "text-lease" }; diff --git a/apps/server/src/features/agents/canonical/__tests__/canonical-execution-semantic-writer.test.ts b/apps/server/src/features/agents/canonical/__tests__/canonical-execution-semantic-writer.test.ts index 69790807d..c272bf315 100644 --- a/apps/server/src/features/agents/canonical/__tests__/canonical-execution-semantic-writer.test.ts +++ b/apps/server/src/features/agents/canonical/__tests__/canonical-execution-semantic-writer.test.ts @@ -3,6 +3,7 @@ import * as NodeFS from "node:fs"; import * as NodeOS from "node:os"; import * as NodePath from "node:path"; import type { Database } from "bun:sqlite"; +import { AgentEventType } from "@mcode/contracts"; import { afterEach, beforeEach, describe, expect, it } from "vitest"; import { openDatabase } from "../../../../runtime/persistence/sqlite/database.js"; @@ -405,6 +406,90 @@ describe("CanonicalExecutionSemanticWriter through ExecutionWorkerHandler", () = .toEqual({ count: 4 }); }); + it("retains terminal live publication behind a completed finalization and rejects invalid routing", async () => { + const ended = { type: AgentEventType.Ended, threadId: THREAD_ID, + turnExecutionId: EXECUTION_ID, outcome: "completed" as const }; + const invalidStart: ExecutionSemanticOperation = { + ...operation(1, { kind: "begin", providerId: "codex", input: startInput() }), + livePublication: [{ after: "terminal", event: ended }], + }; + expect(await writer.transact(invalidStart)).toEqual({ kind: "conflict", operationId: "lease-1:1" }); + expect(db.prepare("SELECT COUNT(*) AS count FROM canonical_writer_operation_receipts WHERE execution_id = ?") + .get(EXECUTION_ID)).toEqual({ count: 0 }); + expect((await send(1, { kind: "start", providerId: "codex", input: startInput() })).kind).toBe("committed"); + + const staged = new MessageRepo(db).create(THREAD_ID, "assistant", "Answer", 2, undefined, undefined, undefined, "model", true); + const input = { threadId: THREAD_ID, turnId: TURN_ID, executionId: EXECUTION_ID, + providerId: "codex", providerIdentities: [], outcome: "completed" as const, + projection: { message: staged, narrative: [] } }; + const finish: ExecutionSemanticOperation = { + ...operation(2, { kind: "finish", outcome: "completed", input }), + livePublication: [{ after: "terminal", event: ended }], + }; + db.run("CREATE TRIGGER fail_live_finish BEFORE UPDATE OF kind ON canonical_writer_operation_receipts WHEN NEW.kind = 'semantic:finish' BEGIN SELECT RAISE(ABORT, 'finish unavailable'); END"); + await expect(writer.transact(finish)).rejects.toThrow("finish unavailable"); + expect(db.prepare("SELECT terminal_outcome FROM canonical_agent_ingest_checkpoints WHERE execution_id = ?") + .get(EXECUTION_ID)).toEqual({ terminal_outcome: null }); + db.run("DROP TRIGGER fail_live_finish"); + + const receipt = await writer.transact(finish); + expect(receipt).toMatchObject({ kind: "committed", livePublication: [{ + publicationId: "lease-1:2:0", after: "terminal", event: ended, + }] }); + expect(db.prepare("SELECT terminal_outcome FROM canonical_agent_ingest_checkpoints WHERE execution_id = ?") + .get(EXECUTION_ID)).toEqual({ terminal_outcome: "completed" }); + db.close(true); + db = openDatabase({ dbPath: path }); + writer = new CanonicalExecutionSemanticWriter(db, () => {}); + expect(await writer.transact(finish)).toEqual(receipt); + expect(await writer.transact({ ...finish, livePublication: [{ after: "terminal", event: { + ...ended, threadId: "another-thread", + } }] })).toEqual({ kind: "conflict", operationId: "lease-1:2" }); + }); + + it("rejects oversized live intents before writes and oversized receipts before replay publication", async () => { + const started = { type: AgentEventType.TurnStarted, threadId: THREAD_ID, turnExecutionId: EXECUTION_ID }; + const begin: ExecutionSemanticOperation = { + ...operation(1, { kind: "begin", providerId: "codex", input: startInput() }), + livePublication: [{ after: "writer", event: started }], + }; + expect(await writer.transact({ ...begin, livePublication: Array.from({ length: 65 }, () => ({ + after: "writer" as const, event: started, + })) })).toEqual({ kind: "conflict", operationId: "lease-1:1" }); + expect(db.prepare("SELECT COUNT(*) AS count FROM canonical_writer_operation_receipts WHERE execution_id = ?") + .get(EXECUTION_ID)).toEqual({ count: 0 }); + + const beginReceipt = await writer.transact(begin); + expect(beginReceipt.kind).toBe("committed"); + const before = db.prepare("SELECT COUNT(*) AS count FROM canonical_agent_events WHERE execution_id = ?") + .get(EXECUTION_ID); + published.length = 0; + const largeDelta: ExecutionSemanticOperation = { + ...operation(2, { kind: "append-events", phase: "running", nativeCursor: null, events: [event()] }), + livePublication: [{ after: "writer", event: { + type: AgentEventType.TextDelta, threadId: THREAD_ID, turnExecutionId: EXECUTION_ID, + delta: "x".repeat(256 * 1024), + } }], + }; + expect(await writer.transact(largeDelta)).toEqual({ kind: "conflict", operationId: "lease-1:2" }); + expect(db.prepare("SELECT COUNT(*) AS count FROM canonical_agent_events WHERE execution_id = ?") + .get(EXECUTION_ID)).toEqual(before); + expect(published).toEqual([]); + + db.prepare("UPDATE canonical_writer_operation_receipts SET receipt_json = ? WHERE execution_id = ? AND operation_id = ?") + .run("x".repeat(512 * 1024 + 1), EXECUTION_ID, begin.operationId); + await expect(writer.transact(begin)).rejects.toThrow("Live publication receipt exceeds its size limit"); + expect(published).toEqual([]); + + db.prepare("UPDATE canonical_writer_operation_receipts SET receipt_json = ? WHERE execution_id = ? AND operation_id = ?") + .run(JSON.stringify({ ...beginReceipt, publicationVersion: 1, livePublication: [{ + publicationId: "lease-1:1:0", after: "writer", + event: { ...started, threadId: "another-thread" }, + }] }), EXECUTION_ID, begin.operationId); + await expect(writer.transact(begin)).rejects.toThrow("Live publication receipt does not match its operation"); + expect(published).toEqual([]); + }); + it("rejects stale leases and skipped ordinals while checkpointing without a new event", async () => { expect((await send(1, { kind: "start", providerId: "codex", input: startInput() })).kind).toBe("committed"); const eventsBefore = db.prepare("SELECT COUNT(*) AS count FROM canonical_agent_events WHERE execution_id = ?").get(EXECUTION_ID); diff --git a/apps/server/src/features/agents/canonical/canonical-execution-semantic-writer.ts b/apps/server/src/features/agents/canonical/canonical-execution-semantic-writer.ts index e99e7d6c7..efb00277f 100644 --- a/apps/server/src/features/agents/canonical/canonical-execution-semantic-writer.ts +++ b/apps/server/src/features/agents/canonical/canonical-execution-semantic-writer.ts @@ -18,6 +18,8 @@ import type { ExecutionSemanticOperation, ExecutionSemanticWriter, ExecutionWriteReceipt, + ExecutionLivePublicationIntent, + ExecutionLivePublicationReceipt, } from "../execution/execution-worker-handler.js"; import type { CanonicalAgentEventPublisher } from "./canonical-agent-boundary.js"; import { CanonicalAgentBoundary } from "./canonical-agent-boundary.js"; @@ -37,6 +39,9 @@ const PENDING_FINISH_KIND = "semantic:finish-pending"; const PUBLICATION_CHUNK_SIZE = 64; const PUBLICATION_CHUNK_PAGE_SIZE = 16; const MAX_BUFFERED_PUBLICATION_EVENTS = 2_048; +const MAX_LIVE_PUBLICATION_EVENTS = 64; +const MAX_LIVE_PUBLICATION_BYTES = 256 * 1024; +const MAX_LIVE_RECEIPT_BYTES = 512 * 1024; const narrativeDeltaSchema = z.object({ executionId: z.string(), items: z.array(ParentNarrativeRecoveryItemSchema()), @@ -80,6 +85,11 @@ const storedReceiptSchema = z.object({ operationId: z.string(), durableRevision: z.number().int(), publicationVersion: z.literal(1), + livePublication: z.array(z.object({ + publicationId: z.string(), + after: z.enum(["writer", "terminal"]), + event: AgentEventSchema(), + })).min(1).max(MAX_LIVE_PUBLICATION_EVENTS).optional(), providerCommit: z.object({ outcome: z.enum(["committed", "duplicate", "conflict", "terminal-outcome-confirmed", "ingest-overflow"]), conversationRevision: z.number().int(), @@ -264,6 +274,7 @@ export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter return conflict(operation); } if (mutation.providerId !== mutation.input.thread.providerId + || !validPublicationProvider(operation, mutation.providerId) || mutation.input.thread.id !== operation.execution.threadId || mutation.input.turnId !== operation.execution.turnId || mutation.input.executionId !== operation.execution.executionId) return conflict(operation); @@ -299,6 +310,7 @@ export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter || event.routing.executionId !== operation.execution.executionId)) return conflict(operation); return this.withBufferedPublication(() => this.db.transaction(() => { const head = this.requireNextHead(operation); + if (!validPublicationProvider(operation, head.providerId)) throw new SemanticConflict(); const result = this.turns.append({ ...operation.execution, phase: mutation.phase, @@ -365,9 +377,7 @@ export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter private async finish(operation: ExecutionSemanticOperation, hash: string): Promise { const mutation = operation.mutation; if (mutation.kind !== "finish" || !validFinish(operation, mutation)) return conflict(operation); - const head = this.loadHead(operation.execution.executionId); - if (!head || !nextHead(head, operation) || head.providerId !== mutation.input.providerId) return conflict(operation); - if (!this.validStagedFinish(operation, head)) return conflict(operation); + if (!this.validFinishHead(operation, mutation.input.providerId)) return conflict(operation); this.reserveFinish(operation, hash); this.publishStoredEvents(operation.execution.executionId, hash); const result = await this.turns.finish(mutation.input, (batch) => { @@ -466,6 +476,12 @@ export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter return head.providerOutcome === mutation.outcome && this.hasStagedTerminalPredecessor(operation); } + private validFinishHead(operation: ExecutionSemanticOperation, providerId: string): boolean { + const head = this.loadHead(operation.execution.executionId); + return Boolean(head && nextHead(head, operation) && head.providerId === providerId + && validPublicationProvider(operation, head.providerId) && this.validStagedFinish(operation, head)); + } + private requireNextHead(operation: ExecutionSemanticOperation): SemanticHead { const head = this.loadHead(operation.execution.executionId); if (!head || !nextHead(head, operation)) throw new SemanticConflict(); @@ -520,6 +536,9 @@ export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter private storeReceipt(operation: ExecutionSemanticOperation, hash: string, receipt: ExecutionWriteReceipt): void { const receiptJson = JSON.stringify({ ...receipt, publicationVersion: 1 }); + if (operation.livePublication && Buffer.byteLength(receiptJson, "utf8") > MAX_LIVE_RECEIPT_BYTES) { + throw new SemanticConflict(); + } if (operation.mutation.kind === "finish") { const updated = this.commitPendingFinish.run( "semantic:finish", receiptJson, operation.execution.executionId, operation.operationId, PENDING_FINISH_KIND, hash, @@ -544,10 +563,16 @@ export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter private readReceipt(operation: ExecutionSemanticOperation, hash: string, stored: StoredOperation): ExecutionWriteReceipt { if (stored.kind !== `semantic:${operation.mutation.kind}` || stored.input_hash !== hash) return conflict(operation); + if (operation.livePublication && Buffer.byteLength(stored.receipt_json, "utf8") > MAX_LIVE_RECEIPT_BYTES) { + throw new Error("Live publication receipt exceeds its size limit"); + } const receipt = storedReceiptSchema.parse(JSON.parse(stored.receipt_json)); + if (fingerprint(receipt.livePublication ?? null) !== fingerprint(livePublicationFor(operation) ?? null)) { + throw new Error("Live publication receipt does not match its operation"); + } return receipt.operationId === operation.operationId && Number.isSafeInteger(receipt.durableRevision) ? committed(operation, receipt.durableRevision, receipt.providerCommit, receipt.providerEvents, - receipt.assistantTextCheckpoint) : conflict(operation); + receipt.assistantTextCheckpoint, receipt.livePublication) : conflict(operation); } private publishStoredEvents(executionId: string, hash: string): void { @@ -607,7 +632,51 @@ function validOperation(operation: ExecutionSemanticOperation): boolean { && operation.operationId !== HEAD_ID && operation.operationId.length <= 256 && Number.isSafeInteger(operation.ordinal) && operation.ordinal > 0 && validIdentity(operation.execution) && validLease(operation.lease) - && validSemanticMutationInput(operation); + && validSemanticMutationInput(operation) + && validLivePublication(operation); +} + +function validLivePublication(operation: ExecutionSemanticOperation): boolean { + const publication = operation.livePublication; + if (publication === undefined) return true; + if (!validLivePublicationShape(publication, operation.mutation.kind)) return false; + const barrier = operation.mutation.kind === "finish" ? "terminal" : "writer"; + let bytes = 0; + for (const intent of publication) { + if (!validLivePublicationIntent(intent, operation, barrier)) return false; + bytes += Buffer.byteLength(JSON.stringify(intent), "utf8"); + if (bytes > MAX_LIVE_PUBLICATION_BYTES) return false; + } + return true; +} + +function validLivePublicationShape( + publication: readonly ExecutionLivePublicationIntent[], + mutationKind: ExecutionSemanticOperation["mutation"]["kind"], +): boolean { + return Array.isArray(publication) && publication.length > 0 && publication.length <= MAX_LIVE_PUBLICATION_EVENTS + && (mutationKind === "begin" || mutationKind === "append-events" || mutationKind === "finish"); +} + +function validLivePublicationIntent( + intent: NonNullable[number], + operation: ExecutionSemanticOperation, + barrier: "writer" | "terminal", +): boolean { + return intent.after === barrier && AgentEventSchema().safeParse(intent.event).success + && intent.event.threadId === operation.execution.threadId + && intent.event.turnExecutionId === operation.execution.executionId + && (operation.mutation.kind !== "begin" || intent.event.type === "turnStarted") + && (intent.event.type !== "turnStarted" || operation.mutation.kind === "begin") + && (barrier === "terminal" || !isTerminalLiveEvent(intent.event.type)); +} + +function isTerminalLiveEvent(type: ExecutionLivePublicationIntent["event"]["type"]): boolean { + return type === "turnComplete" || type === "ended"; +} + +function validPublicationProvider(operation: ExecutionSemanticOperation, providerId: string): boolean { + return operation.livePublication === undefined || providerId === "codex"; } function validSemanticMutationInput(operation: ExecutionSemanticOperation): boolean { @@ -716,15 +785,24 @@ function committed( providerCommit?: ExecutionProviderCommitReceipt, providerEvents?: readonly ProjectedCommittedProviderEvent[], assistantTextCheckpoint?: ParentAssistantTextCheckpointResult, + livePublication: readonly ExecutionLivePublicationReceipt[] | undefined = livePublicationFor(operation), ): Extract { return { kind: "committed", operationId: operation.operationId, durableRevision, ...(providerCommit ? { providerCommit } : {}), ...(providerEvents ? { providerEvents } : {}), ...(assistantTextCheckpoint ? { assistantTextCheckpoint } : {}), + ...(livePublication ? { livePublication } : {}), }; } +function livePublicationFor(operation: ExecutionSemanticOperation): readonly ExecutionLivePublicationReceipt[] | undefined { + return operation.livePublication?.map((intent, index) => ({ + publicationId: `${operation.operationId}:${index}`, after: intent.after, + event: AgentEventSchema().parse(intent.event), + })); +} + function conflict(operation: ExecutionSemanticOperation): Extract { return { kind: "conflict", operationId: operation.operationId }; } diff --git a/apps/server/src/features/agents/execution/execution-worker-handler.ts b/apps/server/src/features/agents/execution/execution-worker-handler.ts index aa9fee940..3440ee7db 100644 --- a/apps/server/src/features/agents/execution/execution-worker-handler.ts +++ b/apps/server/src/features/agents/execution/execution-worker-handler.ts @@ -1,4 +1,4 @@ -import type { TurnOutcome } from "@mcode/contracts"; +import type { AgentEvent, TurnOutcome } from "@mcode/contracts"; import type { ProviderEventDraft } from "@mcode/providers"; import type { @@ -41,6 +41,8 @@ export interface ExecutionSemanticOperation { readonly execution: ExecutionIdentity; readonly lease: ExecutionLease; readonly ordinal: number; + /** Live Codex events to release with this operation's durable receipt. */ + readonly livePublication?: readonly ExecutionLivePublicationIntent[]; readonly mutation: | { readonly kind: "begin"; readonly providerId: string; readonly input: DataOnlyParentTurnStartInput } | { readonly kind: "resume"; readonly providerId: string; readonly checkpointId: string } @@ -56,6 +58,17 @@ export interface ExecutionSemanticOperation { | { readonly kind: "finish"; readonly outcome: TurnOutcome; readonly input: DataOnlyParentTurnFinishInput }; } +/** A live event has one durability barrier and stays inert until its semantic effects commit. */ +export interface ExecutionLivePublicationIntent { + readonly event: AgentEvent; + readonly after: "writer" | "terminal"; +} + +/** Stable identity lets the transport deduplicate a replayed publication receipt. */ +export interface ExecutionLivePublicationReceipt extends ExecutionLivePublicationIntent { + readonly publicationId: string; +} + /** Provider-facing receipt values retained with the execution operation. */ export type ExecutionProviderCommitReceipt = Pick< CanonicalAgentCommitResult, @@ -73,7 +86,7 @@ export type ProjectedCommittedProviderEvent = Omit< /** A writer reply is valid only after the semantic operation commits durably. */ export type ExecutionWriteReceipt = - | { readonly kind: "committed"; readonly operationId: string; readonly durableRevision: number; readonly providerCommit?: ExecutionProviderCommitReceipt; readonly providerEvents?: readonly ProjectedCommittedProviderEvent[]; readonly assistantTextCheckpoint?: ParentAssistantTextCheckpointResult } + | { readonly kind: "committed"; readonly operationId: string; readonly durableRevision: number; readonly providerCommit?: ExecutionProviderCommitReceipt; readonly providerEvents?: readonly ProjectedCommittedProviderEvent[]; readonly assistantTextCheckpoint?: ParentAssistantTextCheckpointResult; readonly livePublication?: readonly ExecutionLivePublicationReceipt[] } | { readonly kind: "conflict"; readonly operationId: string; readonly recoveryState?: "not-started" | "already-terminal" }; /** @@ -87,7 +100,7 @@ export interface ExecutionSemanticWriter { /** A command result that never calls an uncommitted mutation successful. */ export type ExecutionWorkerResult = - | { readonly kind: "committed"; readonly operationId: string; readonly durableRevision: number; readonly providerCommit?: ExecutionProviderCommitReceipt; readonly providerEvents?: readonly ProjectedCommittedProviderEvent[]; readonly assistantTextCheckpoint?: ParentAssistantTextCheckpointResult } + | { readonly kind: "committed"; readonly operationId: string; readonly durableRevision: number; readonly providerCommit?: ExecutionProviderCommitReceipt; readonly providerEvents?: readonly ProjectedCommittedProviderEvent[]; readonly assistantTextCheckpoint?: ParentAssistantTextCheckpointResult; readonly livePublication?: readonly ExecutionLivePublicationReceipt[] } | { readonly kind: "released" } | { readonly kind: "rejected"; readonly reason: "no-execution" | "stale-execution" | "out-of-order" | "invalid-transition" | "invalid-event-routing" | "invalid-text-routing" | "invalid-narrative-routing" | "invalid-stop-watermark" | "writer-conflict" }; @@ -356,6 +369,7 @@ function committedResult(receipt: Extract Date: Thu, 24 Sep 2026 22:13:56 +0100 Subject: [PATCH 080/136] feat(server): cover Codex live reducer event variants --- .../codex-live-event-reducer.test.ts | 150 +++++++++++++++++- .../execution/codex-live-event-reducer.ts | 138 ++++++++++++++-- 2 files changed, 275 insertions(+), 13 deletions(-) diff --git a/apps/server/src/features/agents/execution/__tests__/codex-live-event-reducer.test.ts b/apps/server/src/features/agents/execution/__tests__/codex-live-event-reducer.test.ts index 7b380eaa2..ee0a6feb9 100644 --- a/apps/server/src/features/agents/execution/__tests__/codex-live-event-reducer.test.ts +++ b/apps/server/src/features/agents/execution/__tests__/codex-live-event-reducer.test.ts @@ -10,7 +10,11 @@ const execution = { }; function event(type: AgentEvent["type"], fields: Record = {}): AgentEvent { - return AgentEventSchema().parse({ type, threadId: execution.threadId, ...fields }); + return AgentEventSchema().parse({ type, threadId: execution.threadId, turnExecutionId: execution.executionId, ...fields }); +} + +function boundByProvider(mapped: AgentEvent): AgentEvent { + return { ...mapped, turnExecutionId: execution.executionId }; } describe("CodexLiveEventReducer", () => { @@ -29,7 +33,7 @@ describe("CodexLiveEventReducer", () => { ]; for (const notification of notifications) { for (const runtimeEvent of mapper.mapNotification({ jsonrpc: "2.0", ...notification })) { - reductions.push(reducer.reduce(runtimeEvent.event)); + reductions.push(reducer.reduce(boundByProvider(runtimeEvent.event))); } } @@ -89,14 +93,152 @@ describe("CodexLiveEventReducer", () => { expect(completed.publication.after).toBe("terminal"); }); + it("rejects unbound session events and reduces explicitly bound notices and cursors", () => { + const mapper = new CodexEventMapper(execution.threadId); + const reducer = new CodexLiveEventReducer(execution); + const session = reducer.reduce(mapper.sessionStartedEvent().event); + expect(session).toEqual({ + kind: "unsupported", eventType: "system", + reason: "event without execution identity needs the provider event routing owner", + }); + const boundSession = reducer.reduce(event("system", { subtype: "provider.session.started" })); + expect(boundSession).toMatchObject({ kind: "reduced", writer: [{ kind: "notice-session" }] }); + expect(reducer.reduce(event("turnStarted")).kind).toBe("reduced"); + + const notice = mapper.mapNotification({ method: "unknown/new-method", params: {} })[0]?.event; + expect(notice?.type).toBe("system"); + if (!notice) return; + expect(reducer.reduce(notice)).toEqual({ + kind: "unsupported", eventType: "system", + reason: "event without execution identity needs the provider event routing owner", + }); + const reduced = reducer.reduce(boundByProvider(notice)); + expect(reduced.kind).toBe("reduced"); + if (reduced.kind !== "reduced") return; + expect(reduced.writer.map(({ kind }) => kind)).toEqual(["system-notice"]); + expect(reduced.publication.after).toBe("writer"); + expect(structuredClone(reduced)).toEqual(reduced); + + const cursor = reducer.reduce(event("system", { subtype: "sdk_session_id:native-1" })); + expect(cursor.kind).toBe("reduced"); + if (cursor.kind !== "reduced") return; + expect(cursor.writer.map(({ kind }) => kind)).toEqual(["session-cursor"]); + }); + + it("records mapped context outside compaction and leaves an unsupported terminal unchanged", () => { + const mapper = new CodexEventMapper(execution.threadId, "native-main"); + const reducer = new CodexLiveEventReducer(execution); + expect(reducer.reduce(event("turnStarted")).kind).toBe("reduced"); + mapper.prepareForTurn(); + mapper.mapNotification({ method: "turn/started", params: { threadId: "native-main", turn: { id: "native-turn" } } }); + const usage = { totalTokens: 120, inputTokens: 100, cachedInputTokens: 40, outputTokens: 20, reasoningOutputTokens: 5 }; + const mapped = mapper.mapNotification({ method: "thread/tokenUsage/updated", params: { + threadId: "native-main", turnId: "native-turn", + tokenUsage: { total: usage, last: usage, modelContextWindow: 200_000 }, + } })[0]?.event; + expect(mapped?.type).toBe("contextEstimate"); + if (!mapped) return; + expect(reducer.reduce(mapped)).toEqual({ + kind: "unsupported", eventType: "contextEstimate", + reason: "event without execution identity needs the provider event routing owner", + }); + const first = reducer.reduce(boundByProvider(mapped)); + expect(first.kind).toBe("reduced"); + if (first.kind !== "reduced") return; + expect(first.writer).toEqual([{ kind: "context-usage", tokensIn: 100, contextWindow: 200_000 }]); + + expect(reducer.reduce(event("compacting", { active: true }))).toMatchObject({ + kind: "reduced", writer: [{ kind: "compaction-started" }], + }); + const during = reducer.reduce(boundByProvider(mapped)); + expect(during).toMatchObject({ kind: "reduced", writer: [] }); + const completion = event("turnComplete", { reason: "end_turn", costUsd: null, tokensIn: 100, tokensOut: 20 }); + expect(reducer.reduce(completion)).toEqual({ + kind: "unsupported", eventType: "turnComplete", + reason: "turn completion during compaction needs the compaction terminal owner", + }); + expect(reducer.reduce(event("compactSummary", { summary: "Shortened context" }))).toMatchObject({ + kind: "reduced", writer: [{ kind: "compaction-summary", summary: "Shortened context" }], + }); + expect(reducer.reduce(event("compacting", { active: true }))).toMatchObject({ + kind: "reduced", writer: [{ kind: "compaction-started" }], + }); + expect(reducer.reduce(event("compacting", { active: false }))).toMatchObject({ + kind: "reduced", writer: [{ kind: "compaction-divider" }], + }); + expect(reducer.reduce(boundByProvider(mapped))).toMatchObject({ + kind: "reduced", writer: [{ kind: "context-usage", tokensIn: 100, contextWindow: 200_000 }], + }); + expect(reducer.reduce(completion)).toMatchObject({ + kind: "reduced", writer: [{ kind: "context-usage", tokensIn: 100 }, { kind: "terminal-projection" }, { kind: "feature-event" }], + }); + }); + + it("publishes mapper retry and status events and projects a failed turn", () => { + const mapper = new CodexEventMapper(execution.threadId, "native-main"); + const reducer = new CodexLiveEventReducer(execution); + expect(reducer.reduce(event("turnStarted")).kind).toBe("reduced"); + mapper.prepareForTurn(); + mapper.mapNotification({ method: "turn/started", params: { threadId: "native-main", turn: { id: "native-turn" } } }); + const retry = mapper.mapNotification({ method: "error", params: { + threadId: "native-main", error: { message: "temporary" }, willRetry: true, + } })[0]?.event; + expect(retry?.type).toBe("apiRetry"); + if (!retry) return; + expect(reducer.reduce(boundByProvider(retry))).toMatchObject({ kind: "reduced", writer: [], publication: { after: "writer" } }); + const quota = AgentEventSchema().parse({ type: "quotaUpdate", threadId: execution.threadId, providerId: "codex", categories: [] }); + expect(reducer.reduce(quota)).toMatchObject({ kind: "unsupported", eventType: "quotaUpdate" }); + expect(reducer.reduce(boundByProvider(quota))).toMatchObject({ kind: "reduced", writer: [] }); + const status = mapper.mapNotification({ method: "mcpServer/startupStatus/updated", params: { + threadId: "native-main", name: "search", status: "failed", error: "offline", + } })[0]?.event; + expect(status?.type).toBe("mcpServerStartupStatus"); + if (!status) return; + expect(reducer.reduce(boundByProvider(status))).toMatchObject({ kind: "reduced", writer: [] }); + + const failure = mapper.mapNotification({ method: "turn/completed", params: { + threadId: "native-main", turn: { id: "native-turn", status: "failed", items: [], error: { message: "failed" } }, + } }).find(({ event: mapped }) => mapped.type === "error")?.event; + expect(failure?.type).toBe("error"); + if (!failure) return; + const terminal = reducer.reduce(boundByProvider(failure)); + expect(terminal).toMatchObject({ + kind: "reduced", writer: [{ kind: "turn-error", error: "failed" }, { kind: "terminal-projection", source: "error", outcome: "errored" }], + publication: { after: "terminal" }, + }); + expect(structuredClone(terminal)).toEqual(terminal); + }); + + it("keeps post-turn goal and status publications while rejecting unowned goal receipts", () => { + const mapper = new CodexEventMapper(execution.threadId, "native-main"); + const reducer = new CodexLiveEventReducer(execution); + expect(reducer.reduce(event("turnStarted")).kind).toBe("reduced"); + expect(reducer.reduce(event("turnComplete", { reason: "end_turn", costUsd: null, tokensIn: 0, tokensOut: 0 })).kind).toBe("reduced"); + const goalEvents = mapper.mapNotification({ method: "thread/goal/updated", params: { + threadId: "native-main", turnId: "native-turn", + goal: { threadId: "native-main", objective: "Ship", status: "complete", tokenBudget: null, tokensUsed: 5, + timeUsedSeconds: 2, createdAt: 1, updatedAt: 2 }, + } }).map(({ event: mapped }) => mapped); + expect(goalEvents.map(({ type }) => type)).toEqual(["goalUpdated", "message", "goalCleared"]); + const [goalUpdated, receipt, goalCleared] = goalEvents; + if (!goalUpdated || !receipt || !goalCleared) return; + expect(reducer.reduce(goalUpdated)).toMatchObject({ kind: "unsupported", eventType: "goalUpdated" }); + expect(reducer.reduce(boundByProvider(goalUpdated))).toMatchObject({ kind: "reduced", writer: [], publication: { after: "terminal" } }); + expect(reducer.reduce(boundByProvider(receipt))).toEqual({ + kind: "unsupported", eventType: "message", reason: "post-turn goal receipt needs the goal message owner", + }); + expect(reducer.reduce(boundByProvider(goalCleared))).toMatchObject({ kind: "reduced", writer: [], publication: { after: "terminal" } }); + expect(reducer.reduce(event("ended", { turnExecutionId: execution.executionId }))).toMatchObject({ kind: "reduced" }); + }); + it("rejects another execution and unowned events without changing the active execution", () => { const reducer = new CodexLiveEventReducer(execution); expect(reducer.reduce(event("turnStarted")).kind).toBe("reduced"); expect(reducer.reduce(event("textDelta", { delta: "wrong", turnExecutionId: "22222222-2222-4222-8222-222222222222" }))).toEqual({ kind: "unsupported", eventType: "textDelta", reason: "different execution", }); - expect(reducer.reduce(event("goalCleared", { reason: "cleared" }))).toEqual({ - kind: "unsupported", eventType: "goalCleared", reason: "event needs a separate feature owner", + expect(reducer.reduce(event("modelFallback", { requestedModel: "a", actualModel: "b" }))).toEqual({ + kind: "unsupported", eventType: "modelFallback", reason: "model fallback needs the model selection owner", }); const final = reducer.reduce(event("textDelta", { delta: "right", isFinalResponse: true })); expect(final.kind).toBe("reduced"); diff --git a/apps/server/src/features/agents/execution/codex-live-event-reducer.ts b/apps/server/src/features/agents/execution/codex-live-event-reducer.ts index a5830d72b..82fac5c11 100644 --- a/apps/server/src/features/agents/execution/codex-live-event-reducer.ts +++ b/apps/server/src/features/agents/execution/codex-live-event-reducer.ts @@ -7,6 +7,46 @@ type ToolUseEvent = Extract; type ToolResultEvent = Extract; type MessageEvent = Extract; type TextDeltaEvent = Extract; +type ContextEvent = Extract; +type SystemEvent = Extract; + +const BEFORE_START_EVENTS = new Set([ + "system", "quotaUpdate", "goalUpdated", "goalCleared", "mcpServerStartupStatus", +]); +const AFTER_COMPLETION_EVENTS = new Set([ + "ended", "hookStarted", "hookCompleted", ...BEFORE_START_EVENTS, +]); + +// A new contract variant must receive an owner decision before the reducer accepts it. +const UNSUPPORTED_FEATURE_REASON = { + turnStarted: null, + message: null, + generatedAttachment: null, + toolUse: null, + toolResult: null, + turnComplete: null, + error: null, + ended: null, + system: null, + compacting: null, + compactSummary: null, + modelFallback: "model fallback needs the model selection owner", + textDelta: null, + toolInputDelta: "incremental tool event needs the tool lifecycle owner", + toolProgress: "incremental tool event needs the tool lifecycle owner", + contextEstimate: null, + quotaUpdate: null, + providerUnavailable: "provider availability needs the provider dispatch owner", + rateLimited: null, + apiRetry: null, + hookStarted: null, + hookProgress: "hook output needs the hook lifecycle owner", + hookCompleted: null, + assistantMessageBoundary: null, + goalUpdated: null, + goalCleared: null, + mcpServerStartupStatus: null, +} satisfies Record; /** Data to commit for one provider event before releasing its publication. */ export type CodexLiveWriterIntent = @@ -23,7 +63,15 @@ export type CodexLiveWriterIntent = | { readonly kind: "narrative-recovery"; readonly items: ParentNarrativeRecoveryItem[] } | { readonly kind: "narrative-effect"; readonly effect: NarrativeTurnStateEffect } | { readonly kind: "feature-event"; readonly feature: "plan-text" | "assistant-message" | "task-tool" | "goal-refresh"; readonly event: AgentEvent } - | { readonly kind: "terminal-projection"; readonly source: "turnComplete" | "ended"; readonly outcome: "completed" | "errored" | "interrupted"; readonly assistant: AssistantMaterializationInput; readonly narrative: ParentNarrativeRecoveryItem[] } + | { readonly kind: "context-usage"; readonly tokensIn: number; readonly contextWindow?: number } + | { readonly kind: "compaction-started" } + | { readonly kind: "compaction-divider" } + | { readonly kind: "compaction-summary"; readonly summary: string } + | { readonly kind: "notice-session"; readonly event: SystemEvent } + | { readonly kind: "system-notice"; readonly event: SystemEvent } + | { readonly kind: "session-cursor"; readonly event: SystemEvent } + | { readonly kind: "turn-error"; readonly error: string } + | { readonly kind: "terminal-projection"; readonly source: "turnComplete" | "error" | "ended"; readonly outcome: "completed" | "errored" | "interrupted"; readonly assistant: AssistantMaterializationInput; readonly narrative: ParentNarrativeRecoveryItem[] } | { readonly kind: "turn-ended" }; /** A publication is released only after the writer accepts every intent for the event. */ @@ -39,6 +87,8 @@ export type CodexLiveReduction = /** * Reduces one Codex execution's live AgentEvents without database or transport calls. + * The caller must supply an exact turnExecutionId from provider turn binding; + * session-level events without that proof belong to a separate owner. * The caller must discard this instance if its writer operation fails, because * the next event may only observe state whose preceding intents were committed. */ @@ -48,23 +98,25 @@ export class CodexLiveEventReducer { private phase: "awaiting-start" | "active" | "completed" | "ended" = "awaiting-start"; private unknownText = ""; private knownFinalText = false; + private compacting = false; constructor(readonly execution: ExecutionIdentity) { this.narrative = new NarrativeTurnState(execution); } reduce(input: AgentEvent): CodexLiveReduction { - const rejection = this.identityRejection(input) ?? this.phaseRejection(input) ?? this.textRejection(input); + const rejection = this.identityRejection(input) ?? UNSUPPORTED_FEATURE_REASON[input.type] + ?? this.phaseRejection(input) ?? this.textRejection(input); if (rejection) return this.unsupported(input, rejection); let event: AgentEvent; try { - event = structuredClone({ ...input, turnExecutionId: this.execution.executionId }); + event = structuredClone(input); } catch { return this.unsupported(input, "event is not cloneable"); } const writer = this.apply(event); - if (!writer) return this.unsupported(input, "event needs a separate feature owner"); + if (!writer) return this.unsupported(input, "reducer dispatch owner has no handler for this event"); for (const effect of this.narrative.takeEffects()) writer.push({ kind: "narrative-effect", effect }); return structuredClone({ kind: "reduced", @@ -79,20 +131,28 @@ export class CodexLiveEventReducer { private identityRejection(event: AgentEvent): string | undefined { if (event.threadId !== this.execution.threadId) return "different thread"; - if (event.turnExecutionId && event.turnExecutionId !== this.execution.executionId) return "different execution"; + if (!event.turnExecutionId) return "event without execution identity needs the provider event routing owner"; + if (event.turnExecutionId !== this.execution.executionId) return "different execution"; return undefined; } private phaseRejection(event: AgentEvent): string | undefined { if (this.phase === "ended") return "execution already ended"; - if (this.phase === "awaiting-start" && event.type !== "turnStarted") return "turn has not started"; - if (this.phase !== "awaiting-start" && event.type === "turnStarted") return "turn already started"; - if (this.phase === "completed" && event.type !== "ended" && event.type !== "hookStarted" && event.type !== "hookCompleted") { - return "event after turn completion requires another owner"; + if (this.phase === "awaiting-start" && event.type !== "turnStarted" && !BEFORE_START_EVENTS.has(event.type)) { + return "turn has not started"; } + if (this.phase !== "awaiting-start" && event.type === "turnStarted") return "turn already started"; + if (this.phase === "completed") return this.completedPhaseRejection(event); + if (this.compacting && event.type === "turnComplete") return "turn completion during compaction needs the compaction terminal owner"; return undefined; } + private completedPhaseRejection(event: AgentEvent): string | undefined { + if (AFTER_COMPLETION_EVENTS.has(event.type)) return undefined; + return event.type === "message" ? "post-turn goal receipt needs the goal message owner" + : "post-terminal event needs the terminal lifecycle owner"; + } + private textRejection(event: AgentEvent): string | undefined { switch (event.type) { case "assistantMessageBoundary": @@ -101,6 +161,7 @@ export class CodexLiveEventReducer { case "textDelta": return this.deltaRejection(event); case "turnComplete": case "ended": + case "error": return this.unknownText ? "assistant text still lacks a boundary" : undefined; default: return undefined; } @@ -145,7 +206,25 @@ export class CodexLiveEventReducer { switch (event.type) { case "turnStarted": return this.start(event); case "turnComplete": return this.turnComplete(event); + case "error": return this.error(event); case "ended": return this.ended(event); + case "contextEstimate": return this.contextEstimate(event); + case "compacting": return this.compactingEvent(event); + case "compactSummary": return this.compactSummary(event); + default: return this.applyStatus(event); + } + } + + private applyStatus(event: AgentEvent): CodexLiveWriterIntent[] | undefined { + switch (event.type) { + case "system": return this.system(event); + // These status events have no server projection beyond their canonical receipt. + case "apiRetry": + case "rateLimited": + case "quotaUpdate": + case "goalUpdated": + case "goalCleared": + case "mcpServerStartupStatus": return []; default: return undefined; } } @@ -270,12 +349,53 @@ export class CodexLiveEventReducer { private turnComplete(event: Extract): CodexLiveWriterIntent[] { this.phase = "completed"; return [ + ...this.contextUsage(event), { kind: "terminal-projection", source: "turnComplete", outcome: "completed", assistant: this.assistant.materializationInput(null), narrative: this.narrative.recoverySnapshot(event.threadId) }, { kind: "feature-event", feature: "goal-refresh", event }, ]; } + private error(event: Extract): CodexLiveWriterIntent[] { + this.phase = "completed"; + return [ + { kind: "turn-error", error: event.error }, + { kind: "terminal-projection", source: "error", outcome: "errored", + assistant: this.assistant.materializationInput(null), narrative: this.narrative.recoverySnapshot(event.threadId) }, + ]; + } + + private contextEstimate(event: Extract): CodexLiveWriterIntent[] { + return event.totalProcessedTokens === undefined ? [] : this.contextUsage(event); + } + + private contextUsage(event: ContextEvent): CodexLiveWriterIntent[] { + return event.tokensIn > 0 && !this.compacting + ? [{ kind: "context-usage", tokensIn: event.tokensIn, + ...(event.contextWindow !== undefined ? { contextWindow: event.contextWindow } : {}) }] + : []; + } + + private compactingEvent(event: Extract): CodexLiveWriterIntent[] { + this.compacting = event.active; + return [{ kind: event.active ? "compaction-started" : "compaction-divider" }]; + } + + private compactSummary(event: Extract): CodexLiveWriterIntent[] { + this.compacting = false; + return [{ kind: "compaction-summary", summary: event.summary }]; + } + + private system(event: SystemEvent): CodexLiveWriterIntent[] { + const writer: CodexLiveWriterIntent[] = []; + if (event.subtype === "provider.session.started") writer.push({ kind: "notice-session", event }); + if (event.subtype.startsWith("provider.notice.") && event.message) writer.push({ kind: "system-notice", event }); + if (event.subtype.startsWith("sdk_session_id:") || event.subtype === "sdk_session_invalidated") { + writer.push({ kind: "session-cursor", event }); + } + return writer; + } + private ended(event: Extract): CodexLiveWriterIntent[] { const writer: CodexLiveWriterIntent[] = []; if (event.outcome && this.phase !== "completed") { From f450b6de5df14eeadb81a2d20129dc59533e4286 Mon Sep 17 00:00:00 2001 From: "(CJ) Chukwudi Nwobodo" <142016413+chuks-qua@users.noreply.github.com> Date: Thu, 24 Sep 2026 22:17:20 +0100 Subject: [PATCH 081/136] fix(server): bind staged assistant identity to terminal finish --- ...anonical-execution-semantic-writer.test.ts | 29 +++++++++++++++++++ .../canonical-execution-semantic-writer.ts | 10 +++++-- 2 files changed, 37 insertions(+), 2 deletions(-) diff --git a/apps/server/src/features/agents/canonical/__tests__/canonical-execution-semantic-writer.test.ts b/apps/server/src/features/agents/canonical/__tests__/canonical-execution-semantic-writer.test.ts index c272bf315..d27c13101 100644 --- a/apps/server/src/features/agents/canonical/__tests__/canonical-execution-semantic-writer.test.ts +++ b/apps/server/src/features/agents/canonical/__tests__/canonical-execution-semantic-writer.test.ts @@ -212,6 +212,35 @@ describe("CanonicalExecutionSemanticWriter through ExecutionWorkerHandler", () = expect((await writer.transact(finish)).kind).toBe("committed"); }); + it("publishes the assigned live assistant ID only when finish names the staged ID", async () => { + const assignedId = "a".repeat(64); + expect((await send(1, { kind: "start", providerId: "codex", input: startInput() })).kind).toBe("committed"); + expect((await send(2, { kind: "provider-outcome", outcome: "completed" })).kind).toBe("committed"); + expect((await send(3, { kind: "stage-terminal", input: { + threadId: THREAD_ID, executionId: EXECUTION_ID, outcome: "completed", endedAt: NOW, + assistant: { content: "Live answer", model: null, attachments: [], messageId: assignedId }, narrative: [], + } })).kind).toBe("committed"); + + db.close(true); + db = openDatabase({ dbPath: path }); + writer = new CanonicalExecutionSemanticWriter(db, () => {}); + const finishInput = { + threadId: THREAD_ID, turnId: TURN_ID, executionId: EXECUTION_ID, + providerId: "codex" as const, providerIdentities: [], outcome: "completed" as const, + projection: { kind: "writer-staged" as const, messageId: assignedId }, + }; + expect(await writer.transact(operation(4, { kind: "finish", outcome: "completed", + input: { ...finishInput, projection: { kind: "writer-staged" } } }))) + .toEqual({ kind: "conflict", operationId: "lease-1:4" }); + expect(await writer.transact(operation(4, { kind: "finish", outcome: "completed", + input: { ...finishInput, projection: { kind: "writer-staged", messageId: "b".repeat(64) } } }))) + .toEqual({ kind: "conflict", operationId: "lease-1:4" }); + expect((await writer.transact(operation(4, { kind: "finish", outcome: "completed", input: finishInput }))).kind) + .toBe("committed"); + expect(new MessageRepo(db).findByIdInThread(THREAD_ID, assignedId)) + .toMatchObject({ id: assignedId, content: "Live answer", is_internal: false }); + }); + it("fences assistant-text routing, lease, ordinal, input size, and conflicting replay", async () => { expect((await send(1, { kind: "start", providerId: "codex", input: startInput() })).kind).toBe("committed"); const inputs = [{ ...execution, sequence: 1, text: "Saved text" }]; diff --git a/apps/server/src/features/agents/canonical/canonical-execution-semantic-writer.ts b/apps/server/src/features/agents/canonical/canonical-execution-semantic-writer.ts index efb00277f..9d1f3f520 100644 --- a/apps/server/src/features/agents/canonical/canonical-execution-semantic-writer.ts +++ b/apps/server/src/features/agents/canonical/canonical-execution-semantic-writer.ts @@ -79,6 +79,7 @@ const storedHeadSchema = z.object({ stopRequestId: z.string().nullable(), stopWatermark: z.number().int().nullable(), providerOutcome: TurnOutcomeSchema.nullable(), + assignedMessageId: z.string().regex(/^[0-9a-f]{64}$/).nullable().optional(), }); const storedReceiptSchema = z.object({ kind: z.literal("committed"), @@ -126,6 +127,7 @@ interface SemanticHead { readonly stopRequestId: string | null; readonly stopWatermark: number | null; readonly providerOutcome: TurnOutcome | null; + readonly assignedMessageId?: string | null; } type StoredOperation = z.infer; @@ -407,7 +409,8 @@ export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter if (head.providerOutcome !== mutation.input.outcome) throw new SemanticConflict(); this.turns.stageTerminalProjection(mutation.input); const receipt = committed(operation, head.durableRevision); - this.storeHead({ ...head, ordinal: operation.ordinal }); + this.storeHead({ ...head, ordinal: operation.ordinal, + assignedMessageId: mutation.input.assistant.messageId ?? null }); this.storeReceipt(operation, hash, receipt); return receipt; })(); @@ -473,7 +476,10 @@ export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter private validStagedFinish(operation: ExecutionSemanticOperation, head: SemanticHead): boolean { const mutation = operation.mutation; if (mutation.kind !== "finish" || !("kind" in mutation.input.projection)) return true; - return head.providerOutcome === mutation.outcome && this.hasStagedTerminalPredecessor(operation); + if (head.providerOutcome !== mutation.outcome || !this.hasStagedTerminalPredecessor(operation)) return false; + return head.assignedMessageId + ? head.assignedMessageId === mutation.input.projection.messageId + : mutation.input.projection.messageId === undefined; } private validFinishHead(operation: ExecutionSemanticOperation, providerId: string): boolean { From b7019d634b3278a190998a2be40d9e5080083a38 Mon Sep 17 00:00:00 2001 From: "(CJ) Chukwudi Nwobodo" <142016413+chuks-qua@users.noreply.github.com> Date: Thu, 24 Sep 2026 22:18:19 +0100 Subject: [PATCH 082/136] refactor(server): extract task tool writer intents --- .../task-tool-intent-reducer.test.ts | 222 ++++++++++++++++++ .../agents/tasks/task-persistence-service.ts | 149 +++--------- .../agents/tasks/task-tool-intent-reducer.ts | 163 +++++++++++++ 3 files changed, 419 insertions(+), 115 deletions(-) create mode 100644 apps/server/src/features/agents/tasks/__tests__/task-tool-intent-reducer.test.ts create mode 100644 apps/server/src/features/agents/tasks/task-tool-intent-reducer.ts diff --git a/apps/server/src/features/agents/tasks/__tests__/task-tool-intent-reducer.test.ts b/apps/server/src/features/agents/tasks/__tests__/task-tool-intent-reducer.test.ts new file mode 100644 index 000000000..046424685 --- /dev/null +++ b/apps/server/src/features/agents/tasks/__tests__/task-tool-intent-reducer.test.ts @@ -0,0 +1,222 @@ +import "reflect-metadata"; +import type { Database } from "bun:sqlite"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { openMemoryDatabase } from "../../../../runtime/persistence/sqlite/database.js"; +import { WorkspaceRepo } from "../../../projects/persistence/workspace-repo.js"; +import { ThreadRepo } from "../../../thread-control/persistence/thread-repo.js"; +import { MessageRepo } from "../../conversation/persistence/message-repo.js"; +import { NarrativeStore } from "../../conversation/narrative/narrative-store.js"; +import { ToolCallRecordRepo } from "../../tools/persistence/tool-call-record-repo.js"; +import { ThoughtSegmentRepo } from "../../conversation/narrative/persistence/thought-segment-repo.js"; +import { HookExecutionRepo } from "../../events/persistence/hook-execution-repo.js"; +import { TaskRepo } from "../../orchestration/persistence/task-repo.js"; +import type { ExecutionIdentity } from "../../execution/execution-mailbox-protocol.js"; +import { TaskPersistenceService } from "../task-persistence-service.js"; +import { + TaskToolIntentReducer, + type TaskToolCommand, + type TaskToolWriteIntent, +} from "../task-tool-intent-reducer.js"; + +describe("TaskToolIntentReducer", () => { + let db: Database; + let tasks: TaskRepo; + let narrative: NarrativeStore; + let service: TaskPersistenceService; + let liveThread: string; + let intentThread: string; + let execution: ExecutionIdentity; + let reducer: TaskToolIntentReducer; + + beforeEach(() => { + db = openMemoryDatabase(); + const workspace = new WorkspaceRepo(db).create("task-tool-reducer", `${process.cwd()}#task-tool-reducer`, false); + const threads = new ThreadRepo(db); + liveThread = threads.create(workspace.id, "live", "direct", "main").id; + intentThread = threads.create(workspace.id, "intents", "direct", "main").id; + tasks = new TaskRepo(db); + narrative = new NarrativeStore( + new MessageRepo(db), + new ToolCallRecordRepo(db), + new ThoughtSegmentRepo(db), + new HookExecutionRepo(db), + db, + ); + narrative.beginTurn(liveThread); + service = new TaskPersistenceService(tasks, narrative); + execution = { threadId: liveThread, turnId: "turn-1", executionId: "execution-1" }; + reducer = new TaskToolIntentReducer(execution); + }); + + afterEach(() => db.close()); + + function applyToIntentThread(intents: readonly TaskToolWriteIntent[]): void { + for (const intent of intents) { + switch (intent.kind) { + case "upsert-group": tasks.upsertGroup(intentThread, intent.group, intent.tasks); break; + case "append-task": tasks.appendTask(intentThread, intent.task); break; + case "update-task": tasks.updateTask(intentThread, intent.id, intent.patch, intent.group); break; + case "remove-task": tasks.removeTask(intentThread, intent.id, intent.group); break; + } + } + } + + function compare(command: TaskToolCommand, applyLive: () => void): TaskToolWriteIntent[] { + const reduction = reducer.reduce(execution, command); + expect(reduction.kind).toBe("intents"); + if (reduction.kind !== "intents") throw new Error("Expected task intents"); + expect(structuredClone(reduction)).toEqual(reduction); + applyToIntentThread(reduction.intents); + applyLive(); + expect(tasks.get(liveThread)).toEqual(tasks.get(intentThread)); + return reduction.intents; + } + + function toolUse( + toolCallId: string, + toolName: string, + toolInput: Record, + parentToolCallId?: string, + ): TaskToolWriteIntent[] { + const attributedParent = narrative.bufferToolCall(liveThread, { + toolCallId, toolName, toolInput, parentToolCallId, + }); + return compare({ + kind: "tool-use", toolName, toolInput, + parentToolCallId: attributedParent, + bufferedCalls: narrative.getBufferedToolCalls(liveThread), + }, () => service.onToolUse(liveThread, { toolName, toolInput, parentToolCallId: attributedParent })); + } + + function toolResult(toolCallId: string, output: string, isError = false): TaskToolWriteIntent[] { + return compare({ + kind: "tool-result", toolCallId, output, isError, + bufferedCalls: narrative.getBufferedToolCalls(liveThread), + }, () => service.onToolResult(liveThread, toolCallId, output, isError)); + } + + it("matches live rows for grouped TodoWrite, update_plan, and status normalization", () => { + expect(toolUse("todos-main", "TodoWrite", { + todos: [ + { content: "main pending", status: "unknown" }, + { content: "main active", status: "in-progress" }, + { content: " ", status: "completed" }, + ], + })).toEqual([{ + kind: "upsert-group", group: "Tasks", + tasks: [ + { content: "main pending", status: "pending", group: "Tasks" }, + { content: "main active", status: "in_progress", group: "Tasks" }, + ], + }]); + + toolUse("agent", "Agent", { description: " Build sub feature " }); + expect(toolUse("todos-sub", "TodoWrite", { + todos: [{ content: "sub cancelled", status: "canceled" }], + }, "agent")).toEqual([{ + kind: "upsert-group", group: "Build sub feature", + tasks: [{ content: "sub cancelled", status: "cancelled", group: "Build sub feature" }], + }]); + narrative.clearAgentStackOnMessage(liveThread); + expect(toolUse("plan", "update_plan", { + tasks: ["first step", { title: "second step", status: "inProgress" }, { description: " " }], + })).toEqual([{ + kind: "upsert-group", group: "Tasks", + tasks: [ + { content: "first step", status: "pending", group: "Tasks" }, + { content: "second step", status: "in_progress", group: "Tasks" }, + ], + }]); + expect(tasks.get(liveThread)).toEqual([ + { content: "sub cancelled", status: "cancelled", group: "Build sub feature" }, + { content: "first step", status: "pending", group: "Tasks" }, + { content: "second step", status: "in_progress", group: "Tasks" }, + ]); + }); + + it("matches live rows for TaskCreate result IDs, updates, and deletion", () => { + toolUse("agent", "Agent", { prompt: " Inspect files " }); + expect(toolUse("create", "TaskCreate", { + subject: " Write tests ", description: " Cover edge cases ", activeForm: " Writing tests ", + }, "agent")).toEqual([]); + expect(toolResult("create", "Created task #27 successfully")).toEqual([{ + kind: "append-task", + task: { + id: "27", content: "Write tests - Cover edge cases", status: "pending", + activeForm: "Writing tests", group: "Inspect files", + }, + }]); + expect(toolUse("update", "TaskUpdate", { + taskId: 27, status: "inProgress", subject: " Test updates ", activeForm: " Checking ", + }, "agent")).toEqual([{ + kind: "update-task", id: "27", group: "Inspect files", + patch: { status: "in_progress", content: "Test updates", activeForm: "Checking" }, + }]); + expect(tasks.get(liveThread)).toEqual([{ + id: "27", content: "Test updates", status: "in_progress", + activeForm: "Checking", group: "Inspect files", + }]); + expect(toolUse("delete", "TaskUpdate", { taskId: "27", status: "deleted" }, "agent")) + .toEqual([{ kind: "remove-task", id: "27", group: "Inspect files" }]); + expect(tasks.get(liveThread)).toEqual([]); + }); + + it("emits no writes for malformed and unsuccessful tool effects", () => { + expect(toolUse("bad-todos", "TodoWrite", { todos: [null, { content: " " }] })).toEqual([]); + expect(toolUse("bad-plan", "update_plan", { plan: [], tasks: [{ title: "ignored" }] })).toEqual([]); + expect(toolUse("bad-update", "TaskUpdate", { taskId: "", status: "completed" })).toEqual([]); + toolUse("create", "TaskCreate", { title: "Valid title" }); + expect(toolResult("create", "Created task #1", true)).toEqual([]); + expect(toolResult("create", "Created task without an id")).toEqual([]); + expect(toolResult("missing", "Created task #1")).toEqual([]); + expect(tasks.get(liveThread)).toBeNull(); + }); + + it("keeps colliding harness IDs scoped to their agent group", () => { + toolUse("agent-a", "Agent", { description: "Agent A" }); + toolUse("agent-b", "Agent", { description: "Agent B" }); + toolUse("create-a", "TaskCreate", { subject: "A task" }, "agent-a"); + toolUse("create-b", "TaskCreate", { subject: "B task" }, "agent-b"); + toolResult("create-a", "Created #1"); + toolResult("create-b", "Created #1"); + toolUse("ambiguous", "TaskUpdate", { taskId: "1", status: "completed" }, "missing-parent"); + expect(tasks.get(liveThread)).toEqual([ + { id: "1", content: "A task", status: "pending", group: "Agent A" }, + { id: "1", content: "B task", status: "pending", group: "Agent B" }, + ]); + toolUse("scoped", "TaskUpdate", { taskId: "1", status: "completed" }, "agent-a"); + expect(tasks.get(liveThread)).toEqual([ + { id: "1", content: "A task", status: "completed", group: "Agent A" }, + { id: "1", content: "B task", status: "pending", group: "Agent B" }, + ]); + toolUse("remove-b", "TaskUpdate", { taskId: "1", status: "deleted" }, "agent-b"); + toolUse("sole-match", "TaskUpdate", { taskId: "1", status: "cancelled" }, "missing-parent"); + expect(tasks.get(liveThread)).toEqual([ + { id: "1", content: "A task", status: "cancelled", group: "Agent A" }, + ]); + }); + + it("rejects stale execution identities before emitting writer intents", () => { + const command: TaskToolCommand = { + kind: "tool-use", toolName: "TodoWrite", + toolInput: { todos: [{ content: "must not appear" }] }, bufferedCalls: [], + }; + expect(reducer.reduce({ ...execution, executionId: "previous" }, command)) + .toEqual({ kind: "stale-execution" }); + expect(reducer.reduce({ ...execution, turnId: "previous" }, command)) + .toEqual({ kind: "stale-execution" }); + expect(reducer.reduce({ ...execution, threadId: intentThread }, command)) + .toEqual({ kind: "stale-execution" }); + const result: TaskToolCommand = { + kind: "tool-result", toolCallId: "create", output: "Created #9", isError: false, + bufferedCalls: [{ toolCallId: "create", toolName: "TaskCreate", _rawToolInput: { subject: "late task" } }], + }; + expect(reducer.reduce({ ...execution, executionId: "previous" }, result)) + .toEqual({ kind: "stale-execution" }); + expect(reducer.reduce(execution, result)).toEqual({ + kind: "intents", execution, + intents: [{ kind: "append-task", task: { id: "9", content: "late task", status: "pending", group: "Tasks" } }], + }); + expect(tasks.get(liveThread)).toBeNull(); + }); +}); diff --git a/apps/server/src/features/agents/tasks/task-persistence-service.ts b/apps/server/src/features/agents/tasks/task-persistence-service.ts index 8624a0dab..6aeba43ae 100644 --- a/apps/server/src/features/agents/tasks/task-persistence-service.ts +++ b/apps/server/src/features/agents/tasks/task-persistence-service.ts @@ -1,7 +1,8 @@ import { inject, injectable } from "tsyringe"; import { logger } from "@mcode/shared"; import { NarrativeStore } from "../conversation/narrative/narrative-store.js"; -import { TaskRepo, type StoredTask } from "../orchestration/persistence/task-repo.js"; +import { TaskRepo } from "../orchestration/persistence/task-repo.js"; +import { taskToolWriteIntents, type TaskToolWriteIntent } from "./task-tool-intent-reducer.js"; /** Persists provider task-tool state for reconnect hydration. */ @injectable() @@ -16,134 +17,52 @@ export class TaskPersistenceService { threadId: string, event: { toolName: string; toolInput: Record; parentToolCallId?: string }, ): void { - const group = event.parentToolCallId ? this.groupFor(threadId, event.parentToolCallId) : "Tasks"; - if (event.toolName === "TodoWrite") { - this.persistTodoWrite(threadId, event.toolInput, group); - return; - } - if (event.toolName === "TaskUpdate") { - this.applyTaskUpdate(threadId, event.toolInput, group); - return; - } - if (event.toolName === "update_plan") this.persistPlanTasks(threadId, event.toolInput, group); + const intents = taskToolWriteIntents({ + kind: "tool-use", + ...event, + bufferedCalls: this.narrative.getBufferedToolCalls(threadId), + }); + const label = event.toolName === "TodoWrite" ? "TodoWrite tasks" + : event.toolName === "update_plan" ? "update_plan tasks" : "TaskUpdate"; + this.applyIntents(threadId, intents, label); } /** Persist one TaskCreate after its result supplies the stable harness identity. */ onToolResult(threadId: string, toolCallId: string, output: string, isError: boolean): void { if (isError) return; - const buffered = this.narrative.getBufferedToolCalls(threadId) - .find((tool) => tool.toolCallId === toolCallId && tool.toolName === "TaskCreate"); - if (!buffered) return; - const id = this.parseHarnessId(output); - const input = buffered._rawToolInput ?? {}; - const content = this.taskContent(input); - if (!id || !content) return; - const group = buffered.parentToolCallId ? this.groupFor(threadId, buffered.parentToolCallId) : "Tasks"; - const activeForm = this.nonEmpty(input.activeForm); - this.tryPersist("TaskCreate task", threadId, () => this.tasks.appendTask(threadId, { - id, - content, - status: "pending", - ...(activeForm ? { activeForm } : {}), - group, - })); - } - - private persistTodoWrite(threadId: string, input: Record, group: string): void { - if (!Array.isArray(input.todos)) return; - const todos = input.todos.flatMap((value): StoredTask[] => { - if (!this.isRecord(value) || !this.nonEmpty(value.content)) return []; - return [{ content: String(value.content), status: this.status(value.status), group }]; - }); - if (todos.length > 0) this.tryPersist("TodoWrite tasks", threadId, () => this.tasks.upsertGroup(threadId, group, todos)); - } - - private persistPlanTasks(threadId: string, input: Record, group: string): void { - const values = Array.isArray(input.plan) - ? input.plan - : Array.isArray(input.tasks) - ? input.tasks - : Array.isArray(input.todos) - ? input.todos - : []; - const tasks = values.flatMap((value): StoredTask[] => { - const item: Record = this.isRecord(value) ? value : { step: value }; - const content = this.nonEmpty(item.step) ?? this.nonEmpty(item.content) - ?? this.nonEmpty(item.title) ?? this.nonEmpty(item.description); - return content ? [{ content, status: this.status(item.status), group }] : []; - }); - if (tasks.length > 0) this.tryPersist("update_plan tasks", threadId, () => this.tasks.upsertGroup(threadId, group, tasks)); - } - - private applyTaskUpdate(threadId: string, input: Record, group: string): void { - const id = input.taskId == null ? "" : String(input.taskId); - if (!id) return; - this.tryPersist("TaskUpdate", threadId, () => { - if (input.status === "deleted") { - this.tasks.removeTask(threadId, id, group); - return; - } - const patch: Partial> = {}; - if (input.status !== undefined) patch.status = this.status(input.status); - const content = this.nonEmpty(input.subject); - const activeForm = this.nonEmpty(input.activeForm); - if (content) patch.content = content; - if (activeForm) patch.activeForm = activeForm; - if (Object.keys(patch).length > 0) this.tasks.updateTask(threadId, id, patch, group); + const intents = taskToolWriteIntents({ + kind: "tool-result", + toolCallId, + output, + isError, + bufferedCalls: this.narrative.getBufferedToolCalls(threadId), }); + this.applyIntents(threadId, intents, "TaskCreate task"); } - private groupFor(threadId: string, parentToolCallId: string): string { - const calls = this.narrative.getBufferedToolCalls(threadId); - let current: string | undefined = parentToolCallId; - while (current) { - const call = calls.find((item) => item.toolCallId === current); - if (!call) break; - if (call.toolName === "Agent") { - const label = this.nonEmpty(call._rawToolInput?.description) ?? this.nonEmpty(call._rawToolInput?.prompt); - return label ? label.slice(0, 80) : "Sub-agent"; - } - current = call.parentToolCallId; + private applyIntents(threadId: string, intents: readonly TaskToolWriteIntent[], label: string): void { + for (const intent of intents) { + this.tryPersist(label, threadId, () => this.applyIntent(threadId, intent)); } - return "Sub-agent"; } - private status(value: unknown): StoredTask["status"] { - switch (value) { - case "inProgress": - case "in-progress": - return "in_progress"; - case "canceled": - return "cancelled"; - case "pending": - case "in_progress": - case "completed": - case "cancelled": - return value; - default: - return "pending"; + private applyIntent(threadId: string, intent: TaskToolWriteIntent): void { + switch (intent.kind) { + case "upsert-group": + this.tasks.upsertGroup(threadId, intent.group, intent.tasks); + return; + case "append-task": + this.tasks.appendTask(threadId, intent.task); + return; + case "update-task": + this.tasks.updateTask(threadId, intent.id, intent.patch, intent.group); + return; + case "remove-task": + this.tasks.removeTask(threadId, intent.id, intent.group); + return; } } - private taskContent(input: Record): string | null { - const subject = this.nonEmpty(input.subject) ?? this.nonEmpty(input.title) ?? this.nonEmpty(input.content); - const description = this.nonEmpty(input.description); - if (!subject) return description; - return description ? `${subject} - ${description}` : subject; - } - - private parseHarnessId(output: string): string | null { - return /#(\d+)/.exec(output)?.[1] ?? null; - } - - private nonEmpty(value: unknown): string | null { - return typeof value === "string" && value.trim().length > 0 ? value.trim() : null; - } - - private isRecord(value: unknown): value is Record { - return value !== null && typeof value === "object"; - } - private tryPersist(label: string, threadId: string, persist: () => void): void { try { persist(); diff --git a/apps/server/src/features/agents/tasks/task-tool-intent-reducer.ts b/apps/server/src/features/agents/tasks/task-tool-intent-reducer.ts new file mode 100644 index 000000000..08c7ae4b1 --- /dev/null +++ b/apps/server/src/features/agents/tasks/task-tool-intent-reducer.ts @@ -0,0 +1,163 @@ +import type { BufferedToolCall } from "../conversation/narrative/narrative-turn-state.js"; +import type { ExecutionIdentity } from "../execution/execution-mailbox-protocol.js"; +import type { StoredTask } from "../orchestration/persistence/task-repo.js"; + +/** Narrative fields needed to resolve TaskCreate and sub-agent groups. */ +export type TaskToolCall = Pick; +type TaskPatch = Partial>; + +/** Data-only task writes for the execution's sole durable writer. */ +export type TaskToolWriteIntent = + | { readonly kind: "upsert-group"; readonly group: string; readonly tasks: StoredTask[] } + | { readonly kind: "append-task"; readonly task: StoredTask } + | { readonly kind: "update-task"; readonly id: string; readonly group: string; readonly patch: TaskPatch } + | { readonly kind: "remove-task"; readonly id: string; readonly group: string }; + +/** Narrative attribution is resolved before a tool event reaches this reducer. */ +export type TaskToolCommand = + | { + readonly kind: "tool-use"; + readonly toolName: string; + readonly toolInput: Record; + readonly parentToolCallId?: string; + readonly bufferedCalls: readonly TaskToolCall[]; + } + | { + readonly kind: "tool-result"; + readonly toolCallId: string; + readonly output: string; + readonly isError: boolean; + readonly bufferedCalls: readonly TaskToolCall[]; + }; + +/** The writer receives intents only when the complete execution identity matches. */ +export type TaskToolReduction = + | { readonly kind: "intents"; readonly execution: ExecutionIdentity; readonly intents: TaskToolWriteIntent[] } + | { readonly kind: "stale-execution" }; + +/** Pure task-tool interpretation shared by the live adapter and execution owner. */ +export function taskToolWriteIntents(command: TaskToolCommand): TaskToolWriteIntent[] { + if (command.kind === "tool-result") return taskCreateIntent(command); + const { toolName, toolInput, parentToolCallId, bufferedCalls } = command; + if (toolName !== "TodoWrite" && toolName !== "TaskUpdate" && toolName !== "update_plan") return []; + const group = parentToolCallId ? groupFor(bufferedCalls, parentToolCallId) : "Tasks"; + if (toolName === "TodoWrite") return todoWriteIntents(toolInput, group); + if (toolName === "TaskUpdate") return taskUpdateIntents(toolInput, group); + return planIntents(toolInput, group); +} + +/** Fences commands to one turn attempt before returning cloneable writer data. */ +export class TaskToolIntentReducer { + readonly execution: ExecutionIdentity; + + constructor(execution: ExecutionIdentity) { + this.execution = { ...execution }; + } + + reduce(execution: ExecutionIdentity, command: TaskToolCommand): TaskToolReduction { + if (execution.threadId !== this.execution.threadId + || execution.turnId !== this.execution.turnId + || execution.executionId !== this.execution.executionId) { + return { kind: "stale-execution" }; + } + return { kind: "intents", execution: { ...this.execution }, intents: taskToolWriteIntents(command) }; + } +} + +function todoWriteIntents(input: Record, group: string): TaskToolWriteIntent[] { + if (!Array.isArray(input.todos)) return []; + const tasks = input.todos.flatMap((value): StoredTask[] => { + if (!isRecord(value) || !nonEmpty(value.content)) return []; + return [{ content: String(value.content), status: status(value.status), group }]; + }); + return tasks.length > 0 ? [{ kind: "upsert-group", group, tasks }] : []; +} + +function planIntents(input: Record, group: string): TaskToolWriteIntent[] { + const values = Array.isArray(input.plan) + ? input.plan + : Array.isArray(input.tasks) + ? input.tasks + : Array.isArray(input.todos) + ? input.todos + : []; + const tasks = values.flatMap((value): StoredTask[] => { + const item: Record = isRecord(value) ? value : { step: value }; + const content = nonEmpty(item.step) ?? nonEmpty(item.content) + ?? nonEmpty(item.title) ?? nonEmpty(item.description); + return content ? [{ content, status: status(item.status), group }] : []; + }); + return tasks.length > 0 ? [{ kind: "upsert-group", group, tasks }] : []; +} + +function taskUpdateIntents(input: Record, group: string): TaskToolWriteIntent[] { + const id = input.taskId == null ? "" : String(input.taskId); + if (!id) return []; + if (input.status === "deleted") return [{ kind: "remove-task", id, group }]; + const patch: TaskPatch = {}; + if (input.status !== undefined) patch.status = status(input.status); + const content = nonEmpty(input.subject); + const activeForm = nonEmpty(input.activeForm); + if (content) patch.content = content; + if (activeForm) patch.activeForm = activeForm; + return Object.keys(patch).length > 0 ? [{ kind: "update-task", id, group, patch }] : []; +} + +function taskCreateIntent(command: Extract): TaskToolWriteIntent[] { + if (command.isError) return []; + const buffered = command.bufferedCalls + .find((tool) => tool.toolCallId === command.toolCallId && tool.toolName === "TaskCreate"); + if (!buffered) return []; + const id = /#(\d+)/.exec(command.output)?.[1]; + const input = buffered._rawToolInput ?? {}; + const content = taskContent(input); + if (!id || !content) return []; + const group = buffered.parentToolCallId ? groupFor(command.bufferedCalls, buffered.parentToolCallId) : "Tasks"; + const activeForm = nonEmpty(input.activeForm); + return [{ + kind: "append-task", + task: { id, content, status: "pending", ...(activeForm ? { activeForm } : {}), group }, + }]; +} + +function groupFor(calls: readonly TaskToolCall[], parentToolCallId: string): string { + let current: string | undefined = parentToolCallId; + while (current) { + const call = calls.find((item) => item.toolCallId === current); + if (!call) break; + if (call.toolName === "Agent") { + const label = nonEmpty(call._rawToolInput?.description) ?? nonEmpty(call._rawToolInput?.prompt); + return label ? label.slice(0, 80) : "Sub-agent"; + } + current = call.parentToolCallId; + } + return "Sub-agent"; +} + +function status(value: unknown): StoredTask["status"] { + switch (value) { + case "inProgress": + case "in-progress": return "in_progress"; + case "canceled": return "cancelled"; + case "pending": + case "in_progress": + case "completed": + case "cancelled": return value; + default: return "pending"; + } +} + +function taskContent(input: Record): string | null { + const subject = nonEmpty(input.subject) ?? nonEmpty(input.title) ?? nonEmpty(input.content); + const description = nonEmpty(input.description); + if (!subject) return description; + return description ? `${subject} - ${description}` : subject; +} + +function nonEmpty(value: unknown): string | null { + return typeof value === "string" && value.trim().length > 0 ? value.trim() : null; +} + +function isRecord(value: unknown): value is Record { + return value !== null && typeof value === "object"; +} From e993ff9ee339eb96a38eec9a5f9cf25e5b72197e Mon Sep 17 00:00:00 2001 From: "(CJ) Chukwudi Nwobodo" <142016413+chuks-qua@users.noreply.github.com> Date: Thu, 24 Sep 2026 22:19:47 +0100 Subject: [PATCH 083/136] fix(server): bound task tool parent traversal --- .../__tests__/task-tool-intent-reducer.test.ts | 15 +++++++++++++++ .../agents/tasks/task-tool-intent-reducer.ts | 4 +++- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/apps/server/src/features/agents/tasks/__tests__/task-tool-intent-reducer.test.ts b/apps/server/src/features/agents/tasks/__tests__/task-tool-intent-reducer.test.ts index 046424685..26a157fbf 100644 --- a/apps/server/src/features/agents/tasks/__tests__/task-tool-intent-reducer.test.ts +++ b/apps/server/src/features/agents/tasks/__tests__/task-tool-intent-reducer.test.ts @@ -172,6 +172,21 @@ describe("TaskToolIntentReducer", () => { expect(tasks.get(liveThread)).toBeNull(); }); + it("bounds malformed parent chains before selecting a sub-agent group", () => { + expect(reducer.reduce(execution, { + kind: "tool-use", toolName: "TodoWrite", + toolInput: { todos: [{ content: "Task" }] }, parentToolCallId: "first", + bufferedCalls: [ + { toolCallId: "first", toolName: "Other", parentToolCallId: "second", _rawToolInput: {} }, + { toolCallId: "second", toolName: "Other", parentToolCallId: "first", _rawToolInput: {} }, + ], + })).toEqual({ + kind: "intents", execution, + intents: [{ kind: "upsert-group", group: "Sub-agent", + tasks: [{ content: "Task", status: "pending", group: "Sub-agent" }] }], + }); + }); + it("keeps colliding harness IDs scoped to their agent group", () => { toolUse("agent-a", "Agent", { description: "Agent A" }); toolUse("agent-b", "Agent", { description: "Agent B" }); diff --git a/apps/server/src/features/agents/tasks/task-tool-intent-reducer.ts b/apps/server/src/features/agents/tasks/task-tool-intent-reducer.ts index 08c7ae4b1..0387515d0 100644 --- a/apps/server/src/features/agents/tasks/task-tool-intent-reducer.ts +++ b/apps/server/src/features/agents/tasks/task-tool-intent-reducer.ts @@ -122,7 +122,9 @@ function taskCreateIntent(command: Extract(); + while (current && !seen.has(current)) { + seen.add(current); const call = calls.find((item) => item.toolCallId === current); if (!call) break; if (call.toolName === "Agent") { From bd5f9af779a3b2d94ee277ff26f84f851d85799a Mon Sep 17 00:00:00 2001 From: "(CJ) Chukwudi Nwobodo" <142016413+chuks-qua@users.noreply.github.com> Date: Thu, 24 Sep 2026 22:20:49 +0100 Subject: [PATCH 084/136] refactor(server): freeze turn diff evidence as execution data --- .../turns/__tests__/turn-diff-service.test.ts | 17 ++++++- .../agents/turns/turn-diff-service.ts | 48 +++++++++++++++---- 2 files changed, 56 insertions(+), 9 deletions(-) diff --git a/apps/server/src/features/agents/turns/__tests__/turn-diff-service.test.ts b/apps/server/src/features/agents/turns/__tests__/turn-diff-service.test.ts index a2876b311..d4ccae5fd 100644 --- a/apps/server/src/features/agents/turns/__tests__/turn-diff-service.test.ts +++ b/apps/server/src/features/agents/turns/__tests__/turn-diff-service.test.ts @@ -4,7 +4,7 @@ import type { Database } from "bun:sqlite"; import type { ProviderTurnDiffUpdate, TurnFileEffectSummary } from "@mcode/contracts"; import { openMemoryDatabase } from "../../../../runtime/persistence/sqlite/database.js"; import { TurnDiffRepo } from "../persistence/turn-diff-repo.js"; -import { TURN_DIFF_MAX_BYTES, TurnDiffService } from "../turn-diff-service.js"; +import { TURN_DIFF_MAX_BYTES, TurnDiffService, selectTurnDiffSettlement } from "../turn-diff-service.js"; const patch = "diff --git a/a.txt b/a.txt\nindex 1..2\n--- a/a.txt\n+++ b/a.txt\n@@ -1 +1 @@\n-old\n+new\n"; const identity = { threadId: "thread-1", turnId: "turn-1", turnExecutionId: "execution-1", deliveryAttempt: 1 }; @@ -72,6 +72,21 @@ describe("TurnDiffService production settlement", () => { expect(service.push(update({ revision: 99 }))).toBe("stale"); }); + it("transfers cloneable terminal evidence with exact execution fencing", () => { + expect(service.push(update())).toBe("accepted"); + expect(service.takeFinalizationEvidence(identity.threadId, "stale-execution")).toBeNull(); + const prepared = service.takeFinalizationEvidence(identity.threadId, identity.turnExecutionId); + expect(prepared).not.toBeNull(); + if (!prepared) throw new Error("Expected frozen terminal evidence"); + expect(service.liveComparison(identity.threadId)).toBeNull(); + expect(service.takeFinalizationEvidence(identity.threadId, identity.turnExecutionId)).toBeNull(); + expect(selectTurnDiffSettlement(structuredClone(prepared), "completed", effects)).toEqual({ + thread_id: identity.threadId, source: "native", patch, revision: 1, + }); + expect(selectTurnDiffSettlement(prepared, "interrupted", effects)).toBeNull(); + expect(repo.latest(identity.threadId)).toBeUndefined(); + }); + it.each(["invalidated", "interrupted", "cancelled", "errored"] as const)("preserves previous settlement on %s", (state) => { service.push(update()); service.prepareFinalization(identity.threadId, identity.turnExecutionId, "completed")("message-1", effects); diff --git a/apps/server/src/features/agents/turns/turn-diff-service.ts b/apps/server/src/features/agents/turns/turn-diff-service.ts index eb45cb945..ebc5e1113 100644 --- a/apps/server/src/features/agents/turns/turn-diff-service.ts +++ b/apps/server/src/features/agents/turns/turn-diff-service.ts @@ -17,6 +17,17 @@ interface ActiveDiff extends TurnDiffIdentity { rejected?: boolean; } +/** Cloneable native evidence frozen for one execution before terminal file settlement. */ +export interface PreparedTurnDiffEvidence extends TurnDiffIdentity { + readonly threadId: string; + readonly revision: number; + readonly evidence: ProviderTurnDiffUpdate | null; + readonly rejected: boolean; +} + +/** Data the terminal writer can store with its assigned assistant message. */ +export type SelectedTurnDiff = Pick; + /** Owns admission, volatile native evidence, and final source reconciliation. */ export class TurnDiffService { private readonly active = new Map(); @@ -57,19 +68,28 @@ export class TurnDiffService { /** Freeze native evidence at the terminal fence before asynchronous file settlement. */ prepareFinalization(threadId: string, executionId: string | undefined, outcome: TurnOutcome): SettleTurnDiff { - const current = [...this.active.values()].find((entry) => entry.threadId === threadId && entry.turnExecutionId === executionId); + const current = this.takeFinalizationEvidence(threadId, executionId); if (!current) return () => {}; - this.clear(current.turnId, executionId); return (messageId, effects, reconstructionPatch) => { - if (outcome !== "completed") return; - const selected = selectSettledEvidence(current, effects, reconstructionPatch); + const selected = selectTurnDiffSettlement(current, outcome, effects, reconstructionPatch); if (!selected) return; - this.repo.create({ id: NodeCrypto.randomUUID(), message_id: messageId, thread_id: threadId, - ...selected, revision: Math.max(0, current.revision) }); + this.repo.create({ id: NodeCrypto.randomUUID(), message_id: messageId, ...selected }); this.changed(threadId); }; } + /** Transfer terminal evidence only for its exact admitted execution. */ + takeFinalizationEvidence(threadId: string, executionId: string | undefined): PreparedTurnDiffEvidence | null { + const current = [...this.active.values()].find((entry) => entry.threadId === threadId && entry.turnExecutionId === executionId); + if (!current) return null; + this.clear(current.turnId, executionId); + return { + threadId: current.threadId, turnId: current.turnId, + turnExecutionId: current.turnExecutionId, deliveryAttempt: current.deliveryAttempt, + revision: current.revision, evidence: structuredClone(current.evidence), rejected: current.rejected ?? false, + }; + } + /** Read current Live metadata; native patch bytes stay on the file-diff endpoint. */ liveComparison(threadId: string): ReviewComparison | null { const current = [...this.active.values()].find((entry) => entry.threadId === threadId); @@ -119,14 +139,26 @@ function isNewRevision(update: ProviderTurnDiffUpdate, current: ActiveDiff): boo return matches(update, current) && Number.isSafeInteger(update.revision) && update.revision > current.revision; } -function selectSettledEvidence(current: ActiveDiff, effects: TurnFileEffectSummary | undefined, reconstructionPatch?: string): Pick | null { +/** Reconcile frozen native evidence with settled file effects without a repository connection. */ +export function selectTurnDiffSettlement( + current: PreparedTurnDiffEvidence, + outcome: TurnOutcome, + effects: TurnFileEffectSummary | undefined, + reconstructionPatch?: string, +): SelectedTurnDiff | null { + if (outcome !== "completed") return null; + const selected = selectSettledEvidence(current, effects, reconstructionPatch); + return selected ? { thread_id: current.threadId, ...selected, revision: Math.max(0, current.revision) } : null; +} + +function selectSettledEvidence(current: PreparedTurnDiffEvidence, effects: TurnFileEffectSummary | undefined, reconstructionPatch?: string): Pick | null { const native = nativeSettlement(current, effects); if (native !== undefined) return native; if (reconstructionPatch && parseTurnDiff(reconstructionPatch)) return { source: "tracked", patch: reconstructionPatch }; return effects?.fileCount ? { source: "git", patch: null } : null; } -function nativeSettlement(current: ActiveDiff, effects: TurnFileEffectSummary | undefined): Pick | null | undefined { +function nativeSettlement(current: PreparedTurnDiffEvidence, effects: TurnFileEffectSummary | undefined): Pick | null | undefined { const evidence = current.evidence; if (!evidence) return undefined; if (evidence.state === "snapshot") return { source: "native", patch: evidence.patch }; From 8d0b5f53ee94dffd47df952cd4e08113ff77f459 Mon Sep 17 00:00:00 2001 From: "(CJ) Chukwudi Nwobodo" <142016413+chuks-qua@users.noreply.github.com> Date: Thu, 24 Sep 2026 22:23:16 +0100 Subject: [PATCH 085/136] feat(server): settle turn diff with terminal assistant --- .../canonical-parent-turn-write.test.ts | 26 +++++++++++++++++++ .../canonical/canonical-parent-turn-write.ts | 26 +++++++++++++++++++ 2 files changed, 52 insertions(+) diff --git a/apps/server/src/features/agents/canonical/__tests__/canonical-parent-turn-write.test.ts b/apps/server/src/features/agents/canonical/__tests__/canonical-parent-turn-write.test.ts index 2d26450cd..ebae45a8a 100644 --- a/apps/server/src/features/agents/canonical/__tests__/canonical-parent-turn-write.test.ts +++ b/apps/server/src/features/agents/canonical/__tests__/canonical-parent-turn-write.test.ts @@ -275,6 +275,32 @@ describe("CanonicalParentTurnWrite", () => { }); }); + it("commits selected diff evidence with the assigned terminal assistant", async () => { + writer.start(startInput()); + const messageId = deriveTurnAssistantMessageId(THREAD_ID, "user-1"); + const input = terminalProjectionInput(); + input.assistant.messageId = messageId; + writer.stageTerminalProjection(input); + const patch = "diff --git a/a.txt b/a.txt\nindex 1..2\n--- a/a.txt\n+++ b/a.txt\n@@ -1 +1 @@\n-old\n+new\n"; + const finish: DataOnlyParentTurnFinishInput = { + ...finishInput(new MessageRepo(db).findByIdInThreadIncludingInternal(THREAD_ID, messageId)!), + projection: { kind: "writer-staged", messageId }, + selectedTurnDiff: { thread_id: THREAD_ID, source: "native", patch, revision: 2 }, + }; + expect(() => writer.finish({ ...finish, selectedTurnDiff: { ...finish.selectedTurnDiff!, thread_id: "other" } })) + .toThrow("Invalid selected turn diff"); + db.run("CREATE TRIGGER fail_turn_diff BEFORE INSERT ON turn_diff_snapshots BEGIN SELECT RAISE(ABORT, 'diff unavailable'); END"); + await expect(writer.finish(finish)).rejects.toThrow("diff unavailable"); + expect(db.prepare("SELECT is_internal FROM messages WHERE id = ?").get(messageId)).toEqual({ is_internal: 1 }); + expect(db.prepare("SELECT terminal_outcome FROM canonical_agent_ingest_checkpoints WHERE execution_id = ?").get(EXECUTION_ID)) + .toEqual({ terminal_outcome: null }); + db.run("DROP TRIGGER fail_turn_diff"); + expect((await writer.finish(finish)).outcome).toBe("committed"); + expect(db.prepare("SELECT message_id, source, patch, revision FROM turn_diff_snapshots WHERE message_id = ?") + .get(messageId)).toEqual({ message_id: messageId, source: "native", patch, revision: 2 }); + expect(db.prepare("SELECT count(*) AS count FROM turn_diff_snapshots").get()).toEqual({ count: 1 }); + }); + it("refuses to reuse a public assistant row as a staged turn projection", () => { writer.start(startInput()); const input = terminalProjectionInput(); diff --git a/apps/server/src/features/agents/canonical/canonical-parent-turn-write.ts b/apps/server/src/features/agents/canonical/canonical-parent-turn-write.ts index 74d9389d3..6d526a6b5 100644 --- a/apps/server/src/features/agents/canonical/canonical-parent-turn-write.ts +++ b/apps/server/src/features/agents/canonical/canonical-parent-turn-write.ts @@ -1,4 +1,5 @@ import type { Database } from "bun:sqlite"; +import * as NodeCrypto from "node:crypto"; import { ParentNarrativeRecoveryItemSchema, type ParentNarrativeRecoveryItem, @@ -15,6 +16,9 @@ import { PlanQuestionAnswersRepo } from "../planning/persistence/plan-question-a import { ToolCallRecordRepo } from "../tools/persistence/tool-call-record-repo.js"; import { deriveTurnAssistantMessageId } from "../turns/turn-assistant-message-id.js"; import { ParentAssistantTextCheckpointService } from "../turns/parent-assistant-text-checkpoint-service.js"; +import { TURN_DIFF_MAX_BYTES, parseTurnDiff } from "../turns/turn-diff-patch.js"; +import { TurnDiffRepo } from "../turns/persistence/turn-diff-repo.js"; +import type { SelectedTurnDiff } from "../turns/turn-diff-service.js"; import type { ParentTurnFinishInput, ParentTurnProjection, @@ -93,6 +97,7 @@ export interface DataOnlyParentEventInput extends Omit { projection: ParentTurnProjection | { readonly kind: "writer-staged"; readonly messageId?: string }; + selectedTurnDiff?: SelectedTurnDiff; } /** Cloneable terminal data whose compatibility rows are staged on the writer connection. */ @@ -131,6 +136,7 @@ export class CanonicalParentTurnWrite { private readonly planAnswers: PlanQuestionAnswersRepo; private readonly stagedAssistant: ReturnType; private readonly assistantTextCheckpoints: ParentAssistantTextCheckpointService; + private readonly turnDiffs: TurnDiffRepo; constructor(db: Database, publish: CanonicalAgentEventPublisher) { this.db = db; @@ -147,6 +153,7 @@ export class CanonicalParentTurnWrite { this.planAnswers = new PlanQuestionAnswersRepo(db); this.stagedAssistant = db.prepare("SELECT role, content FROM messages WHERE id = ? AND thread_id = ?"); this.assistantTextCheckpoints = new ParentAssistantTextCheckpointService(db); + this.turnDiffs = new TurnDiffRepo(db); } /** Import fsynced text before the interruption transaction reads its durable prefix. */ @@ -328,6 +335,10 @@ export class CanonicalParentTurnWrite { ? this.loadStagedTerminalProjection(input.threadId, input.executionId, input.projection.messageId) : input.projection; this.assertStagedAssistant(input.threadId, staged); + if (input.selectedTurnDiff && (input.outcome !== "completed" || !staged.message + || !validSelectedTurnDiff(input.selectedTurnDiff, input.threadId))) { + throw new Error("Invalid selected turn diff for terminal assistant"); + } const projection: ParentTurnProjection = { message: staged.message ? { @@ -346,6 +357,9 @@ export class CanonicalParentTurnWrite { if (!projection.message) return; this.messages.setAssistantOutcome(projection.message.id, input.outcome, input.executionId); this.messages.publishAssistant(projection.message.id); + if (input.selectedTurnDiff) this.turnDiffs.create({ + id: NodeCrypto.randomUUID(), message_id: projection.message.id, ...input.selectedTurnDiff, + }); }, }, onBatchWrite); } @@ -374,6 +388,18 @@ export class CanonicalParentTurnWrite { } } +function validSelectedTurnDiff(selected: SelectedTurnDiff, threadId: string): boolean { + if (selected.thread_id !== threadId || !Number.isSafeInteger(selected.revision) || selected.revision < 0) return false; + if (selected.source === "git") return selected.patch === null; + if (selected.source !== "native" && selected.source !== "tracked") return false; + return validSelectedPatch(selected.source, selected.patch); +} + +function validSelectedPatch(source: "native" | "tracked", patch: string | null): boolean { + if (typeof patch !== "string" || Buffer.byteLength(patch, "utf8") > TURN_DIFF_MAX_BYTES) return false; + return source === "native" && patch.length === 0 || parseTurnDiff(patch) !== null; +} + function isUnfinishedLostTurn( turn: ReturnType, checkpoint: ReturnType, From 70114bab3d9bda19c70bdcaf92db4b6bb9526e7e Mon Sep 17 00:00:00 2001 From: "(CJ) Chukwudi Nwobodo" <142016413+chuks-qua@users.noreply.github.com> Date: Thu, 24 Sep 2026 22:21:48 +0100 Subject: [PATCH 086/136] feat(server): release committed live events on host --- .../canonical-agent-writer-client.test.ts | 15 ++- .../canonical-execution-writer-port.test.ts | 120 ++++++++++++++++++ .../canonical-execution-writer-port.ts | 6 +- .../execution-live-publication-release.ts | 83 ++++++++++++ .../execution/execution-worker-handler.ts | 17 ++- 5 files changed, 235 insertions(+), 6 deletions(-) create mode 100644 apps/server/src/features/agents/canonical/execution-live-publication-release.ts diff --git a/apps/server/src/features/agents/canonical/__tests__/canonical-agent-writer-client.test.ts b/apps/server/src/features/agents/canonical/__tests__/canonical-agent-writer-client.test.ts index f0676fff5..373080148 100644 --- a/apps/server/src/features/agents/canonical/__tests__/canonical-agent-writer-client.test.ts +++ b/apps/server/src/features/agents/canonical/__tests__/canonical-agent-writer-client.test.ts @@ -8,6 +8,9 @@ import { AgentEventType, type ParentNarrativeRecoveryItem } from "@mcode/contrac import { openDatabase } from "../../../../runtime/persistence/sqlite/database.js"; import type { CanonicalAgentEventDraft } from "../canonical-agent-boundary.js"; import { CanonicalAgentWriterClient } from "../canonical-agent-writer-client.js"; +import { CanonicalExecutionWriterPort } from "../canonical-execution-writer-port.js"; +import { ExecutionLivePublicationRelease } from "../execution-live-publication-release.js"; +import { AgentEventPublicationRegistry } from "../../orchestration/agent-event-publication-registry.js"; import type { CanonicalWriterResponse } from "../canonical-agent-writer-protocol.js"; import { ParentAssistantTextCheckpointService } from "../../turns/parent-assistant-text-checkpoint-service.js"; import type { ExecutionSemanticOperation } from "../../execution/execution-worker-handler.js"; @@ -178,12 +181,18 @@ describe("canonical SQLite writer", () => { } }], }; let created = 0; + const registry = new AgentEventPublicationRegistry(); + const published: string[] = []; + registry.bind((event) => published.push(event.type)); + const release = new ExecutionLivePublicationRelease(registry); writer = new CanonicalAgentWriterClient(dbPath, () => created++ === 0 ? workerDroppingReply("semantic-transacted") : new Worker(new URL("../canonical-agent-writer.worker.ts", import.meta.url), { type: "module" })); + let port = new CanonicalExecutionWriterPort(writer, () => {}, release); - const receipt = await writer.transactSemantic(operation, () => {}); + const receipt = await port.transact(operation); expect(created).toBe(2); + expect(published).toEqual([AgentEventType.TurnStarted]); expect(receipt).toMatchObject({ kind: "committed", livePublication: [{ publicationId: "publication-lease:1:0", after: "writer", event: { type: AgentEventType.TurnStarted, turnExecutionId: EXECUTION_ID }, @@ -195,7 +204,9 @@ describe("canonical SQLite writer", () => { await writer.close(); writer = new CanonicalAgentWriterClient(dbPath); - expect(await writer.transactSemantic(operation, () => {})).toEqual(receipt); + port = new CanonicalExecutionWriterPort(writer, () => {}, release); + expect(await port.transact(operation)).toEqual(receipt); + expect(published).toEqual([AgentEventType.TurnStarted]); }); it("replays durable semantic assistant text after a worker reply is lost", async () => { diff --git a/apps/server/src/features/agents/canonical/__tests__/canonical-execution-writer-port.test.ts b/apps/server/src/features/agents/canonical/__tests__/canonical-execution-writer-port.test.ts index c8dbd2180..d7a74d43b 100644 --- a/apps/server/src/features/agents/canonical/__tests__/canonical-execution-writer-port.test.ts +++ b/apps/server/src/features/agents/canonical/__tests__/canonical-execution-writer-port.test.ts @@ -12,9 +12,11 @@ import { deriveTurnAssistantMessageId } from "../../turns/turn-assistant-message import { ExecutionMailboxScheduler } from "../../execution/execution-mailbox-scheduler.js"; import type { ExecutionLease } from "../../execution/execution-mailbox-protocol.js"; import { ExecutionThreadWorkerPort } from "../../execution/execution-worker-port.js"; +import { AgentEventPublicationRegistry } from "../../orchestration/agent-event-publication-registry.js"; import type { ExecutionSemanticOperation, ExecutionWorkCommand, ExecutionWorkerResult } from "../../execution/execution-worker-handler.js"; import { CanonicalAgentWriterClient } from "../canonical-agent-writer-client.js"; import { CanonicalExecutionWriterPort } from "../canonical-execution-writer-port.js"; +import { ExecutionLivePublicationRelease } from "../execution-live-publication-release.js"; const NOW = "2026-09-24T10:00:00.000Z"; const THREAD_ID = "semantic-worker-thread"; @@ -78,6 +80,124 @@ describe("execution semantic writer transport", () => { NodeFS.rmSync(directory, { recursive: true, force: true }); }); + it("releases live receipts in order after their writer and terminal barriers", async () => { + writer = new CanonicalAgentWriterClient(NodePath.join(directory, "app.sqlite")); + const registry = new AgentEventPublicationRegistry(); + const published: AgentEvent[] = []; + registry.bind((event) => published.push(event)); + const release = new ExecutionLivePublicationRelease(registry); + const port = new CanonicalExecutionWriterPort(writer, () => {}, release); + const started = { type: AgentEventType.TurnStarted, threadId: THREAD_ID, turnExecutionId: EXECUTION_ID }; + const begin: ExecutionSemanticOperation = { + ...beginOperation(), livePublication: [{ after: "writer", event: started }], + }; + expect((await port.transact(begin)).kind).toBe("committed"); + expect(published.map((event) => event.type)).toEqual([AgentEventType.TurnStarted]); + + const delta: AgentEvent = { type: AgentEventType.TextDelta, threadId: THREAD_ID, + turnExecutionId: EXECUTION_ID, delta: "answer", isFinalResponse: true }; + const system: AgentEvent = { type: AgentEventType.System, threadId: THREAD_ID, + turnExecutionId: EXECUTION_ID, subtype: "test" }; + const append: ExecutionSemanticOperation = { + operationId: `${lease.leaseId}:2`, execution, lease, ordinal: 2, + mutation: { kind: "append-events", phase: "running", nativeCursor: null, + events: [runtimeDraft(1, delta), runtimeDraft(2, system)] }, + livePublication: [{ after: "writer", event: delta }, { after: "writer", event: system }], + }; + expect((await port.transact(append)).kind).toBe("committed"); + expect(published.map((event) => event.type)) + .toEqual([AgentEventType.TurnStarted, AgentEventType.TextDelta, AgentEventType.System]); + expect((await port.transact(append)).kind).toBe("committed"); + expect(published).toHaveLength(3); + + const staged = new MessageRepo(db).create(THREAD_ID, "assistant", "Answer", 2, + undefined, undefined, undefined, "model", true); + const ended: AgentEvent = { type: AgentEventType.Ended, threadId: THREAD_ID, + turnExecutionId: EXECUTION_ID, outcome: "completed" }; + const finish: ExecutionSemanticOperation = { + operationId: `${lease.leaseId}:3`, execution, lease, ordinal: 3, + mutation: { kind: "finish", outcome: "completed", input: { + threadId: THREAD_ID, turnId: TURN_ID, executionId: EXECUTION_ID, providerId: "codex", + providerIdentities: [], outcome: "completed", projection: { message: staged, narrative: [] }, + } }, + livePublication: [{ after: "terminal", event: ended }], + }; + db.run("CREATE TRIGGER fail_live_release BEFORE UPDATE OF kind ON canonical_writer_operation_receipts WHEN NEW.kind = 'semantic:finish' BEGIN SELECT RAISE(ABORT, 'finish unavailable'); END"); + await expect(port.transact(finish)).rejects.toThrow("Canonical writer write-failed"); + expect(published).toHaveLength(3); + expect(db.prepare("SELECT terminal_outcome FROM canonical_agent_ingest_checkpoints WHERE execution_id = ?") + .get(EXECUTION_ID)).toEqual({ terminal_outcome: null }); + db.run("DROP TRIGGER fail_live_release"); + + expect((await port.transact(finish)).kind).toBe("committed"); + expect(published.map((event) => event.type)) + .toEqual([AgentEventType.TurnStarted, AgentEventType.TextDelta, AgentEventType.System, AgentEventType.Ended]); + expect((await port.transact(finish)).kind).toBe("committed"); + expect(published).toHaveLength(4); + }); + + it("carries a live publication from the execution worker to the bound host publisher", async () => { + writer = new CanonicalAgentWriterClient(NodePath.join(directory, "app.sqlite")); + const registry = new AgentEventPublicationRegistry(); + const published: AgentEvent[] = []; + registry.bind((event) => published.push(event)); + const release = new ExecutionLivePublicationRelease(registry); + const port = new CanonicalExecutionWriterPort(writer, () => {}, release); + const scheduler = new ExecutionMailboxScheduler({ + workerCount: 1, + limits: { + maxPending: 4, maxPendingBytes: 8_000, reservedControl: 2, reservedControlBytes: 4_000, + maxPerExecutionPending: 4, maxPerExecutionBytes: 8_000, + reservedPerExecutionControl: 2, reservedPerExecutionControlBytes: 4_000, + }, + createWorker: () => new ExecutionThreadWorkerPort(port), + onWorkerLost: () => { throw new Error("Execution worker was lost"); }, + }); + try { + const claim = scheduler.claim(execution, 1); + if (claim.kind !== "claimed") throw new Error(`Execution claim failed: ${claim.kind}`); + const begin = beginOperation().mutation; + if (begin.kind !== "begin") throw new Error("Unexpected begin operation"); + const admission = scheduler.submit({ execution, lease: claim.lease, byteLength: 1_000, + command: { kind: "start", providerId: "codex", input: begin.input, + livePublication: [{ after: "writer", event: { + type: AgentEventType.TurnStarted, threadId: THREAD_ID, turnExecutionId: EXECUTION_ID, + } }], + }, + }); + if (admission.kind !== "admitted") throw new Error(`Execution admission failed: ${admission.kind}`); + await expect(admission.completion).resolves.toMatchObject({ kind: "reply", result: { + kind: "committed", livePublication: [{ publicationId: `${claim.lease.leaseId}:1:0` }], + } }); + expect(published.map((event) => event.type)).toEqual([AgentEventType.TurnStarted]); + expect(db.prepare("SELECT COUNT(*) AS count FROM messages WHERE thread_id = ?").get(THREAD_ID)) + .toEqual({ count: 1 }); + } finally { + scheduler.shutdown(); + } + }); + + it("replays a committed live receipt when the public publisher becomes available", async () => { + writer = new CanonicalAgentWriterClient(NodePath.join(directory, "app.sqlite")); + const registry = new AgentEventPublicationRegistry(); + const release = new ExecutionLivePublicationRelease(registry); + const port = new CanonicalExecutionWriterPort(writer, () => {}, release); + const begin: ExecutionSemanticOperation = { ...beginOperation(), livePublication: [{ + after: "writer", event: { + type: AgentEventType.TurnStarted, threadId: THREAD_ID, turnExecutionId: EXECUTION_ID, + }, + }] }; + await expect(port.transact(begin)).rejects.toThrow("Live AgentEvent publisher is not bound"); + expect(db.prepare("SELECT COUNT(*) AS count FROM messages WHERE thread_id = ?").get(THREAD_ID)) + .toEqual({ count: 1 }); + const published: AgentEvent[] = []; + registry.bind((event) => published.push(event)); + expect((await port.transact(begin)).kind).toBe("committed"); + expect(published.map((event) => event.type)).toEqual([AgentEventType.TurnStarted]); + expect((await port.transact(begin)).kind).toBe("committed"); + expect(published).toHaveLength(1); + }); + it("commits and replays a semantic start through the dedicated SQLite worker", async () => { writer = new CanonicalAgentWriterClient(NodePath.join(directory, "app.sqlite")); const published: string[] = []; diff --git a/apps/server/src/features/agents/canonical/canonical-execution-writer-port.ts b/apps/server/src/features/agents/canonical/canonical-execution-writer-port.ts index 0c1b02c90..77f777d41 100644 --- a/apps/server/src/features/agents/canonical/canonical-execution-writer-port.ts +++ b/apps/server/src/features/agents/canonical/canonical-execution-writer-port.ts @@ -6,17 +6,21 @@ import type { import type { CanonicalAgentEventPublisher } from "./canonical-agent-boundary.js"; import { CanonicalAgentWriterClient } from "./canonical-agent-writer-client.js"; import type { LostExecutionInterruption } from "./canonical-execution-semantic-writer.js"; +import type { ExecutionLivePublicationRelease } from "./execution-live-publication-release.js"; /** Bridges an execution worker to the sole SQLite writer and publishes only committed events. */ export class CanonicalExecutionWriterPort implements ExecutionSemanticWriter { constructor( private readonly writer: CanonicalAgentWriterClient, private readonly publish: CanonicalAgentEventPublisher, + private readonly livePublication?: ExecutionLivePublicationRelease, ) {} /** Resolve after the durable writer receipt and its canonical event publication. */ async transact(operation: ExecutionSemanticOperation): Promise { - return this.writer.transactSemantic(operation, this.publish); + const receipt = await this.writer.transactSemantic(operation, this.publish); + this.livePublication?.release(operation, receipt); + return receipt; } /** Reconcile a lost worker through the sole writer and publish its committed interruption. */ diff --git a/apps/server/src/features/agents/canonical/execution-live-publication-release.ts b/apps/server/src/features/agents/canonical/execution-live-publication-release.ts new file mode 100644 index 000000000..5e8bd728d --- /dev/null +++ b/apps/server/src/features/agents/canonical/execution-live-publication-release.ts @@ -0,0 +1,83 @@ +import { AgentEventSchema, type AgentEvent } from "@mcode/contracts"; +import * as NodeUtil from "node:util"; + +import type { + ExecutionLivePublicationReceipt, ExecutionSemanticOperation, ExecutionWriteReceipt, +} from "../execution/execution-worker-handler.js"; + +const MAX_TRACKED_PUBLICATIONS = 8_192; +const MAX_RECEIPT_EVENTS = 64; +const MAX_RECEIPT_BYTES = 512 * 1024; + +/** The bound public AgentEvent bridge, normally AgentEventPublicationRegistry. */ +export interface LiveAgentEventPublisher { + isBound(): boolean; + publish(event: AgentEvent): void; +} + +/** Releases committed live events once per host lifetime, even when a writer reply is replayed. */ +export class ExecutionLivePublicationRelease { + private readonly published = new Set(); + + constructor(private readonly publisher: LiveAgentEventPublisher) {} + + /** Validate the entire receipt before exposing any event in its original order. */ + release(operation: ExecutionSemanticOperation, receipt: ExecutionWriteReceipt): void { + if (receipt.kind !== "committed" || !receipt.livePublication) return; + const events = this.validate(operation, receipt); + if (!this.publisher.isBound()) throw new Error("Live AgentEvent publisher is not bound"); + const newCount = events.filter((entry) => !this.published.has(entry.key)).length; + if (this.published.size + newCount > MAX_TRACKED_PUBLICATIONS) { + throw new Error("Live AgentEvent publication tracking is full"); + } + for (const entry of events) { + if (this.published.has(entry.key)) continue; + this.publisher.publish(entry.event); + this.published.add(entry.key); + } + } + + private validate( + operation: ExecutionSemanticOperation, + receipt: Extract, + ) { + const entries = receipt.livePublication ?? []; + if (receipt.operationId !== operation.operationId || !operation.livePublication + || entries.length !== operation.livePublication.length || entries.length > MAX_RECEIPT_EVENTS + || Buffer.byteLength(JSON.stringify(entries), "utf8") > MAX_RECEIPT_BYTES) { + throw new Error("Live AgentEvent publication receipt is invalid"); + } + return entries.map((entry, index) => this.validateEntry(operation, entry, index)); + } + + private validateEntry(operation: ExecutionSemanticOperation, entry: ExecutionLivePublicationReceipt, index: number) { + const expected = operation.livePublication?.[index]; + if (!expected || !samePublicationHeader(operation, entry, expected.after, index)) { + throw new Error("Live AgentEvent publication receipt is invalid"); + } + const parsed = AgentEventSchema().safeParse(entry.event); + const original = AgentEventSchema().safeParse(expected.event); + if (!parsed.success || !original.success) throw new Error("Live AgentEvent publication receipt is invalid"); + if (parsed.data.threadId !== operation.execution.threadId + || parsed.data.turnExecutionId !== operation.execution.executionId + || !NodeUtil.isDeepStrictEqual(parsed.data, original.data)) { + throw new Error("Live AgentEvent publication receipt is invalid"); + } + return { key: JSON.stringify([operation.execution.executionId, entry.publicationId]), event: parsed.data }; + } +} + +function samePublicationHeader( + operation: ExecutionSemanticOperation, + entry: ExecutionLivePublicationReceipt, + expectedBarrier: "writer" | "terminal", + index: number, +): boolean { + return entry.publicationId === `${operation.operationId}:${index}` + && entry.after === expectedBarrier && entry.after === barrierFor(operation.mutation.kind); +} + +function barrierFor(kind: ExecutionSemanticOperation["mutation"]["kind"]): "writer" | "terminal" | null { + if (kind === "begin" || kind === "append-events") return "writer"; + return kind === "finish" ? "terminal" : null; +} diff --git a/apps/server/src/features/agents/execution/execution-worker-handler.ts b/apps/server/src/features/agents/execution/execution-worker-handler.ts index 3440ee7db..d6b8d4333 100644 --- a/apps/server/src/features/agents/execution/execution-worker-handler.ts +++ b/apps/server/src/features/agents/execution/execution-worker-handler.ts @@ -23,16 +23,16 @@ import type { ExecutionMailboxCommand } from "./execution-mailbox-scheduler.js"; /** Data that an execution worker may receive without a server dependency container. */ export type ExecutionWorkCommand = - | { readonly kind: "start"; readonly providerId: string; readonly input: DataOnlyParentTurnStartInput } + | { readonly kind: "start"; readonly providerId: string; readonly input: DataOnlyParentTurnStartInput; readonly livePublication?: readonly ExecutionLivePublicationIntent[] } | { readonly kind: "resume"; readonly providerId: string; readonly checkpointId: string } - | { readonly kind: "event"; readonly phase: string; readonly nativeCursor: unknown | null; readonly events: readonly ProviderEventDraft[] } + | { readonly kind: "event"; readonly phase: string; readonly nativeCursor: unknown | null; readonly events: readonly ProviderEventDraft[]; readonly livePublication?: readonly ExecutionLivePublicationIntent[] } | { readonly kind: "assistant-text"; readonly inputs: readonly ParentAssistantTextCheckpointInput[] } | { readonly kind: "narrative-delta"; readonly input: ParentNarrativeRecoveryCommit } | { readonly kind: "checkpoint"; readonly phase: string; readonly nativeCursor: unknown | null } | { readonly kind: "effect-result"; readonly effectId: string; readonly settled: boolean } | { readonly kind: "provider-outcome"; readonly outcome: TurnOutcome } | { readonly kind: "stage-terminal"; readonly input: DataOnlyParentTerminalProjectionInput } - | { readonly kind: "finalize"; readonly outcome: TurnOutcome; readonly input: DataOnlyParentTurnFinishInput } + | { readonly kind: "finalize"; readonly outcome: TurnOutcome; readonly input: DataOnlyParentTurnFinishInput; readonly livePublication?: readonly ExecutionLivePublicationIntent[] } | { readonly kind: "release" }; /** One complete semantic mutation. The single writer decides its SQL transaction. */ @@ -341,15 +341,26 @@ function operationFor( request: ExecutionWorkerRequest, mutation: ExecutionSemanticOperation["mutation"], ): ExecutionSemanticOperation { + const livePublication = livePublicationFor(request.command); return { operationId: operationId(request), execution: request.execution, lease: request.lease, ordinal: request.ordinal, mutation, + ...(livePublication ? { livePublication } : {}), }; } +function livePublicationFor(command: WorkerCommand): readonly ExecutionLivePublicationIntent[] | undefined { + switch (command.kind) { + case "start": + case "event": + case "finalize": return command.livePublication; + default: return undefined; + } +} + function operationId(request: ExecutionWorkerRequest): string { return `${request.lease.leaseId}:${request.ordinal}`; } From e9107f22d734b72a100b6bd89d84c8ee43910ee0 Mon Sep 17 00:00:00 2001 From: "(CJ) Chukwudi Nwobodo" <142016413+chuks-qua@users.noreply.github.com> Date: Thu, 24 Sep 2026 22:24:10 +0100 Subject: [PATCH 087/136] test(server): exercise live receipts through real execution workers --- .../execution-worker-loss-coordinator.test.ts | 96 +++++++++++++++++++ 1 file changed, 96 insertions(+) diff --git a/apps/server/src/features/agents/execution/__tests__/execution-worker-loss-coordinator.test.ts b/apps/server/src/features/agents/execution/__tests__/execution-worker-loss-coordinator.test.ts index 2ed8edb66..72b1b4f86 100644 --- a/apps/server/src/features/agents/execution/__tests__/execution-worker-loss-coordinator.test.ts +++ b/apps/server/src/features/agents/execution/__tests__/execution-worker-loss-coordinator.test.ts @@ -120,6 +120,102 @@ describe("ExecutionWorkerLossCoordinator with a file-backed writer", () => { return claim.lease; } + it("carries committed start, tool narrative, live publication, and finish through both workers", async () => { + const claim = coordinator.scheduler.claim(execution, 1); + if (claim.kind !== "claimed") throw new Error(`Claim failed: ${claim.kind}`); + const send = async (command: ExecutionWorkCommand) => { + const admitted = coordinator.scheduler.submit({ execution, lease: claim.lease, command, + byteLength: Buffer.byteLength(JSON.stringify(command), "utf8") }); + if (admitted.kind !== "admitted") throw new Error(`Command not admitted: ${admitted.kind}`); + const completion = await admitted.completion; + if (completion.kind !== "reply" || completion.result.kind !== "committed") { + throw new Error(`Command did not commit: ${JSON.stringify(completion)}`); + } + return completion.result; + }; + const started = { type: "turnStarted" as const, threadId: execution.threadId, + turnExecutionId: execution.executionId }; + const begin = await send({ + kind: "start", providerId: "codex", + input: { + thread: { id: execution.threadId, workspaceId: "lost-worker-workspace", providerId: "codex", createdAt: NOW }, + turnId: execution.turnId, executionId: execution.executionId, + permissionMode: "supervised", providerIdentities: [], + userMessage: { kind: "create", messageId: `${execution.threadId}-user`, content: "Question", sequence: 1 }, + }, + livePublication: [{ after: "writer", event: started }], + }); + expect(begin.livePublication).toEqual([{ + publicationId: `${claim.lease.leaseId}:1:0`, after: "writer", event: started, + }]); + expect(db.prepare("SELECT receipt_json FROM canonical_writer_operation_receipts WHERE execution_id = ? AND operation_id = ?") + .get(execution.executionId, begin.operationId)) + .toMatchObject({ receipt_json: expect.stringContaining(`${claim.lease.leaseId}:1:0`) }); + + const tool = { kind: "toolCall" as const, sequence: 2, sortOrder: 0, record: { + id: "transport-tool", message_id: "", parent_tool_call_id: null, + tool_name: "Read", input_summary: "CONTEXT.md", output_summary: "read", + status: "completed" as const, started_at: NOW, completed_at: NOW, sort_order: 0, + } }; + await send({ kind: "narrative-delta", input: { executionId: execution.executionId, items: [tool] } }); + expect(db.prepare("SELECT id FROM canonical_agent_items WHERE id = ?").get("toolCall:transport-tool")) + .toEqual({ id: "toolCall:transport-tool" }); + + const toolUse = { type: "toolUse" as const, threadId: execution.threadId, + turnExecutionId: execution.executionId, toolCallId: "transport-tool", toolName: "Read", + toolInput: { path: "CONTEXT.md" } }; + const event = await send({ kind: "event", phase: "running", nativeCursor: null, events: [{ + eventId: `${execution.executionId}:transport-tool`, + routing: { ...execution, itemId: "transport-tool" }, + sourceProviderId: "codex", sourceIdentities: [], sourceSequence: 1, + payload: { type: "item.recorded", item: { + id: "transport-tool", threadId: execution.threadId, turnId: execution.turnId, + kind: "tool-call", providerIdentities: [], + payload: { projection: "toolCall", toolName: "Read", path: "CONTEXT.md" }, + createdAt: NOW, updatedAt: NOW, + } }, + }], livePublication: [{ after: "writer", event: toolUse }] }); + expect(event.livePublication).toEqual([{ + publicationId: `${claim.lease.leaseId}:3:0`, after: "writer", event: toolUse, + }]); + expect(published).toContain(`${execution.executionId}:transport-tool`); + expect(db.prepare("SELECT receipt_json FROM canonical_writer_operation_receipts WHERE execution_id = ? AND operation_id = ?") + .get(execution.executionId, event.operationId)) + .toMatchObject({ receipt_json: expect.stringContaining(`${claim.lease.leaseId}:3:0`) }); + + await send({ kind: "provider-outcome", outcome: "completed" }); + await send({ kind: "stage-terminal", input: { + threadId: execution.threadId, executionId: execution.executionId, + outcome: "completed", endedAt: NOW, + assistant: { content: "Done", model: null, attachments: [] }, narrative: [tool], + } }); + const ended = { type: "ended" as const, threadId: execution.threadId, + turnExecutionId: execution.executionId, outcome: "completed" as const }; + const finish = await send({ kind: "finalize", outcome: "completed", input: { + ...execution, providerId: "codex", providerIdentities: [], outcome: "completed", + projection: { kind: "writer-staged" }, + }, livePublication: [{ after: "terminal", event: ended }] }); + expect(finish.livePublication).toEqual([{ + publicationId: `${claim.lease.leaseId}:6:0`, after: "terminal", event: ended, + }]); + expect(db.prepare("SELECT terminal_outcome FROM canonical_agent_ingest_checkpoints WHERE execution_id = ?") + .get(execution.executionId)).toEqual({ terminal_outcome: "completed" }); + expect(new MessageRepo(db).listIncludingInternal(execution.threadId)).toContainEqual(expect.objectContaining({ + role: "assistant", content: "Done", outcome: "completed", is_internal: false, + })); + expect(db.prepare("SELECT id, status FROM tool_call_records WHERE id = ?").get("transport-tool")) + .toEqual({ id: "transport-tool", status: "completed" }); + expect(published.indexOf(`${execution.executionId}:transport-tool`)) + .toBeLessThan(published.indexOf(`${execution.executionId}:turn.completed`)); + + workers[0]?.crash(); + expect(await coordinator.waitForRecovery(0)).toEqual({ kind: "recovered", workerIndex: 0 }); + expect(db.prepare("SELECT terminal_outcome FROM canonical_agent_ingest_checkpoints WHERE execution_id = ?") + .get(execution.executionId)).toEqual({ terminal_outcome: "completed" }); + expect(db.prepare("SELECT COUNT(*) AS count FROM messages WHERE thread_id = ? AND role = 'assistant' AND is_internal = 0") + .get(execution.threadId)).toEqual({ count: 1 }); + }); + it("interrupts a crashed worker before replacing its slot", async () => { const lease = await start(coordinator); new ParentAssistantTextCheckpointService(db).appendChunk([{ ...execution, sequence: 1, text: "Partial answer" }]); From c5263a52d319d7e98a4edf6a5f91d40d32987731 Mon Sep 17 00:00:00 2001 From: "(CJ) Chukwudi Nwobodo" <142016413+chuks-qua@users.noreply.github.com> Date: Thu, 24 Sep 2026 22:22:33 +0100 Subject: [PATCH 088/136] feat(server): persist Codex context projections in semantic writer --- ...anonical-execution-semantic-writer.test.ts | 90 ++++++++++++++++++- ...canonical-context-compaction-projection.ts | 56 ++++++++++++ .../canonical-execution-semantic-writer.ts | 10 ++- 3 files changed, 154 insertions(+), 2 deletions(-) create mode 100644 apps/server/src/features/agents/canonical/canonical-context-compaction-projection.ts diff --git a/apps/server/src/features/agents/canonical/__tests__/canonical-execution-semantic-writer.test.ts b/apps/server/src/features/agents/canonical/__tests__/canonical-execution-semantic-writer.test.ts index d27c13101..2d629073e 100644 --- a/apps/server/src/features/agents/canonical/__tests__/canonical-execution-semantic-writer.test.ts +++ b/apps/server/src/features/agents/canonical/__tests__/canonical-execution-semantic-writer.test.ts @@ -3,7 +3,7 @@ import * as NodeFS from "node:fs"; import * as NodeOS from "node:os"; import * as NodePath from "node:path"; import type { Database } from "bun:sqlite"; -import { AgentEventType } from "@mcode/contracts"; +import { AgentEventType, type AgentEvent } from "@mcode/contracts"; import { afterEach, beforeEach, describe, expect, it } from "vitest"; import { openDatabase } from "../../../../runtime/persistence/sqlite/database.js"; @@ -63,6 +63,25 @@ function event() { }; } +function runtimeEvent(sequence: number, agentEvent: AgentEvent): Extract< + ExecutionSemanticOperation["mutation"], { kind: "append-events" } +>["events"][number] { + const itemId = `runtime-${sequence}`; + return { + eventId: `${EXECUTION_ID}:${itemId}`, + routing: { ...execution, itemId }, + sourceProviderId: "codex", + sourceIdentities: [], + sourceSequence: sequence, + payload: { type: "item.recorded", item: { + id: itemId, threadId: THREAD_ID, turnId: TURN_ID, kind: "system", + providerIdentities: [], + payload: { projection: "providerRuntimeEvent", runtimeEvent: { event: agentEvent } }, + createdAt: NOW, updatedAt: NOW, + } }, + }; +} + function toolNarrative(messageId: string, count: number) { return Array.from({ length: count }, (_, index) => ({ kind: "toolCall" as const, @@ -122,6 +141,75 @@ describe("CanonicalExecutionSemanticWriter through ExecutionWorkerHandler", () = recoveryIncidentId: "incident-1", }; + it("commits context and compaction projections before publishing their canonical events", async () => { + const seenAtPublication: { eventId: string; state: unknown }[] = []; + writer = new CanonicalExecutionSemanticWriter(db, (events) => { + for (const item of events) { + const state = db.prepare("SELECT last_context_tokens, last_compact_summary FROM threads WHERE id = ?").get(THREAD_ID); + seenAtPublication.push({ eventId: item.eventId, state }); + } + }); + handler = new ExecutionWorkerHandler(writer); + expect((await send(1, { kind: "start", providerId: "codex", input: startInput() })).kind).toBe("committed"); + async function append(ordinal: number, agentEvent: AgentEvent) { + return send(ordinal, { kind: "event", phase: "running", nativeCursor: null, events: [runtimeEvent(ordinal, agentEvent)] }); + } + const identity = { threadId: THREAD_ID, turnExecutionId: EXECUTION_ID }; + expect((await append(2, { ...identity, type: "contextEstimate", tokensIn: 100, totalProcessedTokens: 120, contextWindow: 200_000 })).kind).toBe("committed"); + expect((await append(3, { ...identity, type: "compacting", active: true })).kind).toBe("committed"); + expect((await append(4, { ...identity, type: "contextEstimate", tokensIn: 90, totalProcessedTokens: 140 })).kind).toBe("committed"); + expect((await append(5, { ...identity, type: "compactSummary", summary: "Short context" })).kind).toBe("committed"); + expect((await append(6, { ...identity, type: "contextEstimate", tokensIn: 30, totalProcessedTokens: 150 })).kind).toBe("committed"); + expect((await append(7, { ...identity, type: "compacting", active: true })).kind).toBe("committed"); + expect((await append(8, { ...identity, type: "compacting", active: false })).kind).toBe("committed"); + expect((await append(9, { ...identity, type: "turnComplete", reason: "end_turn", costUsd: null, tokensIn: 40, tokensOut: 5 })).kind).toBe("committed"); + + expect(seenAtPublication.filter(({ eventId }) => eventId.includes(":runtime-"))).toEqual([ + { eventId: `${EXECUTION_ID}:runtime-2`, state: { last_context_tokens: 100, last_compact_summary: null } }, + { eventId: `${EXECUTION_ID}:runtime-3`, state: { last_context_tokens: 100, last_compact_summary: null } }, + { eventId: `${EXECUTION_ID}:runtime-4`, state: { last_context_tokens: 100, last_compact_summary: null } }, + { eventId: `${EXECUTION_ID}:runtime-5`, state: { last_context_tokens: 100, last_compact_summary: "Short context" } }, + { eventId: `${EXECUTION_ID}:runtime-6`, state: { last_context_tokens: 30, last_compact_summary: "Short context" } }, + { eventId: `${EXECUTION_ID}:runtime-7`, state: { last_context_tokens: 30, last_compact_summary: "Short context" } }, + { eventId: `${EXECUTION_ID}:runtime-8`, state: { last_context_tokens: 30, last_compact_summary: "Short context" } }, + { eventId: `${EXECUTION_ID}:runtime-9`, state: { last_context_tokens: 40, last_compact_summary: "Short context" } }, + ]); + expect(new MessageRepo(db).listIncludingInternal(THREAD_ID)).toContainEqual(expect.objectContaining({ role: "system", content: "Context compacted" })); + + db.close(true); + db = openDatabase({ dbPath: path }); + writer = new CanonicalExecutionSemanticWriter(db, () => {}); + const dividerCount = db.prepare("SELECT COUNT(*) AS count FROM messages WHERE thread_id = ? AND content = 'Context compacted'") + .get(THREAD_ID); + expect(await writer.transact(operation(8, { kind: "append-events", phase: "running", nativeCursor: null, + events: [runtimeEvent(8, { ...identity, type: "compacting", active: false })] }))).toMatchObject({ kind: "committed" }); + expect(db.prepare("SELECT COUNT(*) AS count FROM messages WHERE thread_id = ? AND content = 'Context compacted'") + .get(THREAD_ID)).toEqual(dividerCount); + }); + + it("rolls back context projections and rejects completion during compaction", async () => { + expect((await send(1, { kind: "start", providerId: "codex", input: startInput() })).kind).toBe("committed"); + const identity = { threadId: THREAD_ID, turnExecutionId: EXECUTION_ID }; + db.run("CREATE TRIGGER fail_context_receipt BEFORE INSERT ON canonical_writer_operation_receipts WHEN NEW.kind = 'semantic:append-events' BEGIN SELECT RAISE(ABORT, 'receipt unavailable'); END"); + await expect(send(2, { kind: "event", phase: "running", nativeCursor: null, events: [ + runtimeEvent(2, { ...identity, type: "contextEstimate", tokensIn: 80, totalProcessedTokens: 90 }), + ] })).rejects.toThrow("receipt unavailable"); + expect(db.prepare("SELECT last_context_tokens FROM threads WHERE id = ?").get(THREAD_ID)).toEqual({ last_context_tokens: null }); + expect(published).not.toContain(`${EXECUTION_ID}:runtime-2`); + db.run("DROP TRIGGER fail_context_receipt"); + expect((await send(2, { kind: "event", phase: "running", nativeCursor: null, events: [ + runtimeEvent(2, { ...identity, type: "compacting", active: true }), + ] })).kind).toBe("committed"); + expect(await send(3, { kind: "event", phase: "running", nativeCursor: null, events: [ + runtimeEvent(3, { ...identity, type: "turnComplete", reason: "end_turn", costUsd: null, tokensIn: 50, tokensOut: 1 }), + ] })).toEqual({ kind: "rejected", reason: "writer-conflict" }); + expect(db.prepare("SELECT event_id FROM canonical_agent_events WHERE event_id = ?").get(`${EXECUTION_ID}:runtime-3`)).toBeNull(); + expect(published).not.toContain(`${EXECUTION_ID}:runtime-3`); + expect((await send(3, { kind: "event", phase: "running", nativeCursor: null, events: [ + runtimeEvent(3, { ...identity, type: "compactSummary", summary: "Recovered" }), + ] })).kind).toBe("committed"); + }); + it("fences a lost worker while retaining durable text and narrative", async () => { expect((await send(1, { kind: "start", providerId: "codex", input: startInput() })).kind).toBe("committed"); const text = new ParentAssistantTextCheckpointService(db); diff --git a/apps/server/src/features/agents/canonical/canonical-context-compaction-projection.ts b/apps/server/src/features/agents/canonical/canonical-context-compaction-projection.ts new file mode 100644 index 000000000..888521e14 --- /dev/null +++ b/apps/server/src/features/agents/canonical/canonical-context-compaction-projection.ts @@ -0,0 +1,56 @@ +import type { Database } from "bun:sqlite"; + +import { ThreadRepo } from "../../thread-control/persistence/thread-repo.js"; +import { MessageRepo } from "../conversation/persistence/message-repo.js"; +import type { ProjectedCommittedProviderEvent } from "../execution/execution-worker-handler.js"; + +/** Applies Codex context and compaction effects on the canonical writer connection. */ +export class CanonicalContextCompactionProjection { + private readonly threads: ThreadRepo; + private readonly messages: MessageRepo; + + constructor(db: Database) { + this.threads = new ThreadRepo(db); + this.messages = new MessageRepo(db); + } + + /** Return the compaction state after ordered events; null means an unsupported terminal arrived. */ + apply(threadId: string, compacting: boolean, events: readonly ProjectedCommittedProviderEvent[]): boolean | null { + let active = compacting; + for (const { providerId, event } of events) { + if (providerId !== "codex") continue; + const next = this.applyEvent(threadId, active, event); + if (next === null) return null; + active = next; + } + return active; + } + + private applyEvent(threadId: string, active: boolean, event: ProjectedCommittedProviderEvent["event"]): boolean | null { + switch (event.type) { + case "contextEstimate": + if (event.totalProcessedTokens !== undefined && !active) this.recordUsage(threadId, event.tokensIn, event.contextWindow); + return active; + case "turnComplete": + if (active) return null; + this.recordUsage(threadId, event.tokensIn, event.contextWindow); + return active; + case "compacting": + if (!event.active) this.persistDivider(threadId); + return event.active; + case "compactSummary": + this.threads.updateCompactSummary(threadId, event.summary); + return false; + default: return active; + } + } + + private recordUsage(threadId: string, tokensIn: number, contextWindow?: number): void { + if (tokensIn > 0) this.threads.updateContextUsage(threadId, tokensIn, contextWindow); + } + + private persistDivider(threadId: string): void { + const sequence = this.messages.getLatestSequenceIncludingInternal(threadId) + 1; + this.messages.create(threadId, "system", "Context compacted", sequence); + } +} diff --git a/apps/server/src/features/agents/canonical/canonical-execution-semantic-writer.ts b/apps/server/src/features/agents/canonical/canonical-execution-semantic-writer.ts index 9d1f3f520..faa933085 100644 --- a/apps/server/src/features/agents/canonical/canonical-execution-semantic-writer.ts +++ b/apps/server/src/features/agents/canonical/canonical-execution-semantic-writer.ts @@ -24,6 +24,7 @@ import type { import type { CanonicalAgentEventPublisher } from "./canonical-agent-boundary.js"; import { CanonicalAgentBoundary } from "./canonical-agent-boundary.js"; import { CanonicalCommittedProviderProjector } from "./canonical-committed-provider-projector.js"; +import { CanonicalContextCompactionProjection } from "./canonical-context-compaction-projection.js"; import { CanonicalParentTurnWrite } from "./canonical-parent-turn-write.js"; import { ParentAssistantTextCheckpointService, @@ -80,6 +81,7 @@ const storedHeadSchema = z.object({ stopWatermark: z.number().int().nullable(), providerOutcome: TurnOutcomeSchema.nullable(), assignedMessageId: z.string().regex(/^[0-9a-f]{64}$/).nullable().optional(), + compacting: z.boolean().default(false), }); const storedReceiptSchema = z.object({ kind: z.literal("committed"), @@ -128,6 +130,7 @@ interface SemanticHead { readonly stopWatermark: number | null; readonly providerOutcome: TurnOutcome | null; readonly assignedMessageId?: string | null; + readonly compacting: boolean; } type StoredOperation = z.infer; @@ -146,6 +149,7 @@ export interface LostExecutionInterruption { export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter { private readonly turns: CanonicalParentTurnWrite; private readonly providerProjector: CanonicalCommittedProviderProjector; + private readonly contextCompaction: CanonicalContextCompactionProjection; private readonly assistantText: ParentAssistantTextCheckpointService; private readonly canonical: CanonicalAgentBoundary; private readonly findOperation: ReturnType; @@ -171,6 +175,7 @@ export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter }); this.canonical = codexBoundary; this.providerProjector = new CanonicalCommittedProviderProjector(codexBoundary); + this.contextCompaction = new CanonicalContextCompactionProjection(db); this.assistantText = new ParentAssistantTextCheckpointService(db); this.findOperation = db.prepare("SELECT kind, input_hash, receipt_json FROM canonical_writer_operation_receipts WHERE execution_id = ? AND operation_id = ?"); this.listPublicationChunks = db.prepare("SELECT operation_id, kind, input_hash, receipt_json FROM canonical_writer_operation_receipts WHERE execution_id = ? AND operation_id > ? AND operation_id < ? ORDER BY operation_id LIMIT ?"); @@ -295,6 +300,7 @@ export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter stopRequestId: null, stopWatermark: null, providerOutcome: null, + compacting: false, }; this.insertOperation.run(operation.execution.executionId, HEAD_ID, HEAD_KIND, fingerprint(head.lease), JSON.stringify(head)); this.storePublicationChunks(operation.execution.executionId, hash, result.events.map((event) => event.acceptedSequence)); @@ -321,6 +327,8 @@ export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter }); if (result.outcome !== "committed") throw new SemanticConflict(); const providerEvents = this.providerProjector.project(result.events); + const compacting = this.contextCompaction.apply(operation.execution.threadId, head.compacting, providerEvents); + if (compacting === null) throw new SemanticConflict(); const checkpoint = durableSequenceRowSchema.parse(this.findDurableSequence.get(operation.execution.executionId)); const providerCommit: ExecutionProviderCommitReceipt = { outcome: result.outcome, @@ -331,7 +339,7 @@ export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter eventCount: result.events.length, }; const receipt = committed(operation, checkpoint.last_durable_sequence, providerCommit, providerEvents); - this.storeHead({ ...head, ordinal: operation.ordinal, durableRevision: receipt.durableRevision }); + this.storeHead({ ...head, ordinal: operation.ordinal, durableRevision: receipt.durableRevision, compacting }); const publication = this.bufferedPublication?.flat() ?? []; // A Codex child write has its own execution sequence, even when the parent event caused it. this.storePublicationChunks(operation.execution.executionId, hash, publication.map((event) => ({ From 0684444779c86e5ce06661f3c4fd124559db3baf Mon Sep 17 00:00:00 2001 From: "(CJ) Chukwudi Nwobodo" <142016413+chuks-qua@users.noreply.github.com> Date: Thu, 24 Sep 2026 22:23:37 +0100 Subject: [PATCH 089/136] feat(server): commit compound live events through execution handler --- .../execution-worker-handler.test.ts | 75 +++++++++++++++++ .../execution/execution-worker-handler.ts | 83 +++++++++++++++++-- 2 files changed, 150 insertions(+), 8 deletions(-) diff --git a/apps/server/src/features/agents/execution/__tests__/execution-worker-handler.test.ts b/apps/server/src/features/agents/execution/__tests__/execution-worker-handler.test.ts index 8a89f64a8..bca31588e 100644 --- a/apps/server/src/features/agents/execution/__tests__/execution-worker-handler.test.ts +++ b/apps/server/src/features/agents/execution/__tests__/execution-worker-handler.test.ts @@ -21,6 +21,7 @@ import type { } from "../execution-mailbox-protocol.js"; import { ExecutionWorkerHandler, + type ExecutionLivePublicationIntent, type ExecutionSemanticOperation, type ExecutionSemanticWriter, type ExecutionWorkCommand, @@ -66,6 +67,17 @@ const NARRATIVE_INPUT: ParentNarrativeRecoveryCommit = { }], }; +const PUBLICATION = { + after: "writer", + event: { type: "turnStarted", threadId: EXECUTION.threadId }, +} satisfies ExecutionLivePublicationIntent; + +const TEXT_INPUT = { + ...EXECUTION, + sequence: 1, + text: "Hello", +}; + const LIMITS: ExecutionMailboxLimits = { maxPending: 12, maxPendingBytes: 12_000, @@ -163,6 +175,69 @@ async function committed(admission: ReturnType, revision: number) } describe("ExecutionWorkerHandler through its scheduler", () => { + it("commits one compound live event and forwards its publication receipt", async () => { + class PublicationWriter extends RecordingWriter { + override async transact(operation: ExecutionSemanticOperation): Promise { + const receipt = await super.transact(operation); + return receipt.kind === "committed" && operation.livePublication + ? { ...receipt, livePublication: operation.livePublication.map((intent) => ({ + ...intent, publicationId: `${operation.operationId}:0`, + })) } + : receipt; + } + } + + const { scheduler, writer, lease } = fixture(new PublicationWriter()); + await committed(submit(scheduler, lease, { kind: "start", providerId: "codex", input: START_INPUT }), 1); + const admission = submit(scheduler, lease, { + kind: "live-event", text: { kind: "append", inputs: [TEXT_INPUT] }, + narrative: NARRATIVE_INPUT, publication: PUBLICATION, + }); + await expect(admission.completion).resolves.toMatchObject({ + kind: "reply", + result: { + kind: "committed", operationId: `${lease.leaseId}:2`, durableRevision: 2, + livePublication: [{ ...PUBLICATION, publicationId: `${lease.leaseId}:2:0` }], + }, + }); + expect(writer.operations).toHaveLength(2); + expect(writer.operations[1]).toEqual({ + operationId: `${lease.leaseId}:2`, execution: EXECUTION, lease, ordinal: 2, + mutation: { + kind: "live-event", text: { kind: "append", inputs: [TEXT_INPUT] }, narrative: NARRATIVE_INPUT, + }, + livePublication: [PUBLICATION], + }); + scheduler.shutdown(); + }); + + it("rejects live-event text or narrative for a different execution before writing", async () => { + const { scheduler, handler, writer, lease } = fixture(); + await committed(submit(scheduler, lease, { kind: "start", providerId: "codex", input: START_INPUT }), 1); + + for (const command of [ + { kind: "live-event", text: { kind: "append", inputs: [{ ...TEXT_INPUT, turnId: "other-turn" }] }, publication: PUBLICATION }, + { kind: "live-event", text: { kind: "promote", input: { ...TEXT_INPUT, executionId: "other-execution" } }, publication: PUBLICATION }, + ] as const) { + await expect(handler.handle({ + requestId: 2, execution: EXECUTION, lease, ordinal: 2, command, + })).resolves.toMatchObject({ + result: { kind: "rejected", reason: "invalid-text-routing" }, + }); + } + await expect(handler.handle({ + requestId: 2, execution: EXECUTION, lease, ordinal: 2, + command: { + kind: "live-event", text: { kind: "unchanged" }, + narrative: { ...NARRATIVE_INPUT, executionId: "other-execution" }, publication: PUBLICATION, + }, + })).resolves.toMatchObject({ + result: { kind: "rejected", reason: "invalid-narrative-routing" }, + }); + expect(writer.operations.map((operation) => operation.mutation.kind)).toEqual(["begin"]); + scheduler.shutdown(); + }); + it("maps a running narrative delta to one fenced semantic operation", async () => { const { scheduler, writer, lease } = fixture(); await committed(submit(scheduler, lease, { kind: "start", providerId: "codex", input: START_INPUT }), 1); diff --git a/apps/server/src/features/agents/execution/execution-worker-handler.ts b/apps/server/src/features/agents/execution/execution-worker-handler.ts index d6b8d4333..7f17534c3 100644 --- a/apps/server/src/features/agents/execution/execution-worker-handler.ts +++ b/apps/server/src/features/agents/execution/execution-worker-handler.ts @@ -28,6 +28,16 @@ export type ExecutionWorkCommand = | { readonly kind: "event"; readonly phase: string; readonly nativeCursor: unknown | null; readonly events: readonly ProviderEventDraft[]; readonly livePublication?: readonly ExecutionLivePublicationIntent[] } | { readonly kind: "assistant-text"; readonly inputs: readonly ParentAssistantTextCheckpointInput[] } | { readonly kind: "narrative-delta"; readonly input: ParentNarrativeRecoveryCommit } + | { + readonly kind: "live-event"; + readonly text: + | { readonly kind: "unchanged" } + | { readonly kind: "append"; readonly inputs: readonly ParentAssistantTextCheckpointInput[] } + | { readonly kind: "reclassify"; readonly expectedText: string } + | { readonly kind: "promote"; readonly input: ParentAssistantTextCheckpointInput }; + readonly narrative?: ParentNarrativeRecoveryCommit; + readonly publication: ExecutionLivePublicationIntent; + } | { readonly kind: "checkpoint"; readonly phase: string; readonly nativeCursor: unknown | null } | { readonly kind: "effect-result"; readonly effectId: string; readonly settled: boolean } | { readonly kind: "provider-outcome"; readonly outcome: TurnOutcome } @@ -49,6 +59,11 @@ export interface ExecutionSemanticOperation { | { readonly kind: "append-events"; readonly phase: string; readonly nativeCursor: unknown | null; readonly events: readonly ProviderEventDraft[] } | { readonly kind: "append-assistant-text"; readonly inputs: readonly ParentAssistantTextCheckpointInput[] } | { readonly kind: "narrative-delta"; readonly input: ParentNarrativeRecoveryCommit } + | { + readonly kind: "live-event"; + readonly text: Extract["text"]; + readonly narrative?: ParentNarrativeRecoveryCommit; + } | { readonly kind: "checkpoint"; readonly phase: string; readonly nativeCursor: unknown | null } | { readonly kind: "stop-requested"; readonly requestId: string; readonly lastAdmittedOrdinal: number } | { readonly kind: "effect-result"; readonly effectId: string; readonly settled: boolean } @@ -216,6 +231,8 @@ function mutationFor( return eventMutation(command, request.execution, state); case "narrative-delta": return narrativeMutation(command, request.execution, state); + case "live-event": + return liveEventMutation(command, request.execution, state); case "stop": return stopMutation(command, request, state); case "finalize": @@ -263,6 +280,18 @@ function narrativeMutation( return { kind: "narrative-delta", input: command.input }; } +function liveEventMutation( + command: Extract, + execution: ExecutionIdentity, + state: ExecutionState, +): ExecutionSemanticOperation["mutation"] | undefined { + if (state.phase !== "running" || !validLiveEventRouting(command, execution)) return undefined; + return { + kind: "live-event", text: command.text, + ...(command.narrative ? { narrative: command.narrative } : {}), + }; +} + function stopMutation( command: Extract, request: ExecutionWorkerRequest, @@ -292,16 +321,35 @@ function commandRejection( function invalidReason(request: ExecutionWorkerRequest): Extract["reason"] { if (request.command.kind === "stop" && request.stopWatermark !== request.ordinal - 1) return "invalid-stop-watermark"; - if (request.command.kind === "event" && !validEventRouting(request.command.events, request.execution)) { - return "invalid-event-routing"; + if (request.command.kind === "live-event") { + return invalidLiveEventReason(request.command, request.execution) ?? "invalid-transition"; } - if (request.command.kind === "assistant-text" && !validTextRouting(request.command.inputs, request.execution)) { - return "invalid-text-routing"; - } - if (request.command.kind === "narrative-delta" && request.command.input?.executionId !== request.execution.executionId) { - return "invalid-narrative-routing"; + return invalidRoutingReason(request.command, request.execution) ?? "invalid-transition"; +} + +function invalidRoutingReason( + command: Exclude, + execution: ExecutionIdentity, +): "invalid-event-routing" | "invalid-text-routing" | "invalid-narrative-routing" | null { + switch (command.kind) { + case "event": + return validEventRouting(command.events, execution) ? null : "invalid-event-routing"; + case "assistant-text": + return validTextRouting(command.inputs, execution) ? null : "invalid-text-routing"; + case "narrative-delta": + return command.input?.executionId === execution.executionId ? null : "invalid-narrative-routing"; + default: + return null; } - return "invalid-transition"; +} + +function invalidLiveEventReason( + command: Extract, + execution: ExecutionIdentity, +): "invalid-text-routing" | "invalid-narrative-routing" | null { + if (validLiveEventRouting(command, execution)) return null; + return command.narrative !== undefined && command.narrative?.executionId !== execution.executionId + ? "invalid-narrative-routing" : "invalid-text-routing"; } function validEventRouting(events: readonly ProviderEventDraft[], execution: ExecutionIdentity): boolean { @@ -315,6 +363,24 @@ function validTextRouting(inputs: readonly ParentAssistantTextCheckpointInput[], && input.turnId === execution.turnId && input.executionId === execution.executionId); } +function validLiveEventRouting( + command: Extract, + execution: ExecutionIdentity, +): boolean { + if (command.narrative !== undefined && command.narrative?.executionId !== execution.executionId) return false; + switch (command.text?.kind) { + case "unchanged": + case "reclassify": + return true; + case "append": + return validTextRouting(command.text.inputs, execution); + case "promote": + return validTextRouting([command.text.input], execution); + default: + return false; + } +} + function validStartInput( command: Extract, execution: ExecutionIdentity, @@ -357,6 +423,7 @@ function livePublicationFor(command: WorkerCommand): readonly ExecutionLivePubli case "start": case "event": case "finalize": return command.livePublication; + case "live-event": return [command.publication]; default: return undefined; } } From 4e19561838205ff2c5b84cd59abd499fa6d9ab32 Mon Sep 17 00:00:00 2001 From: "(CJ) Chukwudi Nwobodo" <142016413+chuks-qua@users.noreply.github.com> Date: Thu, 24 Sep 2026 22:26:56 +0100 Subject: [PATCH 090/136] feat(server): commit live text and narrative as one operation --- .../canonical-agent-writer-client.test.ts | 60 +++++++ ...anonical-execution-semantic-writer.test.ts | 84 ++++++++++ .../canonical-execution-semantic-writer.ts | 156 ++++++++++++++++-- 3 files changed, 290 insertions(+), 10 deletions(-) diff --git a/apps/server/src/features/agents/canonical/__tests__/canonical-agent-writer-client.test.ts b/apps/server/src/features/agents/canonical/__tests__/canonical-agent-writer-client.test.ts index 373080148..94ad3678a 100644 --- a/apps/server/src/features/agents/canonical/__tests__/canonical-agent-writer-client.test.ts +++ b/apps/server/src/features/agents/canonical/__tests__/canonical-agent-writer-client.test.ts @@ -297,6 +297,66 @@ describe("canonical SQLite writer", () => { .get("writer-recovery-tool")).toEqual({ id: "writer-recovery-tool", status: "completed" }); }); + it("replays one live reclassification receipt after the worker loses its reply", async () => { + const execution = { threadId: THREAD_ID, turnId: TURN_ID, executionId: EXECUTION_ID }; + const lease = { ownerEpoch: 1, workerIndex: 0, workerGeneration: 1, leaseId: "live-text-lease" }; + const begin: ExecutionSemanticOperation = { + operationId: "live-text-lease:1", execution, lease, ordinal: 1, + mutation: { kind: "begin", providerId: "codex", input: { + thread: { id: THREAD_ID, workspaceId: "writer-workspace", providerId: "codex", createdAt: NOW }, + turnId: TURN_ID, executionId: EXECUTION_ID, permissionMode: "supervised", providerIdentities: [], + userMessage: { kind: "create", messageId: "live-text-user", content: "Question", sequence: 1 }, + } }, + }; + writer = new CanonicalAgentWriterClient(dbPath); + expect((await writer.transactSemantic(begin, () => {})).kind).toBe("committed"); + const provisional: ExecutionSemanticOperation = { + operationId: "live-text-lease:2", execution, lease, ordinal: 2, + mutation: { kind: "live-event", text: { kind: "append", inputs: [{ + ...execution, sequence: 1, text: "provisional thought", + }] } }, + livePublication: [{ after: "writer", event: { + type: AgentEventType.TextDelta, threadId: THREAD_ID, + turnExecutionId: EXECUTION_ID, delta: "provisional thought", + } }], + }; + expect((await writer.transactSemantic(provisional, () => {})).kind).toBe("committed"); + await writer.close(); + + const thought = { kind: "narrationSegment" as const, record: { + id: "live-thought", message_id: "", text: "provisional thought", + started_at: NOW, ended_at: NOW, sort_order: 0, + } }; + const reclassified: ExecutionSemanticOperation = { + operationId: "live-text-lease:3", execution, lease, ordinal: 3, + mutation: { kind: "live-event", text: { kind: "reclassify", expectedText: "provisional thought" }, + narrative: { executionId: EXECUTION_ID, items: [thought], discardedItemIds: [] } }, + livePublication: [{ after: "writer", event: { + type: AgentEventType.AssistantMessageBoundary, threadId: THREAD_ID, + turnExecutionId: EXECUTION_ID, isFinalResponse: false, + } }], + }; + let created = 0; + writer = new CanonicalAgentWriterClient(dbPath, () => created++ === 0 + ? workerDroppingReply("semantic-transacted") + : new Worker(new URL("../canonical-agent-writer.worker.ts", import.meta.url), { type: "module" })); + const receipt = await writer.transactSemantic(reclassified, () => {}); + expect(receipt).toMatchObject({ kind: "committed", operationId: "live-text-lease:3", + livePublication: [{ publicationId: "live-text-lease:3:0", after: "writer" }] }); + expect(created).toBe(2); + expect(new ParentAssistantTextCheckpointService(db).restore(EXECUTION_ID)).toBe(""); + expect(db.prepare("SELECT COUNT(*) AS count FROM canonical_agent_items WHERE id = ?") + .get("narrationSegment:live-thought")).toEqual({ count: 1 }); + await writer.close(); + + writer = new CanonicalAgentWriterClient(dbPath); + expect(await writer.transactSemantic(reclassified, () => {})).toEqual(receipt); + expect((await writer.interruptWorkerLoss({ execution, lease, reason: "worker exited", + recoveryIncidentId: "live-thought-loss" }, () => {})).kind).toBe("committed"); + expect(db.prepare("SELECT id, text FROM thought_segments WHERE id = ?").get("live-thought")) + .toEqual({ id: "live-thought", text: "provisional thought" }); + }); + it("rejects a database failure without reporting a durable receipt", async () => { writer = new CanonicalAgentWriterClient(dbPath); await writer.whenReady(); diff --git a/apps/server/src/features/agents/canonical/__tests__/canonical-execution-semantic-writer.test.ts b/apps/server/src/features/agents/canonical/__tests__/canonical-execution-semantic-writer.test.ts index 2d629073e..46b65c9a5 100644 --- a/apps/server/src/features/agents/canonical/__tests__/canonical-execution-semantic-writer.test.ts +++ b/apps/server/src/features/agents/canonical/__tests__/canonical-execution-semantic-writer.test.ts @@ -437,6 +437,90 @@ describe("CanonicalExecutionSemanticWriter through ExecutionWorkerHandler", () = expect((await writer.transact(delta)).kind).toBe("committed"); }); + it("commits reclassification and promotion with their narrative changes before live publication", async () => { + expect((await send(1, { kind: "start", providerId: "codex", input: startInput() })).kind).toBe("committed"); + const publicationCount = published.length; + const unknown = operation(2, { kind: "live-event", text: { kind: "append", inputs: [ + { ...execution, sequence: 1, text: "unknown draft" }, + ] } }); + const unknownEvent = { type: AgentEventType.TextDelta, threadId: THREAD_ID, + turnExecutionId: EXECUTION_ID, delta: "unknown draft" }; + const appended = await writer.transact({ ...unknown, livePublication: [{ after: "writer", event: unknownEvent }] }); + expect(appended).toMatchObject({ kind: "committed", assistantTextCheckpoint: { durableThrough: 1 }, + livePublication: [{ publicationId: "lease-1:2:0", event: unknownEvent }] }); + expect(new ParentAssistantTextCheckpointService(db).restore(EXECUTION_ID)).toBe("unknown draft"); + + const thought = { kind: "narrationSegment" as const, record: { + id: "thought-1", message_id: "", text: "unknown draft", started_at: NOW, + ended_at: NOW, sort_order: 0, + } }; + const boundary = { type: AgentEventType.AssistantMessageBoundary, threadId: THREAD_ID, + turnExecutionId: EXECUTION_ID, isFinalResponse: false }; + const reclassified = { ...operation(3, { kind: "live-event", text: { + kind: "reclassify", expectedText: "unknown draft", + }, narrative: { executionId: EXECUTION_ID, items: [thought], discardedItemIds: [] } }), + livePublication: [{ after: "writer" as const, event: boundary }] }; + db.run("CREATE TRIGGER fail_compound_receipt BEFORE INSERT ON canonical_writer_operation_receipts WHEN NEW.kind = 'semantic:live-event' AND NEW.operation_id = 'lease-1:3' BEGIN SELECT RAISE(ABORT, 'compound receipt unavailable'); END"); + await expect(writer.transact(reclassified)).rejects.toThrow("compound receipt unavailable"); + expect(new ParentAssistantTextCheckpointService(db).restore(EXECUTION_ID)).toBe("unknown draft"); + expect(new CanonicalAgentBoundary(db, () => {}).loadParentNarrativeRecovery(TURN_ID)).toEqual([]); + expect(published).toHaveLength(publicationCount); + db.run("DROP TRIGGER fail_compound_receipt"); + const reclassifiedReceipt = await writer.transact(reclassified); + expect(reclassifiedReceipt).toMatchObject({ kind: "committed", livePublication: [ + { publicationId: "lease-1:3:0", event: boundary }, + ] }); + expect(new ParentAssistantTextCheckpointService(db).restore(EXECUTION_ID)).toBe(""); + expect(new CanonicalAgentBoundary(db, () => {}).loadParentNarrativeRecovery(TURN_ID)).toEqual([thought]); + + db.close(true); + db = openDatabase({ dbPath: path }); + writer = new CanonicalExecutionSemanticWriter(db, () => {}); + expect(await writer.transact(reclassified)).toEqual(reclassifiedReceipt); + expect(new ParentAssistantTextCheckpointService(db).restore(EXECUTION_ID)).toBe(""); + + const finalBoundary = { ...boundary, isFinalResponse: true }; + const promoted = { ...operation(4, { kind: "live-event", text: { kind: "promote", input: { + ...execution, sequence: 1, text: "final thought", + } }, narrative: { executionId: EXECUTION_ID, items: [], discardedItemIds: ["narrationSegment:thought-1"] } }), + livePublication: [{ after: "writer" as const, event: finalBoundary }] }; + expect(await writer.transact(promoted)).toMatchObject({ kind: "committed", + assistantTextCheckpoint: { durableThrough: 1 }, + livePublication: [{ publicationId: "lease-1:4:0", event: finalBoundary }], + }); + expect(new ParentAssistantTextCheckpointService(db).restore(EXECUTION_ID)).toBe("final thought"); + expect(new CanonicalAgentBoundary(db, () => {}).loadParentNarrativeRecovery(TURN_ID)).toEqual([]); + expect(writer.interruptWorkerLoss(loss).kind).toBe("committed"); + expect(new MessageRepo(db).listIncludingInternal(THREAD_ID)).toContainEqual(expect.objectContaining({ + role: "assistant", content: "final thought", outcome: "interrupted", + })); + }); + + it("rejects a compound live event that exceeds its shared row or byte budget", async () => { + expect((await send(1, { kind: "start", providerId: "codex", input: startInput() })).kind).toBe("committed"); + const event = { type: AgentEventType.TextDelta, threadId: THREAD_ID, + turnExecutionId: EXECUTION_ID, delta: "short text" }; + const text = { kind: "append" as const, inputs: [{ ...execution, sequence: 1, text: event.delta }] }; + const publication = [{ after: "writer" as const, event }]; + expect(await writer.transact({ ...operation(2, { kind: "live-event", text, + narrative: { executionId: EXECUTION_ID, items: toolNarrative("", 62), discardedItemIds: [] }, + }), livePublication: publication })).toEqual({ kind: "conflict", operationId: "lease-1:2" }); + const largeItem = { ...toolNarrative("", 1)[0]!, record: { + ...toolNarrative("", 1)[0]!.record, input_summary: "x".repeat(256 * 1024 - 900), + } }; + expect(await writer.transact({ ...operation(2, { kind: "live-event", text, + narrative: { executionId: EXECUTION_ID, items: [largeItem], discardedItemIds: [] }, + }), livePublication: publication })).toEqual({ kind: "conflict", operationId: "lease-1:2" }); + expect(await writer.transact({ ...operation(2, { kind: "live-event", text }), + livePublication: [{ after: "writer", event: { ...event, delta: "different" } }], + })).toEqual({ kind: "conflict", operationId: "lease-1:2" }); + expect(new ParentAssistantTextCheckpointService(db).restore(EXECUTION_ID)).toBe(""); + expect(db.prepare("SELECT COUNT(*) AS count FROM canonical_agent_items WHERE turn_id = ? AND id LIKE 'toolCall:%'") + .get(TURN_ID)).toEqual({ count: 0 }); + expect(db.prepare("SELECT COUNT(*) AS count FROM canonical_writer_operation_receipts WHERE execution_id = ? AND operation_id = ?") + .get(EXECUTION_ID, "lease-1:2")).toEqual({ count: 0 }); + }); + it("rejects stale worker-loss leases without changing the unfinished turn", async () => { expect((await send(1, { kind: "start", providerId: "codex", input: startInput() })).kind).toBe("committed"); expect(writer.interruptWorkerLoss({ ...loss, lease: { ...lease, ownerEpoch: 2 } })) diff --git a/apps/server/src/features/agents/canonical/canonical-execution-semantic-writer.ts b/apps/server/src/features/agents/canonical/canonical-execution-semantic-writer.ts index faa933085..494f97730 100644 --- a/apps/server/src/features/agents/canonical/canonical-execution-semantic-writer.ts +++ b/apps/server/src/features/agents/canonical/canonical-execution-semantic-writer.ts @@ -14,11 +14,11 @@ import { ACTIVE_TURN_WRITE_BATCH_LIMITS } from "../../../runtime/persistence/sql import type { ExecutionIdentity, ExecutionLease } from "../execution/execution-mailbox-protocol.js"; import type { ExecutionProviderCommitReceipt, + ExecutionLivePublicationIntent, ProjectedCommittedProviderEvent, ExecutionSemanticOperation, ExecutionSemanticWriter, ExecutionWriteReceipt, - ExecutionLivePublicationIntent, ExecutionLivePublicationReceipt, } from "../execution/execution-worker-handler.js"; import type { CanonicalAgentEventPublisher } from "./canonical-agent-boundary.js"; @@ -43,6 +43,9 @@ const MAX_BUFFERED_PUBLICATION_EVENTS = 2_048; const MAX_LIVE_PUBLICATION_EVENTS = 64; const MAX_LIVE_PUBLICATION_BYTES = 256 * 1024; const MAX_LIVE_RECEIPT_BYTES = 512 * 1024; +const LIVE_RECOVERY_KINDS: ReadonlySet = new Set([ + "append-assistant-text", "narrative-delta", "live-event", +]); const narrativeDeltaSchema = z.object({ executionId: z.string(), items: z.array(ParentNarrativeRecoveryItemSchema()), @@ -197,6 +200,10 @@ export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter if (existing.kind === "committed" && operation.mutation.kind === "finish") { this.assistantText.retire(operation.execution.executionId); } + if (existing.kind === "committed" && operation.mutation.kind === "live-event" + && operation.mutation.text.kind === "reclassify") { + this.assistantText.discardRecoveryJournal(operation.execution.executionId); + } return existing; } try { @@ -254,11 +261,10 @@ export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter } private async applySupported(operation: ExecutionSemanticOperation, hash: string): Promise { + if (LIVE_RECOVERY_KINDS.has(operation.mutation.kind)) return this.applyLiveRecovery(operation, hash); switch (operation.mutation.kind) { case "begin": return this.begin(operation, hash); case "append-events": return this.append(operation, hash); - case "append-assistant-text": return this.appendAssistantText(operation, hash); - case "narrative-delta": return this.recordNarrativeDelta(operation, hash); case "checkpoint": case "stop-requested": case "provider-outcome": return this.control(operation, hash); @@ -371,12 +377,8 @@ export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter if (mutation.kind !== "narrative-delta") return conflict(operation); return this.db.transaction(() => { const head = this.requireNextHead(operation); - const checkpoint = this.canonical.loadCheckpoint(operation.execution.executionId); - if (!checkpoint || checkpoint.threadId !== operation.execution.threadId - || checkpoint.turnId !== operation.execution.turnId || checkpoint.terminalOutcome !== null) { - throw new SemanticConflict(); - } - if (!this.canonical.recordParentNarrativeRecovery(mutation.input)) throw new SemanticConflict(); + this.requireUnfinishedCheckpoint(operation.execution); + this.persistNarrativeDelta(mutation.input); const receipt = committed(operation, head.durableRevision); this.storeHead({ ...head, ordinal: operation.ordinal }); this.storeReceipt(operation, hash, receipt); @@ -384,6 +386,62 @@ export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter })(); } + private applyLiveRecovery(operation: ExecutionSemanticOperation, hash: string): ExecutionWriteReceipt { + if (operation.mutation.kind === "append-assistant-text") return this.appendAssistantText(operation, hash); + if (operation.mutation.kind === "narrative-delta") return this.recordNarrativeDelta(operation, hash); + return this.recordLiveEvent(operation, hash); + } + + private recordLiveEvent(operation: ExecutionSemanticOperation, hash: string): ExecutionWriteReceipt { + const mutation = operation.mutation; + if (mutation.kind !== "live-event") return conflict(operation); + const receipt = this.db.transaction(() => { + const head = this.requireNextHead(operation); + if (head.providerId !== "codex") throw new SemanticConflict(); + this.requireUnfinishedCheckpoint(operation.execution); + const textResult = this.applyLiveText(mutation.text, operation.execution.executionId); + if (mutation.narrative) this.persistNarrativeDelta(mutation.narrative); + if (mutation.text.kind === "reclassify" && !this.assistantText.resetInTransaction(operation.execution.executionId)) { + throw new SemanticConflict(); + } + const committedReceipt = committed(operation, head.durableRevision, undefined, undefined, textResult); + this.storeHead({ ...head, ordinal: operation.ordinal }); + this.storeReceipt(operation, hash, committedReceipt); + return committedReceipt; + })(); + if (mutation.text.kind === "reclassify") { + this.assistantText.discardRecoveryJournal(operation.execution.executionId); + } + return receipt; + } + + private applyLiveText( + text: Extract["text"], + executionId: string, + ): ParentAssistantTextCheckpointResult | undefined { + if (text.kind === "unchanged") return undefined; + if (text.kind === "reclassify") { + if (this.assistantText.restore(executionId) !== text.expectedText) throw new SemanticConflict(); + return undefined; + } + const result = this.assistantText.appendChunk(text.kind === "append" ? text.inputs : [text.input]); + if (result.outcome !== "committed") throw new SemanticConflict(); + return result; + } + + private requireUnfinishedCheckpoint(execution: ExecutionIdentity): void { + const checkpoint = this.canonical.loadCheckpoint(execution.executionId); + if (!checkpoint || checkpoint.threadId !== execution.threadId + || checkpoint.turnId !== execution.turnId || checkpoint.terminalOutcome !== null) { + throw new SemanticConflict(); + } + } + + private persistNarrativeDelta(input: Extract["input"]): void { + if (!this.canonical.recordParentNarrativeRecovery(input)) throw new SemanticConflict(); + } + private async finish(operation: ExecutionSemanticOperation, hash: string): Promise { const mutation = operation.mutation; if (mutation.kind !== "finish" || !validFinish(operation, mutation)) return conflict(operation); @@ -669,7 +727,8 @@ function validLivePublicationShape( mutationKind: ExecutionSemanticOperation["mutation"]["kind"], ): boolean { return Array.isArray(publication) && publication.length > 0 && publication.length <= MAX_LIVE_PUBLICATION_EVENTS - && (mutationKind === "begin" || mutationKind === "append-events" || mutationKind === "finish"); + && (mutationKind === "begin" || mutationKind === "append-events" || mutationKind === "finish" + || mutationKind === "live-event" && publication.length === 1); } function validLivePublicationIntent( @@ -700,9 +759,86 @@ function validSemanticMutationInput(operation: ExecutionSemanticOperation): bool if (operation.mutation.kind === "narrative-delta") { return validNarrativeDeltaInput(operation.mutation.input, operation.execution); } + if (operation.mutation.kind === "live-event") return validLiveEventInput(operation); return true; } +function validLiveEventInput(operation: ExecutionSemanticOperation): boolean { + const mutation = operation.mutation; + if (mutation.kind !== "live-event") return false; + const publication = liveEventPublication(operation); + if (!publication) return false; + const event = publication.event; + if (!validLiveTextPayload(mutation, operation.execution) + || !AgentEventSchema().safeParse(event).success || !validLiveTextAssociation(mutation.text, event)) return false; + if (mutation.narrative && !validNarrativeDeltaInput(mutation.narrative, operation.execution)) return false; + return validLiveEventBudget(operation, mutation); +} + +function liveEventPublication(operation: ExecutionSemanticOperation): ExecutionLivePublicationIntent | null { + const publication = operation.livePublication; + return Array.isArray(publication) && publication.length === 1 && publication[0]?.after === "writer" + ? publication[0] : null; +} + +function validLiveEventBudget( + operation: ExecutionSemanticOperation, + mutation: Extract, +): boolean { + const narrativeRows = mutation.narrative + ? mutation.narrative.items.length + (mutation.narrative.discardedItemIds?.length ?? 0) : 0; + const textRows = mutation.text.kind === "append" ? mutation.text.inputs.length + : mutation.text.kind === "unchanged" ? 0 : 1; + if (textRows + narrativeRows === 0 || textRows + narrativeRows > ACTIVE_TURN_WRITE_BATCH_LIMITS.maxRows - 2) return false; + return Buffer.byteLength(JSON.stringify(operation), "utf8") <= ACTIVE_TURN_WRITE_BATCH_LIMITS.maxBytes; +} + +function validLiveTextPayload( + mutation: Extract, + execution: ExecutionIdentity, +): boolean { + if (!mutation.text) return false; + if (mutation.text.kind === "append") return validAssistantTextInput(mutation.text.inputs, execution); + if (mutation.text.kind === "promote") return validAssistantTextInput([mutation.text.input], execution); + if (mutation.text.kind === "reclassify") return validReclassification(mutation.text.expectedText, mutation.narrative); + return mutation.text.kind === "unchanged"; +} + +function validReclassification( + expectedText: string, + narrative: Extract["narrative"], +): boolean { + if (typeof expectedText !== "string" || !expectedText) return false; + return expectedText.trim().length === 0 || Boolean(narrative?.items.some((item) => + item.kind === "narrationSegment" && item.record.text === expectedText)); +} + +function validLiveTextAssociation( + text: Extract["text"], + event: ExecutionLivePublicationIntent["event"], +): boolean { + switch (text.kind) { + case "unchanged": return validNarrativeEvent(event); + case "append": return validAppendEvent(event, text.inputs); + case "reclassify": return event.type === "assistantMessageBoundary" && event.isFinalResponse === false + && text.expectedText.length > 0; + case "promote": return event.type === "assistantMessageBoundary" && event.isFinalResponse === true; + } +} + +function validNarrativeEvent(event: ExecutionLivePublicationIntent["event"]): boolean { + return event.type === "textDelta" ? event.isFinalResponse === false + : ["assistantMessageBoundary", "toolUse", "toolResult", "hookStarted", "hookCompleted"].includes(event.type); +} + +function validAppendEvent( + event: ExecutionLivePublicationIntent["event"], + inputs: readonly ParentAssistantTextCheckpointInput[], +): boolean { + return event.type === "textDelta" && event.isFinalResponse !== false + && inputs.map((input) => input.text).join("") === event.delta; +} + function validNarrativeDeltaInput( input: Extract["input"], execution: ExecutionIdentity, From c03b1e5cd9da55f417d817ae0bdeef48f0bcde0a Mon Sep 17 00:00:00 2001 From: "(CJ) Chukwudi Nwobodo" <142016413+chuks-qua@users.noreply.github.com> Date: Thu, 24 Sep 2026 22:29:15 +0100 Subject: [PATCH 091/136] fix(server): acknowledge effect-free live boundaries --- .../canonical-execution-semantic-writer.test.ts | 15 +++++++++++++++ .../canonical-execution-semantic-writer.ts | 2 +- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/apps/server/src/features/agents/canonical/__tests__/canonical-execution-semantic-writer.test.ts b/apps/server/src/features/agents/canonical/__tests__/canonical-execution-semantic-writer.test.ts index 46b65c9a5..7d8718e7f 100644 --- a/apps/server/src/features/agents/canonical/__tests__/canonical-execution-semantic-writer.test.ts +++ b/apps/server/src/features/agents/canonical/__tests__/canonical-execution-semantic-writer.test.ts @@ -496,6 +496,21 @@ describe("CanonicalExecutionSemanticWriter through ExecutionWorkerHandler", () = })); }); + it("durably acknowledges a boundary with no new text or narrative rows", async () => { + expect((await send(1, { kind: "start", providerId: "codex", input: startInput() })).kind).toBe("committed"); + const boundary = { type: AgentEventType.AssistantMessageBoundary, threadId: THREAD_ID, + turnExecutionId: EXECUTION_ID, isFinalResponse: true }; + const op = { ...operation(2, { kind: "live-event", text: { kind: "unchanged" } }), + livePublication: [{ after: "writer" as const, event: boundary }] }; + const receipt = await writer.transact(op); + expect(receipt).toMatchObject({ kind: "committed", livePublication: [ + { publicationId: "lease-1:2:0", event: boundary }, + ] }); + expect(await writer.transact(op)).toEqual(receipt); + expect(db.prepare("SELECT COUNT(*) AS count FROM canonical_writer_operation_receipts WHERE execution_id = ? AND operation_id = ?") + .get(EXECUTION_ID, "lease-1:2")).toEqual({ count: 1 }); + }); + it("rejects a compound live event that exceeds its shared row or byte budget", async () => { expect((await send(1, { kind: "start", providerId: "codex", input: startInput() })).kind).toBe("committed"); const event = { type: AgentEventType.TextDelta, threadId: THREAD_ID, diff --git a/apps/server/src/features/agents/canonical/canonical-execution-semantic-writer.ts b/apps/server/src/features/agents/canonical/canonical-execution-semantic-writer.ts index 494f97730..40f16fce4 100644 --- a/apps/server/src/features/agents/canonical/canonical-execution-semantic-writer.ts +++ b/apps/server/src/features/agents/canonical/canonical-execution-semantic-writer.ts @@ -789,7 +789,7 @@ function validLiveEventBudget( ? mutation.narrative.items.length + (mutation.narrative.discardedItemIds?.length ?? 0) : 0; const textRows = mutation.text.kind === "append" ? mutation.text.inputs.length : mutation.text.kind === "unchanged" ? 0 : 1; - if (textRows + narrativeRows === 0 || textRows + narrativeRows > ACTIVE_TURN_WRITE_BATCH_LIMITS.maxRows - 2) return false; + if (textRows + narrativeRows > ACTIVE_TURN_WRITE_BATCH_LIMITS.maxRows - 2) return false; return Buffer.byteLength(JSON.stringify(operation), "utf8") <= ACTIVE_TURN_WRITE_BATCH_LIMITS.maxBytes; } From e5c15e9cb18d240e0fe03958ab055b475773de36 Mon Sep 17 00:00:00 2001 From: "(CJ) Chukwudi Nwobodo" <142016413+chuks-qua@users.noreply.github.com> Date: Thu, 24 Sep 2026 22:31:01 +0100 Subject: [PATCH 092/136] feat(server): commit task tool effects with live events --- ...anonical-execution-semantic-writer.test.ts | 23 ++++++++++ .../canonical-execution-semantic-writer.ts | 44 ++++++++++++++++++- .../execution/execution-worker-handler.ts | 4 ++ 3 files changed, 70 insertions(+), 1 deletion(-) diff --git a/apps/server/src/features/agents/canonical/__tests__/canonical-execution-semantic-writer.test.ts b/apps/server/src/features/agents/canonical/__tests__/canonical-execution-semantic-writer.test.ts index 7d8718e7f..87ce57ea3 100644 --- a/apps/server/src/features/agents/canonical/__tests__/canonical-execution-semantic-writer.test.ts +++ b/apps/server/src/features/agents/canonical/__tests__/canonical-execution-semantic-writer.test.ts @@ -8,6 +8,7 @@ import { afterEach, beforeEach, describe, expect, it } from "vitest"; import { openDatabase } from "../../../../runtime/persistence/sqlite/database.js"; import { MessageRepo } from "../../conversation/persistence/message-repo.js"; +import { TaskRepo } from "../../orchestration/persistence/task-repo.js"; import type { ExecutionSemanticOperation } from "../../execution/execution-worker-handler.js"; import { ExecutionWorkerHandler } from "../../execution/execution-worker-handler.js"; import { ParentAssistantTextCheckpointService } from "../../turns/parent-assistant-text-checkpoint-service.js"; @@ -511,6 +512,28 @@ describe("CanonicalExecutionSemanticWriter through ExecutionWorkerHandler", () = .get(EXECUTION_ID, "lease-1:2")).toEqual({ count: 1 }); }); + it("stores task-tool rows before the matching live tool publication", async () => { + expect((await send(1, { kind: "start", providerId: "codex", input: startInput() })).kind).toBe("committed"); + const toolUse = { type: AgentEventType.ToolUse, threadId: THREAD_ID, turnExecutionId: EXECUTION_ID, + toolCallId: "todo-1", toolName: "TodoWrite", toolInput: { todos: [{ content: "Ship task" }] } }; + const op = { ...operation(2, { kind: "live-event", text: { kind: "unchanged" }, + taskIntents: [{ kind: "upsert-group", group: "Tasks", + tasks: [{ content: "Ship task", status: "pending", group: "Tasks" }] }], + }), livePublication: [{ after: "writer" as const, event: toolUse }] }; + db.run("CREATE TRIGGER fail_task_receipt BEFORE INSERT ON canonical_writer_operation_receipts WHEN NEW.kind = 'semantic:live-event' BEGIN SELECT RAISE(ABORT, 'task receipt unavailable'); END"); + await expect(writer.transact(op)).rejects.toThrow("task receipt unavailable"); + expect(new TaskRepo(db).get(THREAD_ID)).toBeNull(); + db.run("DROP TRIGGER fail_task_receipt"); + const receipt = await writer.transact(op); + expect(receipt).toMatchObject({ kind: "committed", livePublication: [ + { publicationId: "lease-1:2:0", event: toolUse }, + ] }); + expect(new TaskRepo(db).get(THREAD_ID)).toEqual([ + { content: "Ship task", status: "pending", group: "Tasks" }, + ]); + expect(await writer.transact(op)).toEqual(receipt); + }); + it("rejects a compound live event that exceeds its shared row or byte budget", async () => { expect((await send(1, { kind: "start", providerId: "codex", input: startInput() })).kind).toBe("committed"); const event = { type: AgentEventType.TextDelta, threadId: THREAD_ID, diff --git a/apps/server/src/features/agents/canonical/canonical-execution-semantic-writer.ts b/apps/server/src/features/agents/canonical/canonical-execution-semantic-writer.ts index 40f16fce4..14b8f77d2 100644 --- a/apps/server/src/features/agents/canonical/canonical-execution-semantic-writer.ts +++ b/apps/server/src/features/agents/canonical/canonical-execution-semantic-writer.ts @@ -26,6 +26,8 @@ import { CanonicalAgentBoundary } from "./canonical-agent-boundary.js"; import { CanonicalCommittedProviderProjector } from "./canonical-committed-provider-projector.js"; import { CanonicalContextCompactionProjection } from "./canonical-context-compaction-projection.js"; import { CanonicalParentTurnWrite } from "./canonical-parent-turn-write.js"; +import { TaskRepo } from "../orchestration/persistence/task-repo.js"; +import type { TaskToolWriteIntent } from "../tasks/task-tool-intent-reducer.js"; import { ParentAssistantTextCheckpointService, PARENT_ASSISTANT_TEXT_QUEUE_POLICY, @@ -51,6 +53,21 @@ const narrativeDeltaSchema = z.object({ items: z.array(ParentNarrativeRecoveryItemSchema()), discardedItemIds: z.array(z.string().regex(/^(toolCall|narrationSegment|hook):.+$/)).optional(), }); +const storedTaskSchema = z.object({ + id: z.string().max(256).optional(), + content: z.string().min(1).max(16 * 1024), + status: z.enum(["pending", "in_progress", "completed", "cancelled"]), + activeForm: z.string().max(4096).optional(), + group: z.string().max(128).optional(), +}).strict(); +const taskIntentsSchema = z.array(z.discriminatedUnion("kind", [ + z.object({ kind: z.literal("upsert-group"), group: z.string().max(128), + tasks: z.array(storedTaskSchema).max(256) }).strict(), + z.object({ kind: z.literal("append-task"), task: storedTaskSchema }).strict(), + z.object({ kind: z.literal("update-task"), id: z.string().min(1).max(256), + group: z.string().max(128), patch: storedTaskSchema.pick({ status: true, content: true, activeForm: true }).partial() }).strict(), + z.object({ kind: z.literal("remove-task"), id: z.string().min(1).max(256), group: z.string().max(128) }).strict(), +])).max(16); const storedPublicationSequencesSchema = z.array(z.union([ z.number().int().positive().max(Number.MAX_SAFE_INTEGER), z.object({ executionId: z.string(), sequence: z.number().int().positive().max(Number.MAX_SAFE_INTEGER) }), @@ -153,6 +170,7 @@ export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter private readonly turns: CanonicalParentTurnWrite; private readonly providerProjector: CanonicalCommittedProviderProjector; private readonly contextCompaction: CanonicalContextCompactionProjection; + private readonly tasks: TaskRepo; private readonly assistantText: ParentAssistantTextCheckpointService; private readonly canonical: CanonicalAgentBoundary; private readonly findOperation: ReturnType; @@ -179,6 +197,7 @@ export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter this.canonical = codexBoundary; this.providerProjector = new CanonicalCommittedProviderProjector(codexBoundary); this.contextCompaction = new CanonicalContextCompactionProjection(db); + this.tasks = new TaskRepo(db); this.assistantText = new ParentAssistantTextCheckpointService(db); this.findOperation = db.prepare("SELECT kind, input_hash, receipt_json FROM canonical_writer_operation_receipts WHERE execution_id = ? AND operation_id = ?"); this.listPublicationChunks = db.prepare("SELECT operation_id, kind, input_hash, receipt_json FROM canonical_writer_operation_receipts WHERE execution_id = ? AND operation_id > ? AND operation_id < ? ORDER BY operation_id LIMIT ?"); @@ -401,6 +420,7 @@ export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter this.requireUnfinishedCheckpoint(operation.execution); const textResult = this.applyLiveText(mutation.text, operation.execution.executionId); if (mutation.narrative) this.persistNarrativeDelta(mutation.narrative); + if (mutation.taskIntents) this.applyTaskIntents(operation.execution.threadId, mutation.taskIntents); if (mutation.text.kind === "reclassify" && !this.assistantText.resetInTransaction(operation.execution.executionId)) { throw new SemanticConflict(); } @@ -429,6 +449,17 @@ export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter return result; } + private applyTaskIntents(threadId: string, intents: readonly TaskToolWriteIntent[]): void { + for (const intent of intents) { + switch (intent.kind) { + case "upsert-group": this.tasks.upsertGroup(threadId, intent.group, intent.tasks); break; + case "append-task": this.tasks.appendTask(threadId, intent.task); break; + case "update-task": this.tasks.updateTask(threadId, intent.id, intent.patch, intent.group); break; + case "remove-task": this.tasks.removeTask(threadId, intent.id, intent.group); break; + } + } + } + private requireUnfinishedCheckpoint(execution: ExecutionIdentity): void { const checkpoint = this.canonical.loadCheckpoint(execution.executionId); if (!checkpoint || checkpoint.threadId !== execution.threadId @@ -772,9 +803,19 @@ function validLiveEventInput(operation: ExecutionSemanticOperation): boolean { if (!validLiveTextPayload(mutation, operation.execution) || !AgentEventSchema().safeParse(event).success || !validLiveTextAssociation(mutation.text, event)) return false; if (mutation.narrative && !validNarrativeDeltaInput(mutation.narrative, operation.execution)) return false; + if (!validLiveTaskIntents(mutation.taskIntents, event)) return false; return validLiveEventBudget(operation, mutation); } +function validLiveTaskIntents( + intents: readonly TaskToolWriteIntent[] | undefined, + event: ExecutionLivePublicationIntent["event"], +): boolean { + if (!intents) return true; + if (!taskIntentsSchema.safeParse(intents).success) return false; + return intents.length === 0 || event.type === "toolUse" || event.type === "toolResult"; +} + function liveEventPublication(operation: ExecutionSemanticOperation): ExecutionLivePublicationIntent | null { const publication = operation.livePublication; return Array.isArray(publication) && publication.length === 1 && publication[0]?.after === "writer" @@ -789,7 +830,8 @@ function validLiveEventBudget( ? mutation.narrative.items.length + (mutation.narrative.discardedItemIds?.length ?? 0) : 0; const textRows = mutation.text.kind === "append" ? mutation.text.inputs.length : mutation.text.kind === "unchanged" ? 0 : 1; - if (textRows + narrativeRows > ACTIVE_TURN_WRITE_BATCH_LIMITS.maxRows - 2) return false; + if (textRows + narrativeRows + (mutation.taskIntents?.length ?? 0) + > ACTIVE_TURN_WRITE_BATCH_LIMITS.maxRows - 2) return false; return Buffer.byteLength(JSON.stringify(operation), "utf8") <= ACTIVE_TURN_WRITE_BATCH_LIMITS.maxBytes; } diff --git a/apps/server/src/features/agents/execution/execution-worker-handler.ts b/apps/server/src/features/agents/execution/execution-worker-handler.ts index 7f17534c3..82026b78f 100644 --- a/apps/server/src/features/agents/execution/execution-worker-handler.ts +++ b/apps/server/src/features/agents/execution/execution-worker-handler.ts @@ -13,6 +13,7 @@ import type { ParentAssistantTextCheckpointResult, } from "../turns/parent-assistant-text-checkpoint-service.js"; import type { ParentNarrativeRecoveryCommit } from "../turns/parent-turn-durability.js"; +import type { TaskToolWriteIntent } from "../tasks/task-tool-intent-reducer.js"; import type { ExecutionIdentity, ExecutionLease, @@ -36,6 +37,7 @@ export type ExecutionWorkCommand = | { readonly kind: "reclassify"; readonly expectedText: string } | { readonly kind: "promote"; readonly input: ParentAssistantTextCheckpointInput }; readonly narrative?: ParentNarrativeRecoveryCommit; + readonly taskIntents?: readonly TaskToolWriteIntent[]; readonly publication: ExecutionLivePublicationIntent; } | { readonly kind: "checkpoint"; readonly phase: string; readonly nativeCursor: unknown | null } @@ -63,6 +65,7 @@ export interface ExecutionSemanticOperation { readonly kind: "live-event"; readonly text: Extract["text"]; readonly narrative?: ParentNarrativeRecoveryCommit; + readonly taskIntents?: readonly TaskToolWriteIntent[]; } | { readonly kind: "checkpoint"; readonly phase: string; readonly nativeCursor: unknown | null } | { readonly kind: "stop-requested"; readonly requestId: string; readonly lastAdmittedOrdinal: number } @@ -289,6 +292,7 @@ function liveEventMutation( return { kind: "live-event", text: command.text, ...(command.narrative ? { narrative: command.narrative } : {}), + ...(command.taskIntents ? { taskIntents: command.taskIntents } : {}), }; } From 74c6a30ef9fac969d328ffd34cee9a68ab14de3b Mon Sep 17 00:00:00 2001 From: "(CJ) Chukwudi Nwobodo" <142016413+chuks-qua@users.noreply.github.com> Date: Thu, 24 Sep 2026 22:31:06 +0100 Subject: [PATCH 093/136] feat(server): prepare Codex system and error projections --- ...ical-codex-system-error-projection.test.ts | 149 ++++++++++++++++++ ...canonical-codex-system-error-projection.ts | 122 ++++++++++++++ 2 files changed, 271 insertions(+) create mode 100644 apps/server/src/features/agents/canonical/__tests__/canonical-codex-system-error-projection.test.ts create mode 100644 apps/server/src/features/agents/canonical/canonical-codex-system-error-projection.ts diff --git a/apps/server/src/features/agents/canonical/__tests__/canonical-codex-system-error-projection.test.ts b/apps/server/src/features/agents/canonical/__tests__/canonical-codex-system-error-projection.test.ts new file mode 100644 index 000000000..c010b6894 --- /dev/null +++ b/apps/server/src/features/agents/canonical/__tests__/canonical-codex-system-error-projection.test.ts @@ -0,0 +1,149 @@ +import "reflect-metadata"; +import * as NodeFS from "node:fs"; +import * as NodeOS from "node:os"; +import * as NodePath from "node:path"; +import type { Database } from "bun:sqlite"; +import type { AgentEvent } from "@mcode/contracts"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; + +import { openDatabase } from "../../../../runtime/persistence/sqlite/database.js"; +import { MessageRepo } from "../../conversation/persistence/message-repo.js"; +import { CodexLiveEventReducer } from "../../execution/codex-live-event-reducer.js"; +import { CanonicalAgentBoundary } from "../canonical-agent-boundary.js"; +import { CanonicalCodexSystemErrorProjection } from "../canonical-codex-system-error-projection.js"; +import { CanonicalParentTurnWrite } from "../canonical-parent-turn-write.js"; + +const THREAD_ID = "thread-system-projection"; +const TURN_ID = "turn-system-projection"; +const EXECUTION_ID = "11111111-1111-4111-8111-111111111111"; +const NOW = "2026-09-24T10:00:00.000Z"; +const execution = { threadId: THREAD_ID, turnId: TURN_ID, executionId: EXECUTION_ID }; + +function seedThread(db: Database): void { + db.prepare("INSERT INTO workspaces (id, name, path, created_at, updated_at) VALUES (?, ?, ?, ?, ?)") + .run("workspace-system-projection", "Workspace", "C:/fixture", NOW, NOW); + db.prepare("INSERT INTO threads (id, workspace_id, title, branch, provider, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?)") + .run(THREAD_ID, "workspace-system-projection", "Thread", "main", "codex", NOW, NOW); + const turns = new CanonicalParentTurnWrite(db, () => {}); + expect(turns.start({ + thread: { id: THREAD_ID, workspaceId: "workspace-system-projection", providerId: "codex", createdAt: NOW }, + turnId: TURN_ID, + executionId: EXECUTION_ID, + permissionMode: "supervised", + providerIdentities: [], + userMessage: { kind: "create", messageId: "user-system-projection", content: "Question", sequence: 1 }, + }).outcome).toBe("committed"); +} + +function boundSystem(subtype: string, extra: Partial> = {}): Extract { + return { type: "system", threadId: THREAD_ID, turnExecutionId: EXECUTION_ID, subtype, ...extra }; +} + +describe("CanonicalCodexSystemErrorProjection", () => { + let directory: string; + let path: string; + let db: Database; + let projection: CanonicalCodexSystemErrorProjection; + let reducer: CodexLiveEventReducer; + + beforeEach(() => { + directory = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "mcode-codex-system-")); + path = NodePath.join(directory, "app.sqlite"); + db = openDatabase({ dbPath: path }); + seedThread(db); + projection = new CanonicalCodexSystemErrorProjection(db); + reducer = new CodexLiveEventReducer(execution); + }); + + afterEach(() => { + db.close(true); + NodeFS.rmSync(directory, { recursive: true, force: true }); + }); + + it("persists turn-scoped notices with publication identity and rejects an unbound session event", () => { + const session = boundSystem("provider.session.started", { + systemNotice: { kind: "diagnostic", presentation: "timeline", scope: "session", sessionId: "session-1" }, + }); + const unbound = reducer.reduce({ ...session, turnExecutionId: undefined }); + expect(unbound).toMatchObject({ kind: "unsupported", eventType: "system" }); + expect(() => projection.project(unbound)).toThrow("Unsupported Codex event"); + expect(db.prepare("SELECT current_notice_session_id FROM threads WHERE id = ?").get(THREAD_ID)) + .toEqual({ current_notice_session_id: null }); + + const sessionResult = projection.project(reducer.reduce(session)); + expect(sessionResult).toMatchObject({ kind: "system", event: { subtype: "provider.session.started" } }); + expect(db.prepare("SELECT current_notice_session_id FROM threads WHERE id = ?").get(THREAD_ID)) + .toEqual({ current_notice_session_id: "session-1" }); + expect(reducer.reduce({ type: "turnStarted", threadId: THREAD_ID, turnExecutionId: EXECUTION_ID }).kind).toBe("reduced"); + + const notice = boundSystem("provider.notice.unknown-event", { + message: "Codex reported an update.", + systemNotice: { kind: "diagnostic", presentation: "timeline", scope: "session", + sessionId: "session-1", noticeKey: "notice-1" }, + }); + const reduction = reducer.reduce(notice); + const first = projection.project(reduction); + expect(first.kind).toBe("system"); + if (first.kind !== "system") return; + expect(first.event.messageId).toBeDefined(); + if (!first.event.messageId) return; + expect(new MessageRepo(db).findByIdInThread(THREAD_ID, first.event.messageId)) + .toMatchObject({ role: "system", content: "Codex reported an update." }); + expect(projection.project(reduction)).toEqual(first); + expect(db.prepare("SELECT COUNT(*) AS count FROM messages WHERE thread_id = ? AND content = ?") + .get(THREAD_ID, notice.message)).toEqual({ count: 1 }); + expect(structuredClone(first)).toEqual(first); + + db.close(true); + db = openDatabase({ dbPath: path }); + expect(db.prepare("SELECT current_notice_session_id FROM threads WHERE id = ?").get(THREAD_ID)) + .toEqual({ current_notice_session_id: "session-1" }); + expect(new MessageRepo(db).findByIdInThread(THREAD_ID, first.event.messageId)).not.toBeNull(); + }); + + it("persists a Codex cursor with canonical provenance and clears an invalidated cursor", () => { + const unbound = reducer.reduce({ ...boundSystem("sdk_session_id:session-owned"), turnExecutionId: undefined }); + expect(unbound).toMatchObject({ kind: "unsupported", eventType: "system" }); + expect(() => projection.project(unbound)).toThrow("Unsupported Codex event"); + expect(db.prepare("SELECT sdk_session_id FROM threads WHERE id = ?").get(THREAD_ID)) + .toEqual({ sdk_session_id: null }); + + expect(reducer.reduce({ type: "turnStarted", threadId: THREAD_ID, turnExecutionId: EXECUTION_ID }).kind).toBe("reduced"); + const saved = projection.project(reducer.reduce(boundSystem("sdk_session_id:native-thread"))); + expect(saved).toMatchObject({ kind: "system", event: { subtype: "sdk_session_id:native-thread" } }); + expect(db.prepare("SELECT sdk_session_id FROM threads WHERE id = ?").get(THREAD_ID)) + .toEqual({ sdk_session_id: "native-thread" }); + expect(new CanonicalAgentBoundary(db, () => {}).loadCheckpoint(EXECUTION_ID)) + .toMatchObject({ nativeCursor: { providerId: "codex", scope: "thread", value: "native-thread", provenance: "native" } }); + expect(projection.project(reducer.reduce(boundSystem("sdk_session_invalidated")))).toMatchObject({ kind: "system" }); + expect(db.prepare("SELECT sdk_session_id FROM threads WHERE id = ?").get(THREAD_ID)) + .toEqual({ sdk_session_id: null }); + + db.prepare("UPDATE canonical_agent_ingest_checkpoints SET terminal_outcome = 'completed' WHERE execution_id = ?") + .run(EXECUTION_ID); + expect(() => projection.project(reducer.reduce(boundSystem("sdk_session_id:late-thread")))) + .toThrow("could not be persisted"); + expect(db.prepare("SELECT sdk_session_id FROM threads WHERE id = ?").get(THREAD_ID)) + .toEqual({ sdk_session_id: null }); + }); + + it("returns cloneable error terminal input without publishing or terminalizing early", () => { + expect(reducer.reduce({ type: "turnStarted", threadId: THREAD_ID, turnExecutionId: EXECUTION_ID }).kind).toBe("reduced"); + expect(reducer.reduce({ type: "textDelta", threadId: THREAD_ID, turnExecutionId: EXECUTION_ID, + delta: "Partial answer", isFinalResponse: true }).kind).toBe("reduced"); + const reduction = reducer.reduce({ type: "error", threadId: THREAD_ID, turnExecutionId: EXECUTION_ID, error: "Provider failed" }); + expect(() => projection.project(reduction)).toThrow("end time"); + const result = projection.project(reduction, NOW); + expect(result).toMatchObject({ kind: "error-terminal", event: { error: "Provider failed" }, after: "terminal", + input: { threadId: THREAD_ID, executionId: EXECUTION_ID, outcome: "errored", assistant: { content: "Partial answer" } } }); + expect(structuredClone(result)).toEqual(result); + expect(db.prepare("SELECT terminal_outcome FROM canonical_agent_ingest_checkpoints WHERE execution_id = ?").get(EXECUTION_ID)) + .toEqual({ terminal_outcome: null }); + if (result.kind !== "error-terminal") return; + const staged = new CanonicalParentTurnWrite(db, () => {}).stageTerminalProjection(result.input); + expect(staged.messageId).toBeDefined(); + if (!staged.messageId) return; + expect(new MessageRepo(db).findByIdInThreadIncludingInternal(THREAD_ID, staged.messageId)) + .toMatchObject({ content: "Partial answer", is_internal: true }); + }); +}); diff --git a/apps/server/src/features/agents/canonical/canonical-codex-system-error-projection.ts b/apps/server/src/features/agents/canonical/canonical-codex-system-error-projection.ts new file mode 100644 index 000000000..7ea785785 --- /dev/null +++ b/apps/server/src/features/agents/canonical/canonical-codex-system-error-projection.ts @@ -0,0 +1,122 @@ +import type { Database } from "bun:sqlite"; +import type { AgentEvent } from "@mcode/contracts"; + +import { ThreadRepo } from "../../thread-control/persistence/thread-repo.js"; +import { MessageRepo } from "../conversation/persistence/message-repo.js"; +import type { CodexLiveReduction, CodexLiveWriterIntent } from "../execution/codex-live-event-reducer.js"; +import type { DataOnlyParentTerminalProjectionInput } from "./canonical-parent-turn-write.js"; +import { CanonicalAgentBoundary } from "./canonical-agent-boundary.js"; + +type Reduced = Extract; +type SystemEvent = Extract; +type SystemIntent = Extract; +type ErrorTerminalIntent = Extract; + +function errorTerminalIntent(reduction: Reduced, event: Extract): ErrorTerminalIntent { + const error = reduction.writer.find((intent) => intent.kind === "turn-error"); + const terminal = reduction.writer.find((intent) => intent.kind === "terminal-projection"); + if (error?.kind !== "turn-error" || error.error !== event.error + || terminal?.kind !== "terminal-projection" || terminal.source !== "error" || terminal.outcome !== "errored") { + throw new Error("Codex error needs an errored terminal projection and end time"); + } + return terminal; +} + +/** Cloneable result of writer-local system projection or error terminal preparation. */ +export type CanonicalCodexSystemErrorResult = + | { readonly kind: "system"; readonly event: SystemEvent; readonly after: "writer" | "terminal" } + | { readonly kind: "error-terminal"; readonly event: Extract; readonly after: "terminal"; readonly input: DataOnlyParentTerminalProjectionInput }; + +/** Projects only execution-bound Codex system events and prepares errors for the terminal owner. */ +export class CanonicalCodexSystemErrorProjection { + private readonly threads: ThreadRepo; + private readonly messages: MessageRepo; + private readonly canonical: CanonicalAgentBoundary; + + constructor(private readonly db: Database) { + this.threads = new ThreadRepo(db); + this.messages = new MessageRepo(db); + this.canonical = new CanonicalAgentBoundary(db, () => {}); + } + + /** Call inside the canonical append transaction, before its receipt and publication. Errors still require terminal staging and finish. */ + project(reduction: CodexLiveReduction, endedAt?: string): CanonicalCodexSystemErrorResult { + if (reduction.kind !== "reduced") throw new Error("Unsupported Codex event cannot be projected"); + const event = reduction.publication.event; + if (event.threadId !== reduction.execution.threadId || event.turnExecutionId !== reduction.execution.executionId) { + throw new Error("Codex event lacks exact execution ownership"); + } + if (event.type === "system") return this.projectSystem(reduction, event); + if (event.type === "error") return this.prepareError(reduction, event, endedAt); + throw new Error(`Codex ${event.type} needs another feature owner`); + } + + private projectSystem(reduction: Reduced, event: SystemEvent): CanonicalCodexSystemErrorResult { + return this.db.transaction(() => { + let published: SystemEvent = { ...event }; + for (const intent of reduction.writer) { + if (!this.isSystemIntent(intent) || JSON.stringify(intent.event) !== JSON.stringify(event)) { + throw new Error("Codex system intent does not match its publication"); + } + published = this.applySystemIntent(reduction, intent, published); + } + return structuredClone({ kind: "system", event: published, after: reduction.publication.after } satisfies CanonicalCodexSystemErrorResult); + })(); + } + + private isSystemIntent(intent: CodexLiveWriterIntent): intent is SystemIntent { + return intent.kind === "notice-session" || intent.kind === "system-notice" || intent.kind === "session-cursor"; + } + + private applySystemIntent(reduction: Reduced, intent: SystemIntent, event: SystemEvent): SystemEvent { + switch (intent.kind) { + case "notice-session": + if (event.subtype !== "provider.session.started") throw new Error("Codex notice session subtype is invalid"); + this.messages.beginNoticeSession(event.threadId, event.systemNotice?.sessionId); + return event; + case "system-notice": { + if (!event.subtype.startsWith("provider.notice.") || !event.message) throw new Error("Codex notice is invalid"); + const sequence = this.messages.getLatestSequenceIncludingInternal(event.threadId) + 1; + const message = this.messages.createSystemNotice(event.threadId, event.message ?? "", sequence, event.systemNotice); + return { ...event, messageId: message.id }; + } + case "session-cursor": + this.applyCursor(reduction.execution.executionId, event); + return event; + } + } + + private applyCursor(executionId: string, event: SystemEvent): void { + if (event.subtype === "sdk_session_invalidated") { + if (!this.threads.clearSdkSessionId(event.threadId)) throw new Error("Codex cursor thread is missing"); + return; + } + if (!event.subtype.startsWith("sdk_session_id:")) throw new Error("Codex cursor subtype is invalid"); + const cursor = event.subtype.slice("sdk_session_id:".length); + if (!cursor) return; + if (!this.threads.updateSdkSessionId(event.threadId, cursor) + || !this.canonical.recordNativeCursor(executionId, { + providerId: "codex", scope: "thread", value: cursor, provenance: "native", + })) throw new Error("Codex cursor could not be persisted for this execution"); + } + + private prepareError(reduction: Reduced, event: Extract, endedAt?: string): CanonicalCodexSystemErrorResult { + if (reduction.publication.after !== "terminal" || !endedAt || !Number.isFinite(Date.parse(endedAt))) { + throw new Error("Codex error needs an errored terminal projection and end time"); + } + const terminal = errorTerminalIntent(reduction, event); + return structuredClone({ + kind: "error-terminal", + event, + after: "terminal", + input: { + threadId: reduction.execution.threadId, + executionId: reduction.execution.executionId, + outcome: "errored", + endedAt, + assistant: terminal.assistant, + narrative: terminal.narrative, + }, + } satisfies CanonicalCodexSystemErrorResult); + } +} From 461b9fc953b6b7d918429099af1e306f70cb680f Mon Sep 17 00:00:00 2001 From: "(CJ) Chukwudi Nwobodo" <142016413+chuks-qua@users.noreply.github.com> Date: Thu, 24 Sep 2026 22:37:18 +0100 Subject: [PATCH 094/136] fix(server): release compound live receipts after writer commit --- .../canonical-execution-writer-port.test.ts | 20 +++++++++++++++++++ .../execution-live-publication-release.ts | 2 +- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/apps/server/src/features/agents/canonical/__tests__/canonical-execution-writer-port.test.ts b/apps/server/src/features/agents/canonical/__tests__/canonical-execution-writer-port.test.ts index d7a74d43b..e0f780626 100644 --- a/apps/server/src/features/agents/canonical/__tests__/canonical-execution-writer-port.test.ts +++ b/apps/server/src/features/agents/canonical/__tests__/canonical-execution-writer-port.test.ts @@ -177,6 +177,26 @@ describe("execution semantic writer transport", () => { } }); + it("releases a compound live event after its single writer receipt", async () => { + writer = new CanonicalAgentWriterClient(NodePath.join(directory, "app.sqlite")); + const registry = new AgentEventPublicationRegistry(); + const published: AgentEvent[] = []; + registry.bind((event) => published.push(event)); + const port = new CanonicalExecutionWriterPort(writer, () => {}, new ExecutionLivePublicationRelease(registry)); + expect((await port.transact(beginOperation())).kind).toBe("committed"); + const event: AgentEvent = { type: AgentEventType.AssistantMessageBoundary, threadId: THREAD_ID, + turnExecutionId: EXECUTION_ID, isFinalResponse: false }; + const operation: ExecutionSemanticOperation = { + operationId: `${lease.leaseId}:2`, execution, lease, ordinal: 2, + mutation: { kind: "live-event", text: { kind: "unchanged" } }, + livePublication: [{ after: "writer", event }], + }; + expect((await port.transact(operation)).kind).toBe("committed"); + expect(published).toEqual([event]); + expect((await port.transact(operation)).kind).toBe("committed"); + expect(published).toEqual([event]); + }); + it("replays a committed live receipt when the public publisher becomes available", async () => { writer = new CanonicalAgentWriterClient(NodePath.join(directory, "app.sqlite")); const registry = new AgentEventPublicationRegistry(); diff --git a/apps/server/src/features/agents/canonical/execution-live-publication-release.ts b/apps/server/src/features/agents/canonical/execution-live-publication-release.ts index 5e8bd728d..84831c86e 100644 --- a/apps/server/src/features/agents/canonical/execution-live-publication-release.ts +++ b/apps/server/src/features/agents/canonical/execution-live-publication-release.ts @@ -78,6 +78,6 @@ function samePublicationHeader( } function barrierFor(kind: ExecutionSemanticOperation["mutation"]["kind"]): "writer" | "terminal" | null { - if (kind === "begin" || kind === "append-events") return "writer"; + if (kind === "begin" || kind === "append-events" || kind === "live-event") return "writer"; return kind === "finish" ? "terminal" : null; } From 3fbda03ba7a730c2f8424ad78474569cac9e85b3 Mon Sep 17 00:00:00 2001 From: "(CJ) Chukwudi Nwobodo" <142016413+chuks-qua@users.noreply.github.com> Date: Thu, 24 Sep 2026 22:39:15 +0100 Subject: [PATCH 095/136] feat(server): acknowledge transient Codex statuses per execution --- ...anonical-execution-semantic-writer.test.ts | 13 ++++++++ .../canonical-execution-semantic-writer.ts | 6 +++- .../codex-live-event-reducer.test.ts | 14 +++++++-- .../execution/codex-live-event-reducer.ts | 31 +++++++++---------- 4 files changed, 43 insertions(+), 21 deletions(-) diff --git a/apps/server/src/features/agents/canonical/__tests__/canonical-execution-semantic-writer.test.ts b/apps/server/src/features/agents/canonical/__tests__/canonical-execution-semantic-writer.test.ts index 87ce57ea3..73483b2ee 100644 --- a/apps/server/src/features/agents/canonical/__tests__/canonical-execution-semantic-writer.test.ts +++ b/apps/server/src/features/agents/canonical/__tests__/canonical-execution-semantic-writer.test.ts @@ -534,6 +534,19 @@ describe("CanonicalExecutionSemanticWriter through ExecutionWorkerHandler", () = expect(await writer.transact(op)).toEqual(receipt); }); + it("acknowledges transient tool progress without inventing a durable tool row", async () => { + expect((await send(1, { kind: "start", providerId: "codex", input: startInput() })).kind).toBe("committed"); + const event = { type: AgentEventType.ToolProgress, threadId: THREAD_ID, turnExecutionId: EXECUTION_ID, + toolCallId: "tool-1", toolName: "command_execution", elapsedSeconds: 1 }; + const op = { ...operation(2, { kind: "live-event", text: { kind: "unchanged" } }), + livePublication: [{ after: "writer" as const, event }] }; + const receipt = await writer.transact(op); + expect(receipt).toMatchObject({ kind: "committed", livePublication: [{ event }] }); + expect(await writer.transact(op)).toEqual(receipt); + expect(db.prepare("SELECT COUNT(*) AS count FROM canonical_agent_items WHERE turn_id = ? AND id LIKE 'toolCall:%'") + .get(TURN_ID)).toEqual({ count: 0 }); + }); + it("rejects a compound live event that exceeds its shared row or byte budget", async () => { expect((await send(1, { kind: "start", providerId: "codex", input: startInput() })).kind).toBe("committed"); const event = { type: AgentEventType.TextDelta, threadId: THREAD_ID, diff --git a/apps/server/src/features/agents/canonical/canonical-execution-semantic-writer.ts b/apps/server/src/features/agents/canonical/canonical-execution-semantic-writer.ts index 14b8f77d2..d23d8554c 100644 --- a/apps/server/src/features/agents/canonical/canonical-execution-semantic-writer.ts +++ b/apps/server/src/features/agents/canonical/canonical-execution-semantic-writer.ts @@ -870,7 +870,11 @@ function validLiveTextAssociation( function validNarrativeEvent(event: ExecutionLivePublicationIntent["event"]): boolean { return event.type === "textDelta" ? event.isFinalResponse === false - : ["assistantMessageBoundary", "toolUse", "toolResult", "hookStarted", "hookCompleted"].includes(event.type); + : [ + "assistantMessageBoundary", "toolUse", "toolResult", "hookStarted", "hookCompleted", + "modelFallback", "toolInputDelta", "toolProgress", "providerUnavailable", "hookProgress", + "apiRetry", "rateLimited", "quotaUpdate", "goalUpdated", "goalCleared", "mcpServerStartupStatus", + ].includes(event.type); } function validAppendEvent( diff --git a/apps/server/src/features/agents/execution/__tests__/codex-live-event-reducer.test.ts b/apps/server/src/features/agents/execution/__tests__/codex-live-event-reducer.test.ts index ee0a6feb9..6bf818589 100644 --- a/apps/server/src/features/agents/execution/__tests__/codex-live-event-reducer.test.ts +++ b/apps/server/src/features/agents/execution/__tests__/codex-live-event-reducer.test.ts @@ -237,9 +237,17 @@ describe("CodexLiveEventReducer", () => { expect(reducer.reduce(event("textDelta", { delta: "wrong", turnExecutionId: "22222222-2222-4222-8222-222222222222" }))).toEqual({ kind: "unsupported", eventType: "textDelta", reason: "different execution", }); - expect(reducer.reduce(event("modelFallback", { requestedModel: "a", actualModel: "b" }))).toEqual({ - kind: "unsupported", eventType: "modelFallback", reason: "model fallback needs the model selection owner", - }); + for (const status of [ + event("modelFallback", { requestedModel: "a", actualModel: "b" }), + event("toolInputDelta", { partialJson: "{" }), + event("toolProgress", { toolCallId: "tool-1", toolName: "command_execution", elapsedSeconds: 1 }), + event("providerUnavailable", { providerId: "codex", reason: "disabled" }), + event("hookProgress", { hookName: "check", output: "running" }), + ]) { + expect(reducer.reduce(status)).toMatchObject({ + kind: "reduced", writer: [], publication: { event: status, after: "writer" }, + }); + } const final = reducer.reduce(event("textDelta", { delta: "right", isFinalResponse: true })); expect(final.kind).toBe("reduced"); if (final.kind !== "reduced") return; diff --git a/apps/server/src/features/agents/execution/codex-live-event-reducer.ts b/apps/server/src/features/agents/execution/codex-live-event-reducer.ts index 82fac5c11..6ed0ca625 100644 --- a/apps/server/src/features/agents/execution/codex-live-event-reducer.ts +++ b/apps/server/src/features/agents/execution/codex-live-event-reducer.ts @@ -14,7 +14,12 @@ const BEFORE_START_EVENTS = new Set([ "system", "quotaUpdate", "goalUpdated", "goalCleared", "mcpServerStartupStatus", ]); const AFTER_COMPLETION_EVENTS = new Set([ - "ended", "hookStarted", "hookCompleted", ...BEFORE_START_EVENTS, + "ended", "hookStarted", "hookProgress", "hookCompleted", ...BEFORE_START_EVENTS, +]); +const TRANSIENT_STATUS_EVENTS = new Set([ + "apiRetry", "rateLimited", "quotaUpdate", "providerUnavailable", "modelFallback", + "toolInputDelta", "toolProgress", "hookProgress", "goalUpdated", "goalCleared", + "mcpServerStartupStatus", ]); // A new contract variant must receive an owner decision before the reducer accepts it. @@ -30,17 +35,17 @@ const UNSUPPORTED_FEATURE_REASON = { system: null, compacting: null, compactSummary: null, - modelFallback: "model fallback needs the model selection owner", + modelFallback: null, textDelta: null, - toolInputDelta: "incremental tool event needs the tool lifecycle owner", - toolProgress: "incremental tool event needs the tool lifecycle owner", + toolInputDelta: null, + toolProgress: null, contextEstimate: null, quotaUpdate: null, - providerUnavailable: "provider availability needs the provider dispatch owner", + providerUnavailable: null, rateLimited: null, apiRetry: null, hookStarted: null, - hookProgress: "hook output needs the hook lifecycle owner", + hookProgress: null, hookCompleted: null, assistantMessageBoundary: null, goalUpdated: null, @@ -216,17 +221,9 @@ export class CodexLiveEventReducer { } private applyStatus(event: AgentEvent): CodexLiveWriterIntent[] | undefined { - switch (event.type) { - case "system": return this.system(event); - // These status events have no server projection beyond their canonical receipt. - case "apiRetry": - case "rateLimited": - case "quotaUpdate": - case "goalUpdated": - case "goalCleared": - case "mcpServerStartupStatus": return []; - default: return undefined; - } + if (event.type === "system") return this.system(event); + // The legacy turn application only publishes these transient statuses. + return TRANSIENT_STATUS_EVENTS.has(event.type) ? [] : undefined; } private start(event: Extract): CodexLiveWriterIntent[] { From b80f5e10d0148322e6014584c4a6b972d1259e2f Mon Sep 17 00:00:00 2001 From: "(CJ) Chukwudi Nwobodo" <142016413+chuks-qua@users.noreply.github.com> Date: Thu, 24 Sep 2026 22:37:49 +0100 Subject: [PATCH 096/136] feat(server): fence pre-edit file observations by attempt --- ...execution-file-observation-handoff.test.ts | 116 ++++++++++++++++++ .../execution-file-observation-handoff.ts | 94 ++++++++++++++ .../contracts/src/providers/interfaces.ts | 4 + .../codex/codex-event-mapper.test.ts | 15 +++ .../codex-provider-subagent-turn.test.ts | 29 ++++- .../src/private/codex/codex-event-mapper.ts | 10 +- .../src/private/codex/codex-provider.ts | 20 ++- 7 files changed, 283 insertions(+), 5 deletions(-) create mode 100644 apps/server/src/features/agents/execution/__tests__/execution-file-observation-handoff.test.ts create mode 100644 apps/server/src/features/agents/execution/execution-file-observation-handoff.ts diff --git a/apps/server/src/features/agents/execution/__tests__/execution-file-observation-handoff.test.ts b/apps/server/src/features/agents/execution/__tests__/execution-file-observation-handoff.test.ts new file mode 100644 index 000000000..19f8546f0 --- /dev/null +++ b/apps/server/src/features/agents/execution/__tests__/execution-file-observation-handoff.test.ts @@ -0,0 +1,116 @@ +import * as NodeFSPromises from "node:fs/promises"; +import * as NodeOS from "node:os"; +import * as NodePath from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; + +import type { ProviderFileMutationStart } from "@mcode/contracts"; + +import { TurnFileTracker } from "../../turns/turn-file-tracker.js"; +import { ExecutionFileObservationHandoff } from "../execution-file-observation-handoff.js"; + +const directories: string[] = []; +const platform = "win32" as const; + +async function directory(): Promise { + const path = await NodeFSPromises.mkdtemp(NodePath.join(NodeOS.tmpdir(), "mcode-worker-file-observation-")); + directories.push(path); + return path; +} + +function tracker(): TurnFileTracker { + return new TurnFileTracker(async () => ({ kind: "unavailable" }), () => {}, platform); +} + +function mutation(executionId: string, toolCallId = "edit", deliveryAttempt = 1): ProviderFileMutationStart { + return { threadId: "thread", turnExecutionId: executionId, deliveryAttempt, + toolCallId, toolName: "Edit", toolInput: { file_path: "tracked.txt" } }; +} + +function toolUse(event: ProviderFileMutationStart) { + return { type: "toolUse" as const, threadId: event.threadId, + turnExecutionId: event.turnExecutionId, toolCallId: event.toolCallId, + toolName: event.toolName, toolInput: event.toolInput }; +} + +afterEach(async () => { + await Promise.all(directories.splice(0).map((path) => NodeFSPromises.rm(path, { recursive: true, force: true }))); +}); + +describe("ExecutionFileObservationHandoff", () => { + it("transfers a pre-edit baseline after the provider has changed the file", async () => { + const cwd = await directory(); + const path = NodePath.join(cwd, "tracked.txt"); + await NodeFSPromises.writeFile(path, "before\n"); + const host = tracker(); + const worker = tracker(); + const expected = { threadId: "thread", executionId: "current", cwd }; + const handoff = host.beginExecutionTurn({ ...expected, baselineRef: null }); + const bridge = new ExecutionFileObservationHandoff(host, handoff, 1); + const event = mutation(expected.executionId); + + expect(bridge.capture(event)).toBe(true); + await NodeFSPromises.writeFile(path, "after\nextra\n"); + expect(worker.beginTurnFromHandoff(structuredClone(handoff), expected)).toBe(true); + const captured = bridge.take(toolUse(event), 1); + if (!captured) throw new Error("Expected the pre-edit observation"); + expect(await worker.observeCapturedToolUse(toolUse(event), structuredClone(captured))).toBe(true); + await worker.observeToolResult("thread", "edit"); + expect(await worker.finalizeTurn("thread")).toMatchObject({ fileCount: 1, additions: 2, deletions: 1 }); + expect(bridge.take(toolUse(event), 1)).toBeNull(); + }); + + it("rejects an old child after retry even when its tool call ID collides", async () => { + const cwd = await directory(); + await NodeFSPromises.writeFile(NodePath.join(cwd, "tracked.txt"), "before\n"); + const host = tracker(); + const old = host.beginExecutionTurn({ threadId: "thread", executionId: "same", cwd, baselineRef: null }); + const oldBridge = new ExecutionFileObservationHandoff(host, old, 1); + expect(oldBridge.capture(mutation("same"))).toBe(true); + const current = host.beginExecutionTurn({ threadId: "thread", executionId: "same", cwd, baselineRef: null }); + const bridge = new ExecutionFileObservationHandoff(host, current, 2); + + expect(oldBridge.take(toolUse(mutation("same")), 1)).toBeNull(); + expect(bridge.capture(mutation("same"))).toBe(false); + expect(bridge.capture(mutation("same", "edit", 2))).toBe(true); + expect(bridge.take(toolUse(mutation("same")), 1)).toBeNull(); + expect(bridge.take(toolUse(mutation("same", "edit", 2)), 2)).toMatchObject({ + executionId: "same", generation: current.generation, + }); + }); + + it("fences captures and pending evidence after Stop", async () => { + const cwd = await directory(); + const host = tracker(); + const handoff = host.beginExecutionTurn({ threadId: "thread", executionId: "current", cwd, baselineRef: null }); + const bridge = new ExecutionFileObservationHandoff(host, handoff, 1); + const event = mutation("current"); + expect(bridge.capture(event)).toBe(true); + bridge.close(); + expect(bridge.take(toolUse(event), 1)).toBeNull(); + expect(bridge.capture(mutation("current", "later"))).toBe(false); + }); + + it("discards a colliding tool call without replacing its first pre-edit baseline", async () => { + const cwd = await directory(); + const host = tracker(); + const handoff = host.beginExecutionTurn({ threadId: "thread", executionId: "current", cwd, baselineRef: null }); + const bridge = new ExecutionFileObservationHandoff(host, handoff, 1); + const event = mutation("current"); + expect(bridge.capture(event)).toBe(true); + expect(bridge.capture(event)).toBe(false); + expect(bridge.take(toolUse(event), 1)).toBeNull(); + }); + + it("bounds pending observations and rejects a mutation without an exact attempt", async () => { + const cwd = await directory(); + const host = tracker(); + const handoff = host.beginExecutionTurn({ threadId: "thread", executionId: "current", cwd, baselineRef: null }); + const bridge = new ExecutionFileObservationHandoff(host, handoff, 1); + expect(bridge.capture({ ...mutation("current"), turnExecutionId: undefined })).toBe(false); + expect(bridge.capture({ ...mutation("current"), deliveryAttempt: undefined })).toBe(false); + for (let index = 0; index < 8; index += 1) { + expect(bridge.capture(mutation("current", `edit-${index}`))).toBe(true); + } + expect(bridge.capture(mutation("current", "overflow"))).toBe(false); + }); +}); diff --git a/apps/server/src/features/agents/execution/execution-file-observation-handoff.ts b/apps/server/src/features/agents/execution/execution-file-observation-handoff.ts new file mode 100644 index 000000000..2e99e5b81 --- /dev/null +++ b/apps/server/src/features/agents/execution/execution-file-observation-handoff.ts @@ -0,0 +1,94 @@ +import type { AgentEvent, ProviderFileMutationStart } from "@mcode/contracts"; + +import type { + CapturedToolUseObservation, + FileTurnHandoff, + TurnFileTracker, +} from "../turns/turn-file-tracker.js"; + +const MAX_PENDING = 8; +const MAX_PENDING_BYTES = 4 * 1_048_576; +const MAX_CAPTURE_BYTES = 1_310_720; +const MAX_SEEN = 1_024; + +type PendingObservation = { readonly captured: CapturedToolUseObservation; readonly bytes: number }; +type ToolUse = Extract; + +/** Keeps synchronous pre-edit evidence for one exact execution until its tool event is admitted. */ +export class ExecutionFileObservationHandoff { + private readonly pending = new Map(); + private readonly seen = new Set(); + private pendingBytes = 0; + private closed = false; + + constructor( + private readonly tracker: TurnFileTracker, + private readonly handoff: FileTurnHandoff, + private readonly deliveryAttempt: number, + ) { + if (!Number.isSafeInteger(deliveryAttempt) || deliveryAttempt < 1) { + throw new Error("A positive file observation delivery attempt is required"); + } + } + + /** Capture before returning to the provider notification loop; no asynchronous read precedes this call. */ + capture(event: ProviderFileMutationStart): boolean { + if (!this.matches(event)) return false; + const key = event.toolCallId; + if (this.seen.has(key)) { + this.forget(key); + return false; + } + if (this.seen.size >= MAX_SEEN) return false; + this.seen.add(key); + if (this.pending.size >= MAX_PENDING) return false; + const captured = this.tracker.captureToolUseObservation( + event.threadId, key, event.toolName, event.toolInput, + ); + if (!captured || !this.matchesCapture(captured)) return false; + const bytes = Buffer.byteLength(JSON.stringify(captured), "utf8"); + if (bytes > MAX_CAPTURE_BYTES || this.pendingBytes + bytes > MAX_PENDING_BYTES) return false; + this.pending.set(key, { captured, bytes }); + this.pendingBytes += bytes; + return true; + } + + /** The caller must supply the tool event's source attempt, not its current scheduler attempt. */ + take(event: ToolUse, deliveryAttempt: number): CapturedToolUseObservation | null { + if (!this.matches({ ...event, deliveryAttempt })) return null; + const pending = this.pending.get(event.toolCallId); + if (!pending) return null; + this.forget(event.toolCallId); + return pending.captured; + } + + /** Fence Stop, retry, worker loss, or terminal handling before any later provider callback. */ + close(): void { + this.closed = true; + this.pending.clear(); + this.seen.clear(); + this.pendingBytes = 0; + } + + private matches(event: { readonly threadId: string; readonly turnExecutionId?: string; + readonly deliveryAttempt?: number }): boolean { + return !this.closed && event.threadId === this.handoff.threadId + && event.turnExecutionId === this.handoff.executionId + && event.deliveryAttempt === this.deliveryAttempt + && this.tracker.getCurrentTurnId(this.handoff.threadId) === String(this.handoff.generation); + } + + private matchesCapture(captured: CapturedToolUseObservation): boolean { + return captured.executionId === this.handoff.executionId + && captured.generation === this.handoff.generation + && captured.generationToken === this.handoff.generationToken + && captured.canonicalRoot === this.handoff.canonicalRoot; + } + + private forget(key: string): void { + const pending = this.pending.get(key); + if (!pending) return; + this.pending.delete(key); + this.pendingBytes -= pending.bytes; + } +} diff --git a/packages/contracts/src/providers/interfaces.ts b/packages/contracts/src/providers/interfaces.ts index e213a5b30..6e3060bfa 100644 --- a/packages/contracts/src/providers/interfaces.ts +++ b/packages/contracts/src/providers/interfaces.ts @@ -32,6 +32,10 @@ export interface CompletionOptions { /** Explicit provider file-tool start used to capture a mutation baseline without publishing narrative UI. */ export interface ProviderFileMutationStart { threadId: string; + /** Exact dispatched attempt, when the provider can bind this private notification. */ + turnExecutionId?: string; + /** Provider-bound retry ordinal for this private notification. */ + deliveryAttempt?: number; toolCallId: string; toolName: string; toolInput: Record; diff --git a/packages/providers/src/__tests__/codex/codex-event-mapper.test.ts b/packages/providers/src/__tests__/codex/codex-event-mapper.test.ts index c69670f65..99f5ce345 100644 --- a/packages/providers/src/__tests__/codex/codex-event-mapper.test.ts +++ b/packages/providers/src/__tests__/codex/codex-event-mapper.test.ts @@ -3065,6 +3065,21 @@ describe("CodexEventMapper", () => { ]); }); + it("reports native turn identity synchronously with an early child file mutation start", () => { + const observed: Array<{ nativeThreadId?: string; nativeTurnId?: string; toolCallId: string }> = []; + mapper = new CodexEventMapper("test-thread", "parent-thread", (event) => observed.push(event)); + mapper.mapNotification({ + jsonrpc: "2.0", method: "item/started", + params: { threadId: "child-thread", turnId: "native-turn", item: { + type: "fileChange", id: "file-child", changes: [{ path: "src/child.ts", kind: "edit" }], + } }, + }); + expect(observed).toEqual([{ nativeThreadId: "child-thread", nativeTurnId: "native-turn", + toolCallId: "file-child", threadId: "test-thread", toolName: "file_change", + toolInput: expect.objectContaining({ changes: [{ path: "src/child.ts", kind: "edit" }] }), + }]); + }); + it("drops an unrelated unknown-thread notification instead of replaying it", () => { mapper = new CodexEventMapper("test-thread", "parent-thread"); const unknown = mapper.mapNotification({ diff --git a/packages/providers/src/__tests__/codex/codex-provider-subagent-turn.test.ts b/packages/providers/src/__tests__/codex/codex-provider-subagent-turn.test.ts index cb66dbdcc..2d1e0fda0 100644 --- a/packages/providers/src/__tests__/codex/codex-provider-subagent-turn.test.ts +++ b/packages/providers/src/__tests__/codex/codex-provider-subagent-turn.test.ts @@ -51,7 +51,7 @@ vi.mock("../../private/codex/codex-app-server.js", async () => { import { CodexProvider, stubEnvService } from "./codex-provider-test-fixture.js"; import { AgentEventType } from "@mcode/contracts"; -import type { ProviderRuntimeEvent } from "@mcode/contracts"; +import type { ProviderFileMutationStart, ProviderRuntimeEvent } from "@mcode/contracts"; interface PoolEntry { pendingTurnId: string | null; @@ -127,6 +127,33 @@ describe("CodexProvider sub-agent turn lifecycle isolation", () => { loggerWarnMock.mockClear(); }); + it("binds a private early child file mutation only to a known native turn and retry attempt", async () => { + const provider = makeProvider(); + const mutations: ProviderFileMutationStart[] = []; + provider.on("file_mutation_start", (event: ProviderFileMutationStart) => mutations.push(event)); + const entry = await startSession(provider, "mcode-file-attempt", "file-attempt"); + const state = entry as PoolEntry & { + turnExecutionIdsByNativeTurn: Map; + turnDiffRouting?: { turnId: string; turnExecutionId: string; deliveryAttempt: number }; + }; + state.turnExecutionIdsByNativeTurn.set("bound-native", "same-execution"); + state.turnDiffRouting = { turnId: "test-turn", turnExecutionId: "same-execution", deliveryAttempt: 2 }; + const item = { type: "fileChange", id: "edit", changes: [{ path: "tracked.txt", kind: "edit" }] }; + + entry.server.emit("notification", { method: "item/started", + params: { threadId: "early-child", turnId: "old-native", item } }); + entry.server.emit("notification", { method: "item/started", + params: { threadId: "early-child", turnId: "bound-native", item: { ...item, id: "edit-current" } } }); + + expect(mutations).toHaveLength(2); + expect(mutations[0]).toMatchObject({ toolCallId: "edit" }); + expect(mutations[0]?.turnExecutionId).toBeUndefined(); + expect(mutations[0]?.deliveryAttempt).toBeUndefined(); + expect(mutations[1]).toMatchObject({ + toolCallId: "edit-current", turnExecutionId: "same-execution", deliveryAttempt: 2, + }); + }); + it("assigns the active execution to main tool and text events before turn/start binds", async () => { const provider = makeProvider(); const events: ProviderRuntimeEvent[] = []; diff --git a/packages/providers/src/private/codex/codex-event-mapper.ts b/packages/providers/src/private/codex/codex-event-mapper.ts index 6ef45de6f..34a2d7ce8 100644 --- a/packages/providers/src/private/codex/codex-event-mapper.ts +++ b/packages/providers/src/private/codex/codex-event-mapper.ts @@ -37,6 +37,12 @@ type ChildNotificationContext = { nativeTurnId: string | undefined; }; +/** Native identity retained until the provider binds this private notification to an execution. */ +export interface CodexPendingFileMutationStart extends ProviderFileMutationStart { + nativeThreadId?: string; + nativeTurnId?: string; +} + const NOTICE_KIND_BY_SUBTYPE: Record = { "provider.notice.unknown-event": "diagnostic", }; @@ -225,7 +231,7 @@ export class CodexEventMapper { constructor( threadId: string, mainCodexThreadId?: string, - private readonly onPendingMutationStart?: (event: ProviderFileMutationStart) => void, + private readonly onPendingMutationStart?: (event: CodexPendingFileMutationStart) => void, ) { this.threadId = threadId; this.mainCodexThreadId = mainCodexThreadId; @@ -542,6 +548,8 @@ export class CodexEventMapper { if (!toolUse || toolUse.type !== AgentEventType.ToolUse) return; this.onPendingMutationStart({ threadId: toolUse.threadId, + nativeThreadId: this.notificationThreadId(notification), + nativeTurnId: this.nativeTurnId(notification), toolCallId: toolUse.toolCallId, toolName: toolUse.toolName, toolInput: toolUse.toolInput, diff --git a/packages/providers/src/private/codex/codex-provider.ts b/packages/providers/src/private/codex/codex-provider.ts index c9d649b53..e0ef4169c 100644 --- a/packages/providers/src/private/codex/codex-provider.ts +++ b/packages/providers/src/private/codex/codex-provider.ts @@ -42,6 +42,7 @@ import type { ProviderModelInfo, ProviderUsageInfo, ProviderRuntimeEvent, + ProviderFileMutationStart, ProviderTurnDiffUpdate, ProviderCapabilityName, } from "@mcode/contracts"; @@ -1283,7 +1284,7 @@ export class CodexProvider extends NodeEvents.EventEmitter implements IAgentProv const mcodeInstructions = this.codexSpawnInstructions(context); const server = this.createCodexAppServer(context, mcodeInstructions, browserTokenEnvName); - const mapper = this.createCodexEventMapper(threadId); + const mapper = this.createCodexEventMapper(threadId, context.sessionId); this.attachCodexServerEvents(context, server, mapper); const internalMcpStartup = internalMcp ? observeCodexInternalMcpStartup(server) : undefined; @@ -1405,8 +1406,21 @@ export class CodexProvider extends NodeEvents.EventEmitter implements IAgentProv return [...(context.internalMcp?.configOverrides ?? []), ...browserOverrides]; } - private createCodexEventMapper(threadId: string): CodexEventMapper { - const mapper = new CodexEventMapper(threadId, undefined, (event) => this.emit("file_mutation_start", event)); + private createCodexEventMapper(threadId: string, sessionId: string): CodexEventMapper { + const mapper = new CodexEventMapper(threadId, undefined, (event) => { + const state = this.runtime.get(sessionId); + const nativeTurnId = event.nativeTurnId; + const turnExecutionId = nativeTurnId && state?.mapper === mapper + && !state.nativeExecutionConflictKeys.has(nativeTurnId) + ? state.turnExecutionIdsByNativeTurn.get(nativeTurnId) : undefined; + const deliveryAttempt = turnExecutionId && state?.turnDiffRouting?.turnExecutionId === turnExecutionId + ? state.turnDiffRouting.deliveryAttempt : undefined; + this.emit("file_mutation_start", { + threadId: event.threadId, ...(deliveryAttempt ? { turnExecutionId, deliveryAttempt } : {}), + toolCallId: event.toolCallId, + toolName: event.toolName, toolInput: event.toolInput, + } satisfies ProviderFileMutationStart); + }); mapper.setOutputTruncationMode(this.outputTruncationMode); this.emitRuntimeEvent(mapper.sessionStartedEvent()); return mapper; From 4f15a3e3d06be552df93394a61ff554d2265dd7e Mon Sep 17 00:00:00 2001 From: "(CJ) Chukwudi Nwobodo" <142016413+chuks-qua@users.noreply.github.com> Date: Thu, 24 Sep 2026 22:41:12 +0100 Subject: [PATCH 097/136] fix(server): fence error publication behind terminal commit --- .../canonical-execution-semantic-writer.test.ts | 11 +++++++++++ .../canonical/canonical-execution-semantic-writer.ts | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/apps/server/src/features/agents/canonical/__tests__/canonical-execution-semantic-writer.test.ts b/apps/server/src/features/agents/canonical/__tests__/canonical-execution-semantic-writer.test.ts index 73483b2ee..a3d87fce6 100644 --- a/apps/server/src/features/agents/canonical/__tests__/canonical-execution-semantic-writer.test.ts +++ b/apps/server/src/features/agents/canonical/__tests__/canonical-execution-semantic-writer.test.ts @@ -547,6 +547,17 @@ describe("CanonicalExecutionSemanticWriter through ExecutionWorkerHandler", () = .get(TURN_ID)).toEqual({ count: 0 }); }); + it("rejects an error publication before terminal durability", async () => { + expect((await send(1, { kind: "start", providerId: "codex", input: startInput() })).kind).toBe("committed"); + const error: AgentEvent = { type: AgentEventType.Error, threadId: THREAD_ID, + turnExecutionId: EXECUTION_ID, error: "failed" }; + const op = { ...operation(2, { kind: "append-events", phase: "running", nativeCursor: null, + events: [runtimeEvent(2, error)] }), livePublication: [{ after: "writer" as const, event: error }] }; + expect(await writer.transact(op)).toEqual({ kind: "conflict", operationId: "lease-1:2" }); + expect(db.prepare("SELECT COUNT(*) AS count FROM canonical_writer_operation_receipts WHERE execution_id = ? AND operation_id = ?") + .get(EXECUTION_ID, "lease-1:2")).toEqual({ count: 0 }); + }); + it("rejects a compound live event that exceeds its shared row or byte budget", async () => { expect((await send(1, { kind: "start", providerId: "codex", input: startInput() })).kind).toBe("committed"); const event = { type: AgentEventType.TextDelta, threadId: THREAD_ID, diff --git a/apps/server/src/features/agents/canonical/canonical-execution-semantic-writer.ts b/apps/server/src/features/agents/canonical/canonical-execution-semantic-writer.ts index d23d8554c..2adee96e5 100644 --- a/apps/server/src/features/agents/canonical/canonical-execution-semantic-writer.ts +++ b/apps/server/src/features/agents/canonical/canonical-execution-semantic-writer.ts @@ -776,7 +776,7 @@ function validLivePublicationIntent( } function isTerminalLiveEvent(type: ExecutionLivePublicationIntent["event"]["type"]): boolean { - return type === "turnComplete" || type === "ended"; + return type === "turnComplete" || type === "error" || type === "ended"; } function validPublicationProvider(operation: ExecutionSemanticOperation, providerId: string): boolean { From 1e3b5c4ae7b7fbdb2a8d3550ab5f58bec3bf5990 Mon Sep 17 00:00:00 2001 From: "(CJ) Chukwudi Nwobodo" <142016413+chuks-qua@users.noreply.github.com> Date: Thu, 24 Sep 2026 22:42:27 +0100 Subject: [PATCH 098/136] feat(server): commit bound Codex system live events --- ...anonical-execution-semantic-writer.test.ts | 91 +++++++++++++++++++ ...canonical-codex-system-error-projection.ts | 65 +++++++++---- .../canonical-execution-semantic-writer.ts | 61 ++++++++++++- .../execution/execution-worker-handler.ts | 4 + 4 files changed, 202 insertions(+), 19 deletions(-) diff --git a/apps/server/src/features/agents/canonical/__tests__/canonical-execution-semantic-writer.test.ts b/apps/server/src/features/agents/canonical/__tests__/canonical-execution-semantic-writer.test.ts index a3d87fce6..a898caff6 100644 --- a/apps/server/src/features/agents/canonical/__tests__/canonical-execution-semantic-writer.test.ts +++ b/apps/server/src/features/agents/canonical/__tests__/canonical-execution-semantic-writer.test.ts @@ -9,11 +9,13 @@ import { afterEach, beforeEach, describe, expect, it } from "vitest"; import { openDatabase } from "../../../../runtime/persistence/sqlite/database.js"; import { MessageRepo } from "../../conversation/persistence/message-repo.js"; import { TaskRepo } from "../../orchestration/persistence/task-repo.js"; +import { CodexLiveEventReducer } from "../../execution/codex-live-event-reducer.js"; import type { ExecutionSemanticOperation } from "../../execution/execution-worker-handler.js"; import { ExecutionWorkerHandler } from "../../execution/execution-worker-handler.js"; import { ParentAssistantTextCheckpointService } from "../../turns/parent-assistant-text-checkpoint-service.js"; import { NarrativeRecoveryDelta } from "../../turns/narrative-recovery-delta.js"; import { CanonicalAgentBoundary } from "../canonical-agent-boundary.js"; +import type { CodexSystemWriterIntent } from "../canonical-codex-system-error-projection.js"; import { CanonicalExecutionSemanticWriter } from "../canonical-execution-semantic-writer.js"; import type { DataOnlyParentTurnStartInput } from "../canonical-parent-turn-write.js"; @@ -107,6 +109,18 @@ function operation(ordinal: number, mutation: ExecutionSemanticOperation["mutati return { operationId: `${lease.leaseId}:${ordinal}`, execution, lease, ordinal, mutation }; } +function systemLiveOperation(ordinal: number, systemEvent: Extract): ExecutionSemanticOperation { + const reduction = new CodexLiveEventReducer(execution).reduce(systemEvent); + if (reduction.kind !== "reduced" || reduction.publication.after !== "writer") throw new Error("Expected a bound live system event"); + const systemIntents = reduction.writer.filter((intent): intent is CodexSystemWriterIntent => + intent.kind === "notice-session" || intent.kind === "system-notice" || intent.kind === "session-cursor"); + if (systemIntents.length !== reduction.writer.length) throw new Error("Unexpected system reducer intent"); + return { + ...operation(ordinal, { kind: "live-event", text: { kind: "unchanged" }, systemIntents }), + livePublication: [reduction.publication], + }; +} + describe("CanonicalExecutionSemanticWriter through ExecutionWorkerHandler", () => { let directory: string; let path: string; @@ -512,6 +526,83 @@ describe("CanonicalExecutionSemanticWriter through ExecutionWorkerHandler", () = .get(EXECUTION_ID, "lease-1:2")).toEqual({ count: 1 }); }); + it("commits a system notice and its generated message ID in one replayable live receipt", async () => { + expect((await send(1, { kind: "start", providerId: "codex", input: startInput() })).kind).toBe("committed"); + const notice: Extract = { + type: "system", threadId: THREAD_ID, turnExecutionId: EXECUTION_ID, + subtype: "provider.notice.unknown-event", message: "Codex reported an update.", + systemNotice: { kind: "diagnostic", presentation: "timeline", scope: "turn", noticeKey: "notice-1" }, + }; + const op = systemLiveOperation(2, notice); + const publishedBefore = [...published]; + db.run("CREATE TRIGGER fail_system_receipt BEFORE INSERT ON canonical_writer_operation_receipts WHEN NEW.kind = 'semantic:live-event' BEGIN SELECT RAISE(ABORT, 'system receipt unavailable'); END"); + await expect(writer.transact(op)).rejects.toThrow("system receipt unavailable"); + expect(db.prepare("SELECT COUNT(*) AS count FROM messages WHERE thread_id = ? AND role = 'system'").get(THREAD_ID)) + .toEqual({ count: 0 }); + expect(db.prepare("SELECT COUNT(*) AS count FROM canonical_writer_operation_receipts WHERE execution_id = ? AND operation_id = ?") + .get(EXECUTION_ID, op.operationId)).toEqual({ count: 0 }); + expect(published).toEqual(publishedBefore); + db.run("DROP TRIGGER fail_system_receipt"); + + const receipt = await writer.transact(op); + expect(receipt).toMatchObject({ kind: "committed", livePublication: [{ publicationId: "lease-1:2:0", + after: "writer", event: { ...notice, messageId: expect.any(String) } }] }); + if (receipt.kind !== "committed") return; + const messageId = receipt.livePublication?.[0]?.event.type === "system" + ? receipt.livePublication[0].event.messageId : undefined; + expect(messageId).toBeDefined(); + if (!messageId) return; + expect(new MessageRepo(db).findByIdInThread(THREAD_ID, messageId)) + .toMatchObject({ role: "system", content: notice.message }); + expect(await writer.transact(op)).toEqual(receipt); + expect(db.prepare("SELECT COUNT(*) AS count FROM messages WHERE thread_id = ? AND role = 'system'").get(THREAD_ID)) + .toEqual({ count: 1 }); + + db.close(true); + db = openDatabase({ dbPath: path }); + writer = new CanonicalExecutionSemanticWriter(db, () => {}); + expect(await writer.transact(op)).toEqual(receipt); + const stored = db.prepare("SELECT receipt_json FROM canonical_writer_operation_receipts WHERE execution_id = ? AND operation_id = ?") + .get(EXECUTION_ID, op.operationId) as { receipt_json: string }; + const corrupted = JSON.parse(stored.receipt_json) as { livePublication: { event: { messageId: string } }[] }; + corrupted.livePublication[0]!.event.messageId = "not-a-message-id"; + db.prepare("UPDATE canonical_writer_operation_receipts SET receipt_json = ? WHERE execution_id = ? AND operation_id = ?") + .run(JSON.stringify(corrupted), EXECUTION_ID, op.operationId); + await expect(writer.transact(op)).rejects.toThrow("Invalid uuid"); + const altered = JSON.parse(stored.receipt_json) as { livePublication: { event: { message: string } }[] }; + altered.livePublication[0]!.event.message = "Different notice"; + db.prepare("UPDATE canonical_writer_operation_receipts SET receipt_json = ? WHERE execution_id = ? AND operation_id = ?") + .run(JSON.stringify(altered), EXECUTION_ID, op.operationId); + await expect(writer.transact(op)).rejects.toThrow("Live publication receipt does not match"); + }); + + it("fences session cursors to the selected execution in the compound live writer", async () => { + expect((await send(1, { kind: "start", providerId: "codex", input: startInput() })).kind).toBe("committed"); + const cursor: Extract = { + type: "system", threadId: THREAD_ID, turnExecutionId: EXECUTION_ID, subtype: "sdk_session_id:native-cursor", + }; + const unbound = { ...cursor, turnExecutionId: undefined }; + expect(await send(2, { kind: "live-event", text: { kind: "unchanged" }, + systemIntents: [{ kind: "session-cursor", event: unbound }], publication: { event: unbound, after: "writer" }, + })).toEqual({ kind: "rejected", reason: "writer-conflict" }); + expect(db.prepare("SELECT sdk_session_id FROM threads WHERE id = ?").get(THREAD_ID)) + .toEqual({ sdk_session_id: null }); + expect(await send(2, { kind: "live-event", text: { kind: "unchanged" }, + systemIntents: [{ kind: "session-cursor", event: { ...cursor, turnExecutionId: "other-execution" } }], + publication: { event: cursor, after: "writer" }, + })).toEqual({ kind: "rejected", reason: "writer-conflict" }); + expect(db.prepare("SELECT sdk_session_id FROM threads WHERE id = ?").get(THREAD_ID)) + .toEqual({ sdk_session_id: null }); + const receipt = await send(2, { kind: "live-event", text: { kind: "unchanged" }, + systemIntents: [{ kind: "session-cursor", event: cursor }], publication: { event: cursor, after: "writer" }, + }); + expect(receipt).toMatchObject({ kind: "committed", livePublication: [{ event: cursor }] }); + expect(db.prepare("SELECT sdk_session_id FROM threads WHERE id = ?").get(THREAD_ID)) + .toEqual({ sdk_session_id: "native-cursor" }); + expect(new CanonicalAgentBoundary(db, () => {}).loadCheckpoint(EXECUTION_ID)) + .toMatchObject({ nativeCursor: { providerId: "codex", scope: "thread", value: "native-cursor" } }); + }); + it("stores task-tool rows before the matching live tool publication", async () => { expect((await send(1, { kind: "start", providerId: "codex", input: startInput() })).kind).toBe("committed"); const toolUse = { type: AgentEventType.ToolUse, threadId: THREAD_ID, turnExecutionId: EXECUTION_ID, diff --git a/apps/server/src/features/agents/canonical/canonical-codex-system-error-projection.ts b/apps/server/src/features/agents/canonical/canonical-codex-system-error-projection.ts index 7ea785785..2b48eb507 100644 --- a/apps/server/src/features/agents/canonical/canonical-codex-system-error-projection.ts +++ b/apps/server/src/features/agents/canonical/canonical-codex-system-error-projection.ts @@ -1,15 +1,18 @@ import type { Database } from "bun:sqlite"; +import * as NodeUtil from "node:util"; import type { AgentEvent } from "@mcode/contracts"; import { ThreadRepo } from "../../thread-control/persistence/thread-repo.js"; import { MessageRepo } from "../conversation/persistence/message-repo.js"; import type { CodexLiveReduction, CodexLiveWriterIntent } from "../execution/codex-live-event-reducer.js"; +import type { ExecutionIdentity } from "../execution/execution-mailbox-protocol.js"; import type { DataOnlyParentTerminalProjectionInput } from "./canonical-parent-turn-write.js"; import { CanonicalAgentBoundary } from "./canonical-agent-boundary.js"; type Reduced = Extract; type SystemEvent = Extract; -type SystemIntent = Extract; +/** System intents accepted by the execution-bound live writer. */ +export type CodexSystemWriterIntent = Extract; type ErrorTerminalIntent = Extract; function errorTerminalIntent(reduction: Reduced, event: Extract): ErrorTerminalIntent { @@ -24,9 +27,31 @@ function errorTerminalIntent(reduction: Reduced, event: Extract; readonly after: "terminal"; readonly input: DataOnlyParentTerminalProjectionInput }; +type SystemProjectionResult = { readonly kind: "system"; readonly event: SystemEvent; readonly after: "writer" | "terminal" }; + +function expectedSystemIntentKind(event: SystemEvent): CodexSystemWriterIntent["kind"] | null { + if (event.subtype === "provider.session.started") return "notice-session"; + if (event.subtype.startsWith("provider.notice.") && event.message) return "system-notice"; + if (event.subtype.startsWith("sdk_session_id:") || event.subtype === "sdk_session_invalidated") return "session-cursor"; + return null; +} + +/** Check that a bound system event carries exactly its reducer-owned projection intent. */ +export function matchesCodexSystemIntents(event: SystemEvent, intents: readonly CodexSystemWriterIntent[]): boolean { + const expected = expectedSystemIntentKind(event); + if (!Array.isArray(intents) || intents.length !== (expected ? 1 : 0)) return false; + if (!expected) return true; + return intents[0]?.kind === expected && NodeUtil.isDeepStrictEqual(intents[0].event, event) + && (expected !== "system-notice" || !event.messageId); +} + +function isSystemWriterIntent(intent: CodexLiveWriterIntent): intent is CodexSystemWriterIntent { + return intent.kind === "notice-session" || intent.kind === "system-notice" || intent.kind === "session-cursor"; +} + /** Projects only execution-bound Codex system events and prepares errors for the terminal owner. */ export class CanonicalCodexSystemErrorProjection { private readonly threads: ThreadRepo; @@ -46,29 +71,37 @@ export class CanonicalCodexSystemErrorProjection { if (event.threadId !== reduction.execution.threadId || event.turnExecutionId !== reduction.execution.executionId) { throw new Error("Codex event lacks exact execution ownership"); } - if (event.type === "system") return this.projectSystem(reduction, event); + if (event.type === "system") { + if (!reduction.writer.every(isSystemWriterIntent)) throw new Error("Codex system has a non-system intent"); + return this.projectBoundSystem(reduction.execution, event, reduction.writer, reduction.publication.after); + } if (event.type === "error") return this.prepareError(reduction, event, endedAt); throw new Error(`Codex ${event.type} needs another feature owner`); } - private projectSystem(reduction: Reduced, event: SystemEvent): CanonicalCodexSystemErrorResult { + /** Apply one bound system event and return the event that the committed receipt must publish. */ + projectBoundSystem( + execution: ExecutionIdentity, + event: SystemEvent, + intents: readonly CodexSystemWriterIntent[], + after: "writer" | "terminal", + ): SystemProjectionResult { + if (event.threadId !== execution.threadId || event.turnExecutionId !== execution.executionId) { + throw new Error("Codex system event lacks exact execution ownership"); + } + if (!matchesCodexSystemIntents(event, intents)) { + throw new Error("Codex system intents do not match the event"); + } return this.db.transaction(() => { let published: SystemEvent = { ...event }; - for (const intent of reduction.writer) { - if (!this.isSystemIntent(intent) || JSON.stringify(intent.event) !== JSON.stringify(event)) { - throw new Error("Codex system intent does not match its publication"); - } - published = this.applySystemIntent(reduction, intent, published); + for (const intent of intents) { + published = this.applySystemIntent(execution.executionId, intent, published); } - return structuredClone({ kind: "system", event: published, after: reduction.publication.after } satisfies CanonicalCodexSystemErrorResult); + return structuredClone({ kind: "system", event: published, after } satisfies SystemProjectionResult); })(); } - private isSystemIntent(intent: CodexLiveWriterIntent): intent is SystemIntent { - return intent.kind === "notice-session" || intent.kind === "system-notice" || intent.kind === "session-cursor"; - } - - private applySystemIntent(reduction: Reduced, intent: SystemIntent, event: SystemEvent): SystemEvent { + private applySystemIntent(executionId: string, intent: CodexSystemWriterIntent, event: SystemEvent): SystemEvent { switch (intent.kind) { case "notice-session": if (event.subtype !== "provider.session.started") throw new Error("Codex notice session subtype is invalid"); @@ -81,7 +114,7 @@ export class CanonicalCodexSystemErrorProjection { return { ...event, messageId: message.id }; } case "session-cursor": - this.applyCursor(reduction.execution.executionId, event); + this.applyCursor(executionId, event); return event; } } diff --git a/apps/server/src/features/agents/canonical/canonical-execution-semantic-writer.ts b/apps/server/src/features/agents/canonical/canonical-execution-semantic-writer.ts index 2adee96e5..46f193a44 100644 --- a/apps/server/src/features/agents/canonical/canonical-execution-semantic-writer.ts +++ b/apps/server/src/features/agents/canonical/canonical-execution-semantic-writer.ts @@ -24,6 +24,7 @@ import type { import type { CanonicalAgentEventPublisher } from "./canonical-agent-boundary.js"; import { CanonicalAgentBoundary } from "./canonical-agent-boundary.js"; import { CanonicalCommittedProviderProjector } from "./canonical-committed-provider-projector.js"; +import { CanonicalCodexSystemErrorProjection, matchesCodexSystemIntents } from "./canonical-codex-system-error-projection.js"; import { CanonicalContextCompactionProjection } from "./canonical-context-compaction-projection.js"; import { CanonicalParentTurnWrite } from "./canonical-parent-turn-write.js"; import { TaskRepo } from "../orchestration/persistence/task-repo.js"; @@ -170,6 +171,7 @@ export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter private readonly turns: CanonicalParentTurnWrite; private readonly providerProjector: CanonicalCommittedProviderProjector; private readonly contextCompaction: CanonicalContextCompactionProjection; + private readonly systemProjection: CanonicalCodexSystemErrorProjection; private readonly tasks: TaskRepo; private readonly assistantText: ParentAssistantTextCheckpointService; private readonly canonical: CanonicalAgentBoundary; @@ -197,6 +199,7 @@ export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter this.canonical = codexBoundary; this.providerProjector = new CanonicalCommittedProviderProjector(codexBoundary); this.contextCompaction = new CanonicalContextCompactionProjection(db); + this.systemProjection = new CanonicalCodexSystemErrorProjection(db); this.tasks = new TaskRepo(db); this.assistantText = new ParentAssistantTextCheckpointService(db); this.findOperation = db.prepare("SELECT kind, input_hash, receipt_json FROM canonical_writer_operation_receipts WHERE execution_id = ? AND operation_id = ?"); @@ -424,7 +427,8 @@ export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter if (mutation.text.kind === "reclassify" && !this.assistantText.resetInTransaction(operation.execution.executionId)) { throw new SemanticConflict(); } - const committedReceipt = committed(operation, head.durableRevision, undefined, undefined, textResult); + const livePublication = this.projectLiveSystem(operation); + const committedReceipt = committed(operation, head.durableRevision, undefined, undefined, textResult, livePublication); this.storeHead({ ...head, ordinal: operation.ordinal }); this.storeReceipt(operation, hash, committedReceipt); return committedReceipt; @@ -435,6 +439,17 @@ export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter return receipt; } + private projectLiveSystem(operation: ExecutionSemanticOperation): readonly ExecutionLivePublicationReceipt[] | undefined { + const mutation = operation.mutation; + if (mutation.kind !== "live-event" || mutation.systemIntents === undefined) return undefined; + const publication = liveEventPublication(operation); + if (publication?.event.type !== "system") throw new SemanticConflict(); + const result = this.systemProjection.projectBoundSystem( + operation.execution, publication.event, mutation.systemIntents, publication.after, + ); + return [{ publicationId: `${operation.operationId}:0`, after: result.after, event: result.event }]; + } + private applyLiveText( text: Extract["text"], executionId: string, @@ -670,7 +685,7 @@ export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter throw new Error("Live publication receipt exceeds its size limit"); } const receipt = storedReceiptSchema.parse(JSON.parse(stored.receipt_json)); - if (fingerprint(receipt.livePublication ?? null) !== fingerprint(livePublicationFor(operation) ?? null)) { + if (!this.matchesLivePublicationReceipt(operation, receipt.livePublication)) { throw new Error("Live publication receipt does not match its operation"); } return receipt.operationId === operation.operationId && Number.isSafeInteger(receipt.durableRevision) @@ -678,6 +693,28 @@ export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter receipt.assistantTextCheckpoint, receipt.livePublication) : conflict(operation); } + private matchesLivePublicationReceipt( + operation: ExecutionSemanticOperation, + actual: readonly ExecutionLivePublicationReceipt[] | undefined, + ): boolean { + const mutation = operation.mutation; + if (mutation.kind === "live-event" && mutation.systemIntents?.[0]?.kind === "system-notice") { + return this.matchesGeneratedNoticeReceipt(operation, actual); + } + return fingerprint(actual ?? null) === fingerprint(livePublicationFor(operation) ?? null); + } + + private matchesGeneratedNoticeReceipt( + operation: ExecutionSemanticOperation, + actual: readonly ExecutionLivePublicationReceipt[] | undefined, + ): boolean { + const expected = livePublicationFor(operation)?.[0]; + const published = actual?.[0]; + if (actual?.length !== 1 || expected?.event.type !== "system" || expected.event.messageId + || !isGeneratedNoticePublication(published)) return false; + return fingerprint({ ...published, event: { ...published.event, messageId: undefined } }) === fingerprint(expected); + } + private publishStoredEvents(executionId: string, hash: string): void { const prefix = publicationChunkPrefix(hash); let cursor = prefix; @@ -800,6 +837,7 @@ function validLiveEventInput(operation: ExecutionSemanticOperation): boolean { const publication = liveEventPublication(operation); if (!publication) return false; const event = publication.event; + if (!validLiveSystemMutation(mutation, event)) return false; if (!validLiveTextPayload(mutation, operation.execution) || !AgentEventSchema().safeParse(event).success || !validLiveTextAssociation(mutation.text, event)) return false; if (mutation.narrative && !validNarrativeDeltaInput(mutation.narrative, operation.execution)) return false; @@ -807,6 +845,16 @@ function validLiveEventInput(operation: ExecutionSemanticOperation): boolean { return validLiveEventBudget(operation, mutation); } +function validLiveSystemMutation( + mutation: Extract, + event: ExecutionLivePublicationIntent["event"], +): boolean { + if (event.type !== "system") return mutation.systemIntents === undefined; + return Array.isArray(mutation.systemIntents) && mutation.systemIntents.length <= 1 + && mutation.text.kind === "unchanged" && mutation.narrative === undefined && mutation.taskIntents === undefined + && matchesCodexSystemIntents(event, mutation.systemIntents); +} + function validLiveTaskIntents( intents: readonly TaskToolWriteIntent[] | undefined, event: ExecutionLivePublicationIntent["event"], @@ -860,7 +908,7 @@ function validLiveTextAssociation( event: ExecutionLivePublicationIntent["event"], ): boolean { switch (text.kind) { - case "unchanged": return validNarrativeEvent(event); + case "unchanged": return event.type === "system" || validNarrativeEvent(event); case "append": return validAppendEvent(event, text.inputs); case "reclassify": return event.type === "assistantMessageBoundary" && event.isFinalResponse === false && text.expectedText.length > 0; @@ -999,6 +1047,13 @@ function livePublicationFor(operation: ExecutionSemanticOperation): readonly Exe })); } +function isGeneratedNoticePublication( + receipt: ExecutionLivePublicationReceipt | undefined, +): receipt is ExecutionLivePublicationReceipt & { readonly event: Extract & { readonly messageId: string } } { + return receipt?.event.type === "system" && typeof receipt.event.messageId === "string" + && z.string().uuid().safeParse(receipt.event.messageId).success; +} + function conflict(operation: ExecutionSemanticOperation): Extract { return { kind: "conflict", operationId: operation.operationId }; } diff --git a/apps/server/src/features/agents/execution/execution-worker-handler.ts b/apps/server/src/features/agents/execution/execution-worker-handler.ts index 82026b78f..da5574507 100644 --- a/apps/server/src/features/agents/execution/execution-worker-handler.ts +++ b/apps/server/src/features/agents/execution/execution-worker-handler.ts @@ -6,6 +6,7 @@ import type { DataOnlyParentTurnFinishInput, DataOnlyParentTurnStartInput, } from "../canonical/canonical-parent-turn-write.js"; +import type { CodexSystemWriterIntent } from "../canonical/canonical-codex-system-error-projection.js"; import type { CanonicalAgentCommitResult } from "../canonical/canonical-agent-boundary.js"; import type { ProviderEventIngressEvent } from "../../providers/composition/provider-event-ingress.js"; import type { @@ -38,6 +39,7 @@ export type ExecutionWorkCommand = | { readonly kind: "promote"; readonly input: ParentAssistantTextCheckpointInput }; readonly narrative?: ParentNarrativeRecoveryCommit; readonly taskIntents?: readonly TaskToolWriteIntent[]; + readonly systemIntents?: readonly CodexSystemWriterIntent[]; readonly publication: ExecutionLivePublicationIntent; } | { readonly kind: "checkpoint"; readonly phase: string; readonly nativeCursor: unknown | null } @@ -66,6 +68,7 @@ export interface ExecutionSemanticOperation { readonly text: Extract["text"]; readonly narrative?: ParentNarrativeRecoveryCommit; readonly taskIntents?: readonly TaskToolWriteIntent[]; + readonly systemIntents?: readonly CodexSystemWriterIntent[]; } | { readonly kind: "checkpoint"; readonly phase: string; readonly nativeCursor: unknown | null } | { readonly kind: "stop-requested"; readonly requestId: string; readonly lastAdmittedOrdinal: number } @@ -293,6 +296,7 @@ function liveEventMutation( kind: "live-event", text: command.text, ...(command.narrative ? { narrative: command.narrative } : {}), ...(command.taskIntents ? { taskIntents: command.taskIntents } : {}), + ...(command.systemIntents ? { systemIntents: command.systemIntents } : {}), }; } From e04681dd1d6855e7340e6e2ac3ff587fc983f893 Mon Sep 17 00:00:00 2001 From: "(CJ) Chukwudi Nwobodo" <142016413+chuks-qua@users.noreply.github.com> Date: Thu, 24 Sep 2026 22:38:54 +0100 Subject: [PATCH 099/136] feat(server): project Codex plan intents as bounded execution data --- .../codex-live-event-reducer.test.ts | 47 +++++++++++++++++++ .../execution/codex-live-event-reducer.ts | 40 ++++++++++++++-- 2 files changed, 83 insertions(+), 4 deletions(-) diff --git a/apps/server/src/features/agents/execution/__tests__/codex-live-event-reducer.test.ts b/apps/server/src/features/agents/execution/__tests__/codex-live-event-reducer.test.ts index 6bf818589..d73f65b1f 100644 --- a/apps/server/src/features/agents/execution/__tests__/codex-live-event-reducer.test.ts +++ b/apps/server/src/features/agents/execution/__tests__/codex-live-event-reducer.test.ts @@ -18,6 +18,53 @@ function boundByProvider(mapped: AgentEvent): AgentEvent { } describe("CodexLiveEventReducer", () => { + it("carries parsed plan questions as data on the completing text event", () => { + const reducer = new CodexLiveEventReducer(execution, "questions"); + const question = { id: "q1", category: "AUTH", question: "Which login?", options: [ + { id: "o1", title: "Passkey", description: "Use passkeys.", recommended: true }, + { id: "o2", title: "Password", description: "Use passwords." }, + ] }; + const block = `\`\`\`plan-questions\n${JSON.stringify([question])}\n\`\`\``; + expect(reducer.reduce(event("turnStarted")).kind).toBe("reduced"); + const first = reducer.reduce(event("textDelta", { delta: block.slice(0, 20) })); + expect(first.kind).toBe("reduced"); + if (first.kind !== "reduced") return; + expect(first.writer).not.toContainEqual(expect.objectContaining({ kind: "plan-questions" })); + const second = reducer.reduce(event("textDelta", { delta: block.slice(20) })); + expect(second.kind).toBe("reduced"); + if (second.kind !== "reduced") return; + expect(second.writer).toContainEqual({ kind: "plan-questions", questions: [question] }); + expect(second.publication.after).toBe("writer"); + expect(reducer.reduce(event("textDelta", { delta: block }))).toMatchObject({ + kind: "reduced", writer: expect.not.arrayContaining([{ kind: "plan-questions", questions: [question] }]), + }); + }); + + it("carries plan output data with the assistant body and bounds parser input", () => { + const reducer = new CodexLiveEventReducer(execution, "output"); + const plan = { title: "Login plan", sections: [ + { id: "s1", title: "Implementation", level: 1, content: "Add passkey login." }, + ] }; + const block = `\`\`\`plan-output\n${JSON.stringify(plan)}\n\`\`\``; + expect(reducer.reduce(event("turnStarted")).kind).toBe("reduced"); + expect(reducer.reduce(event("textDelta", { delta: block })).kind).toBe("reduced"); + const message = reducer.reduce(event("message", { content: "Provider prose", tokens: null })); + expect(message.kind).toBe("reduced"); + if (message.kind !== "reduced") return; + expect(message.writer).toContainEqual({ kind: "plan-output", output: { + title: "Login plan", contentMd: "## Implementation\n\nAdd passkey login.", + sectionsJson: '[{"id":"s1","title":"Implementation","level":1}]', changeSummary: null, + } }); + expect(message.writer[0]).toMatchObject({ kind: "assistant-body", content: "Provider prose" }); + + const bounded = new CodexLiveEventReducer(execution, "questions"); + expect(bounded.reduce(event("turnStarted")).kind).toBe("reduced"); + expect(bounded.reduce(event("textDelta", { delta: "x".repeat(256 * 1024) })).kind).toBe("reduced"); + expect(bounded.reduce(event("textDelta", { delta: "x" }))).toEqual({ + kind: "unsupported", eventType: "textDelta", reason: "plan text exceeds the execution projection limit", + }); + }); + it("reduces a Codex notification sequence through tool narration and final answer", () => { const mapper = new CodexEventMapper(execution.threadId); const reducer = new CodexLiveEventReducer(execution); diff --git a/apps/server/src/features/agents/execution/codex-live-event-reducer.ts b/apps/server/src/features/agents/execution/codex-live-event-reducer.ts index 6ed0ca625..9ff0fda9e 100644 --- a/apps/server/src/features/agents/execution/codex-live-event-reducer.ts +++ b/apps/server/src/features/agents/execution/codex-live-event-reducer.ts @@ -1,6 +1,7 @@ -import type { AgentEvent, ParentNarrativeRecoveryItem, StoredAttachment } from "@mcode/contracts"; +import type { AgentEvent, ParentNarrativeRecoveryItem, PlanQuestion, StoredAttachment } from "@mcode/contracts"; import { NarrativeTurnState, type NarrativeTurnStateEffect } from "../conversation/narrative/narrative-turn-state.js"; import { AssistantExecutionState, type AssistantMaterializationInput } from "../turns/assistant-execution-state.js"; +import { PlanExecutionState, type PlanPersistenceReady } from "../planning/plan-execution-state.js"; import type { ExecutionIdentity } from "./execution-mailbox-protocol.js"; type ToolUseEvent = Extract; @@ -9,6 +10,10 @@ type MessageEvent = Extract; type TextDeltaEvent = Extract; type ContextEvent = Extract; type SystemEvent = Extract; +const MAX_PLAN_TEXT_BYTES = 256 * 1024; + +/** Which plan parser, if any, was armed when this execution began. */ +export type CodexPlanFeature = "none" | "questions" | "output"; const BEFORE_START_EVENTS = new Set([ "system", "quotaUpdate", "goalUpdated", "goalCleared", "mcpServerStartupStatus", @@ -68,6 +73,8 @@ export type CodexLiveWriterIntent = | { readonly kind: "narrative-recovery"; readonly items: ParentNarrativeRecoveryItem[] } | { readonly kind: "narrative-effect"; readonly effect: NarrativeTurnStateEffect } | { readonly kind: "feature-event"; readonly feature: "plan-text" | "assistant-message" | "task-tool" | "goal-refresh"; readonly event: AgentEvent } + | { readonly kind: "plan-questions"; readonly questions: readonly PlanQuestion[] } + | { readonly kind: "plan-output"; readonly output: PlanPersistenceReady } | { readonly kind: "context-usage"; readonly tokensIn: number; readonly contextWindow?: number } | { readonly kind: "compaction-started" } | { readonly kind: "compaction-divider" } @@ -104,14 +111,20 @@ export class CodexLiveEventReducer { private unknownText = ""; private knownFinalText = false; private compacting = false; + private readonly plan: PlanExecutionState | null; + private planTextBytes = 0; + private planQuestionsResolved = false; - constructor(readonly execution: ExecutionIdentity) { + constructor(readonly execution: ExecutionIdentity, readonly planFeature: CodexPlanFeature = "none") { this.narrative = new NarrativeTurnState(execution); + this.plan = planFeature === "none" ? null : new PlanExecutionState(); + if (planFeature === "questions") this.plan?.beginQuestionGeneration(); + if (planFeature === "output") this.plan?.beginOutputGeneration(); } reduce(input: AgentEvent): CodexLiveReduction { const rejection = this.identityRejection(input) ?? UNSUPPORTED_FEATURE_REASON[input.type] - ?? this.phaseRejection(input) ?? this.textRejection(input); + ?? this.phaseRejection(input) ?? this.textRejection(input) ?? this.planTextRejection(input); if (rejection) return this.unsupported(input, rejection); let event: AgentEvent; try { @@ -178,6 +191,12 @@ export class CodexLiveEventReducer { return undefined; } + private planTextRejection(event: AgentEvent): string | undefined { + if (!this.plan || this.planQuestionsResolved || event.type !== "textDelta") return undefined; + return this.planTextBytes + Buffer.byteLength(event.delta, "utf8") > MAX_PLAN_TEXT_BYTES + ? "plan text exceeds the execution projection limit" : undefined; + } + private publicationEvent(event: AgentEvent): AgentEvent { return event.type === "ended" && event.outcome === "cancelled" ? { ...event, outcome: "interrupted" } @@ -235,6 +254,14 @@ export class CodexLiveEventReducer { private textDelta(event: TextDeltaEvent): CodexLiveWriterIntent[] { const writer: CodexLiveWriterIntent[] = [{ kind: "feature-event", feature: "plan-text", event }]; + if (this.plan && !this.planQuestionsResolved) { + this.planTextBytes += Buffer.byteLength(event.delta, "utf8"); + const ready = this.plan.feedText(event.delta); + if (ready) { + this.planQuestionsResolved = true; + writer.push({ kind: "plan-questions", questions: ready.questions }); + } + } if (event.isFinalResponse === false) { this.narrative.openOrExtendThought(event.threadId, event.delta); writer.push(this.recovery()); @@ -278,10 +305,15 @@ export class CodexLiveEventReducer { this.narrative.clearAgentStackOnMessage(event.threadId); this.knownFinalText = false; this.unknownText = ""; - return [ + const writer: CodexLiveWriterIntent[] = [ { kind: "assistant-body", content: body.content, model: body.model, attachments: body.attachments, tokens: event.tokens }, { kind: "feature-event", feature: "assistant-message", event }, ]; + if (this.plan && this.planFeature === "output") { + const output = this.plan.consumeAssistantMessage(event.content); + if (output) writer.push({ kind: "plan-output", output }); + } + return writer; } private attachment(event: Extract): CodexLiveWriterIntent[] { From fb592a08e7e308112896616d6dba4d39a4ab34d7 Mon Sep 17 00:00:00 2001 From: "(CJ) Chukwudi Nwobodo" <142016413+chuks-qua@users.noreply.github.com> Date: Thu, 24 Sep 2026 22:43:01 +0100 Subject: [PATCH 100/136] feat(server): stage Codex live message before publication --- ...anonical-execution-semantic-writer.test.ts | 98 +++++++++++++++++++ .../canonical-execution-semantic-writer.ts | 66 +++++++++++-- .../canonical/canonical-parent-turn-write.ts | 70 ++++++++++++- .../execution/execution-worker-handler.ts | 4 + 4 files changed, 229 insertions(+), 9 deletions(-) diff --git a/apps/server/src/features/agents/canonical/__tests__/canonical-execution-semantic-writer.test.ts b/apps/server/src/features/agents/canonical/__tests__/canonical-execution-semantic-writer.test.ts index a898caff6..22c1af269 100644 --- a/apps/server/src/features/agents/canonical/__tests__/canonical-execution-semantic-writer.test.ts +++ b/apps/server/src/features/agents/canonical/__tests__/canonical-execution-semantic-writer.test.ts @@ -13,6 +13,7 @@ import { CodexLiveEventReducer } from "../../execution/codex-live-event-reducer. import type { ExecutionSemanticOperation } from "../../execution/execution-worker-handler.js"; import { ExecutionWorkerHandler } from "../../execution/execution-worker-handler.js"; import { ParentAssistantTextCheckpointService } from "../../turns/parent-assistant-text-checkpoint-service.js"; +import { CodexParentMessageProjection } from "../../turns/codex-parent-message-projection.js"; import { NarrativeRecoveryDelta } from "../../turns/narrative-recovery-delta.js"; import { CanonicalAgentBoundary } from "../canonical-agent-boundary.js"; import type { CodexSystemWriterIntent } from "../canonical-codex-system-error-projection.js"; @@ -344,6 +345,103 @@ describe("CanonicalExecutionSemanticWriter through ExecutionWorkerHandler", () = .toMatchObject({ id: assignedId, content: "Live answer", is_internal: false }); }); + it("stages a projected Codex message with its receipt before live publication and reuses it at finish", async () => { + expect((await send(1, { kind: "start", providerId: "codex", input: startInput() })).kind).toBe("committed"); + const projection = new CodexParentMessageProjection({ execution, turnKind: "ordinary" }); + const projected = projection.projectMessage({ + execution, content: "Live answer", model: null, precedingMessageId: "user-1", postTurnGoalReceipt: false, + }); + const message = { + precedingMessageId: "user-1", messageId: projected.messageId, + content: "Live answer", model: projected.model, attachments: projected.attachments ?? [], + }; + const event = { type: AgentEventType.Message, threadId: THREAD_ID, turnExecutionId: EXECUTION_ID, + content: message.content, tokens: null, messageId: message.messageId, model: message.model }; + const liveOperation = { ...operation(2, { kind: "live-event", text: { kind: "unchanged" }, message }), + livePublication: [{ after: "writer" as const, event }] }; + + expect(await writer.transact({ ...liveOperation, livePublication: [{ after: "writer", + event: { ...event, content: "Different public body" } }] })) + .toEqual({ kind: "conflict", operationId: "lease-1:2" }); + expect(new MessageRepo(db).findByIdInThreadIncludingInternal(THREAD_ID, message.messageId)).toBeNull(); + + db.run("CREATE TRIGGER fail_message_receipt BEFORE INSERT ON canonical_writer_operation_receipts WHEN NEW.kind = 'semantic:live-event' AND NEW.operation_id = 'lease-1:2' BEGIN SELECT RAISE(ABORT, 'message receipt unavailable'); END"); + await expect(writer.transact(liveOperation)).rejects.toThrow("message receipt unavailable"); + expect(new MessageRepo(db).findByIdInThreadIncludingInternal(THREAD_ID, message.messageId)).toBeNull(); + expect(db.prepare("SELECT COUNT(*) AS count FROM canonical_writer_operation_receipts WHERE execution_id = ? AND operation_id = ?") + .get(EXECUTION_ID, "lease-1:2")).toEqual({ count: 0 }); + db.run("DROP TRIGGER fail_message_receipt"); + + const committedMessage = await send(2, { + kind: "live-event", text: { kind: "unchanged" }, message, + publication: { after: "writer", event }, + }); + expect(committedMessage).toMatchObject({ kind: "committed", livePublication: [{ + publicationId: "lease-1:2:0", event, + }] }); + expect(new MessageRepo(db).findByIdInThreadIncludingInternal(THREAD_ID, message.messageId)) + .toMatchObject({ id: message.messageId, content: message.content, model: null, is_internal: true }); + + db.close(true); + db = openDatabase({ dbPath: path }); + writer = new CanonicalExecutionSemanticWriter(db, () => {}); + expect(await writer.transact(liveOperation)).toEqual(committedMessage); + expect(db.prepare("SELECT COUNT(*) AS count FROM messages WHERE id = ?").get(message.messageId)) + .toEqual({ count: 1 }); + expect(await writer.transact({ ...liveOperation, mutation: { kind: "live-event", text: { kind: "unchanged" }, + message: { ...message, content: "Different body" } }, + livePublication: [{ after: "writer", event: { ...event, content: "Different body" } }] })) + .toEqual({ kind: "conflict", operationId: "lease-1:2" }); + expect(await writer.transact({ ...liveOperation, ordinal: 3, operationId: "lease-1:3", + mutation: { kind: "live-event", text: { kind: "unchanged" }, message: { ...message, content: "Different body" } }, + livePublication: [{ after: "writer", event: { ...event, content: "Different body" } }], + })).toEqual({ kind: "conflict", operationId: "lease-1:3" }); + expect(new MessageRepo(db).findByIdInThreadIncludingInternal(THREAD_ID, message.messageId)) + .toMatchObject({ content: "Live answer" }); + + expect((await writer.transact(operation(3, { kind: "provider-outcome", outcome: "completed" }))).kind) + .toBe("committed"); + const terminal = projection.projectTerminal({ + execution, outcome: "completed", endedAt: NOW, fallbackModel: null, narrative: [], + }); + expect(await writer.transact(operation(4, { kind: "stage-terminal", input: { + ...terminal, assistant: { ...terminal.assistant, messageId: "b".repeat(64) }, + } }))).toEqual({ kind: "conflict", operationId: "lease-1:4" }); + expect((await writer.transact(operation(4, { kind: "stage-terminal", input: terminal }))).kind) + .toBe("committed"); + expect((await writer.transact(operation(5, { kind: "finish", outcome: "completed", input: { + threadId: THREAD_ID, turnId: TURN_ID, executionId: EXECUTION_ID, + providerId: "codex", providerIdentities: [], outcome: "completed", + projection: { kind: "writer-staged", messageId: message.messageId }, + } }))).kind).toBe("committed"); + expect(new MessageRepo(db).findByIdInThread(THREAD_ID, message.messageId)) + .toMatchObject({ id: message.messageId, content: "Live answer", outcome: "completed", is_internal: false }); + expect(db.prepare("SELECT COUNT(*) AS count FROM messages WHERE id = ?").get(message.messageId)) + .toEqual({ count: 1 }); + }); + + it("uses a live message row with its assigned ID when the worker is lost", async () => { + expect((await send(1, { kind: "start", providerId: "codex", input: startInput() })).kind).toBe("committed"); + const projection = new CodexParentMessageProjection({ execution, turnKind: "ordinary" }); + const projected = projection.projectMessage({ + execution, content: "Durable body", model: null, precedingMessageId: "user-1", postTurnGoalReceipt: false, + }); + const message = { + precedingMessageId: "user-1", messageId: projected.messageId, + content: "Durable body", model: projected.model, attachments: projected.attachments ?? [], + }; + expect((await send(2, { kind: "live-event", text: { kind: "unchanged" }, message, + publication: { after: "writer", event: { type: AgentEventType.Message, + threadId: THREAD_ID, turnExecutionId: EXECUTION_ID, content: message.content, + tokens: null, messageId: message.messageId, model: message.model } }, + })).kind).toBe("committed"); + expect(writer.interruptWorkerLoss(loss).kind).toBe("committed"); + expect(new MessageRepo(db).findByIdInThread(THREAD_ID, message.messageId)) + .toMatchObject({ content: "Durable body", outcome: "interrupted", is_internal: false }); + expect(db.prepare("SELECT COUNT(*) AS count FROM messages WHERE thread_id = ? AND role = 'assistant'") + .get(THREAD_ID)).toEqual({ count: 1 }); + }); + it("fences assistant-text routing, lease, ordinal, input size, and conflicting replay", async () => { expect((await send(1, { kind: "start", providerId: "codex", input: startInput() })).kind).toBe("committed"); const inputs = [{ ...execution, sequence: 1, text: "Saved text" }]; diff --git a/apps/server/src/features/agents/canonical/canonical-execution-semantic-writer.ts b/apps/server/src/features/agents/canonical/canonical-execution-semantic-writer.ts index 46f193a44..bf24715be 100644 --- a/apps/server/src/features/agents/canonical/canonical-execution-semantic-writer.ts +++ b/apps/server/src/features/agents/canonical/canonical-execution-semantic-writer.ts @@ -2,6 +2,7 @@ import type { Database } from "bun:sqlite"; import * as NodeCrypto from "node:crypto"; import { AgentEventSchema, + AgentEventType, CanonicalAgentEventEnvelopeSchema, ParentNarrativeRecoveryItemSchema, ProviderIdSchema, @@ -26,7 +27,7 @@ import { CanonicalAgentBoundary } from "./canonical-agent-boundary.js"; import { CanonicalCommittedProviderProjector } from "./canonical-committed-provider-projector.js"; import { CanonicalCodexSystemErrorProjection, matchesCodexSystemIntents } from "./canonical-codex-system-error-projection.js"; import { CanonicalContextCompactionProjection } from "./canonical-context-compaction-projection.js"; -import { CanonicalParentTurnWrite } from "./canonical-parent-turn-write.js"; +import { CanonicalParentTurnWrite, type DataOnlyParentLiveMessageInput } from "./canonical-parent-turn-write.js"; import { TaskRepo } from "../orchestration/persistence/task-repo.js"; import type { TaskToolWriteIntent } from "../tasks/task-tool-intent-reducer.js"; import { @@ -272,7 +273,8 @@ export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter const current = this.loadHead(input.execution.executionId); if (!current || current.terminal || !sameExecutionAndLease(current, input)) throw new SemanticConflict(); const result = this.turns.interruptLostExecution({ ...input.execution, - reason: input.reason, recoveryIncidentId: input.recoveryIncidentId }); + reason: input.reason, recoveryIncidentId: input.recoveryIncidentId, + ...(current.assignedMessageId ? { assignedMessageId: current.assignedMessageId } : {}) }); const receipt = committed(operation, result.durableThrough); const sequences = this.bufferedPublication?.flatMap((batch) => batch.map((event) => event.acceptedSequence)) ?? []; this.storePublicationChunks(input.execution.executionId, hash, sequences); @@ -421,6 +423,7 @@ export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter const head = this.requireNextHead(operation); if (head.providerId !== "codex") throw new SemanticConflict(); this.requireUnfinishedCheckpoint(operation.execution); + this.stageLiveMessage(operation.execution, head, mutation.message); const textResult = this.applyLiveText(mutation.text, operation.execution.executionId); if (mutation.narrative) this.persistNarrativeDelta(mutation.narrative); if (mutation.taskIntents) this.applyTaskIntents(operation.execution.threadId, mutation.taskIntents); @@ -429,7 +432,8 @@ export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter } const livePublication = this.projectLiveSystem(operation); const committedReceipt = committed(operation, head.durableRevision, undefined, undefined, textResult, livePublication); - this.storeHead({ ...head, ordinal: operation.ordinal }); + this.storeHead({ ...head, ordinal: operation.ordinal, + ...(mutation.message ? { assignedMessageId: mutation.message.messageId } : {}) }); this.storeReceipt(operation, hash, committedReceipt); return committedReceipt; })(); @@ -450,6 +454,16 @@ export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter return [{ publicationId: `${operation.operationId}:0`, after: result.after, event: result.event }]; } + private stageLiveMessage( + execution: ExecutionIdentity, + head: SemanticHead, + message: DataOnlyParentLiveMessageInput | undefined, + ): void { + if (!message) return; + if (head.assignedMessageId && head.assignedMessageId !== message.messageId) throw new SemanticConflict(); + if (!this.turns.stageLiveAssistant(execution, message)) throw new SemanticConflict(); + } + private applyLiveText( text: Extract["text"], executionId: string, @@ -519,10 +533,13 @@ export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter return this.db.transaction(() => { const head = this.requireNextHead(operation); if (head.providerOutcome !== mutation.input.outcome) throw new SemanticConflict(); + if (head.assignedMessageId && head.assignedMessageId !== mutation.input.assistant.messageId) { + throw new SemanticConflict(); + } this.turns.stageTerminalProjection(mutation.input); const receipt = committed(operation, head.durableRevision); this.storeHead({ ...head, ordinal: operation.ordinal, - assignedMessageId: mutation.input.assistant.messageId ?? null }); + assignedMessageId: mutation.input.assistant.messageId ?? head.assignedMessageId ?? null }); this.storeReceipt(operation, hash, receipt); return receipt; })(); @@ -839,7 +856,8 @@ function validLiveEventInput(operation: ExecutionSemanticOperation): boolean { const event = publication.event; if (!validLiveSystemMutation(mutation, event)) return false; if (!validLiveTextPayload(mutation, operation.execution) - || !AgentEventSchema().safeParse(event).success || !validLiveTextAssociation(mutation.text, event)) return false; + || !AgentEventSchema().safeParse(event).success + || !validLiveEventAssociation(mutation, event, operation.execution)) return false; if (mutation.narrative && !validNarrativeDeltaInput(mutation.narrative, operation.execution)) return false; if (!validLiveTaskIntents(mutation.taskIntents, event)) return false; return validLiveEventBudget(operation, mutation); @@ -878,8 +896,8 @@ function validLiveEventBudget( ? mutation.narrative.items.length + (mutation.narrative.discardedItemIds?.length ?? 0) : 0; const textRows = mutation.text.kind === "append" ? mutation.text.inputs.length : mutation.text.kind === "unchanged" ? 0 : 1; - if (textRows + narrativeRows + (mutation.taskIntents?.length ?? 0) - > ACTIVE_TURN_WRITE_BATCH_LIMITS.maxRows - 2) return false; + const rows = textRows + narrativeRows + (mutation.taskIntents?.length ?? 0) + (mutation.message ? 1 : 0); + if (rows > ACTIVE_TURN_WRITE_BATCH_LIMITS.maxRows - 2) return false; return Buffer.byteLength(JSON.stringify(operation), "utf8") <= ACTIVE_TURN_WRITE_BATCH_LIMITS.maxBytes; } @@ -916,6 +934,40 @@ function validLiveTextAssociation( } } +function validLiveEventAssociation( + mutation: Extract, + event: ExecutionLivePublicationIntent["event"], + execution: ExecutionIdentity, +): boolean { + if (mutation.message) { + return mutation.text.kind === "unchanged" && validLiveMessageAssociation(mutation.message, event, execution); + } + return event.type !== AgentEventType.Message && validLiveTextAssociation(mutation.text, event); +} + +function validLiveMessageAssociation( + message: DataOnlyParentLiveMessageInput, + event: ExecutionLivePublicationIntent["event"], + execution: ExecutionIdentity, +): boolean { + return event.type === AgentEventType.Message && validLiveMessageIdentity(message) + && validLiveMessageBody(message) && event.threadId === execution.threadId + && event.messageId === message.messageId && event.content === message.content + && (event.model ?? null) === message.model + && JSON.stringify(event.attachments ?? []) === JSON.stringify(message.attachments); +} + +function validLiveMessageIdentity(message: DataOnlyParentLiveMessageInput): boolean { + return typeof message.precedingMessageId === "string" && message.precedingMessageId.length > 0 + && typeof message.messageId === "string" && /^[0-9a-f]{64}$/.test(message.messageId); +} + +function validLiveMessageBody(message: DataOnlyParentLiveMessageInput): boolean { + if (typeof message.content !== "string" || !Array.isArray(message.attachments)) return false; + return (message.model === null || typeof message.model === "string") + && (message.content.trim().length > 0 || message.attachments.length > 0); +} + function validNarrativeEvent(event: ExecutionLivePublicationIntent["event"]): boolean { return event.type === "textDelta" ? event.isFinalResponse === false : [ diff --git a/apps/server/src/features/agents/canonical/canonical-parent-turn-write.ts b/apps/server/src/features/agents/canonical/canonical-parent-turn-write.ts index 6d526a6b5..a938865e9 100644 --- a/apps/server/src/features/agents/canonical/canonical-parent-turn-write.ts +++ b/apps/server/src/features/agents/canonical/canonical-parent-turn-write.ts @@ -15,6 +15,7 @@ import { HookExecutionRepo } from "../events/persistence/hook-execution-repo.js" import { PlanQuestionAnswersRepo } from "../planning/persistence/plan-question-answers-repo.js"; import { ToolCallRecordRepo } from "../tools/persistence/tool-call-record-repo.js"; import { deriveTurnAssistantMessageId } from "../turns/turn-assistant-message-id.js"; +import type { ExecutionIdentity } from "../execution/execution-mailbox-protocol.js"; import { ParentAssistantTextCheckpointService } from "../turns/parent-assistant-text-checkpoint-service.js"; import { TURN_DIFF_MAX_BYTES, parseTurnDiff } from "../turns/turn-diff-patch.js"; import { TurnDiffRepo } from "../turns/persistence/turn-diff-repo.js"; @@ -110,6 +111,15 @@ export interface DataOnlyParentTerminalProjectionInput { narrative: readonly ParentNarrativeRecoveryItem[]; } +/** Assistant body and assigned ID that must exist before its live message is published. */ +export interface DataOnlyParentLiveMessageInput { + readonly precedingMessageId: string; + readonly messageId: string; + readonly content: string; + readonly model: string | null; + readonly attachments: readonly StoredAttachment[]; +} + /** A staged projection ready for the canonical terminal commit. */ export interface StagedParentTerminalProjection { projection: ParentTurnProjection; @@ -124,6 +134,7 @@ export interface LostParentExecutionInput { executionId: string; reason: string; recoveryIncidentId: string; + assignedMessageId?: string; } /** Writer-local semantic operations with cloneable inputs and durable receipts. */ @@ -169,9 +180,10 @@ export class CanonicalParentTurnWrite { throw new Error(`Unfinished parent execution not found: ${input.executionId}`); } const existingAssistant = this.canonical.loadTerminalProjection(input.turnId).message; + const assignedAssistant = this.assignedAssistantForLoss(input, existingAssistant); const recoveredNarrative = this.canonical.loadParentNarrativeRecovery(input.turnId); - const text = existingAssistant ? "" : this.assistantTextCheckpoints.restore(input.executionId); - const assistant = existingAssistant ?? this.stageRecoveredAssistant(input, text, recoveredNarrative.length); + const text = existingAssistant || assignedAssistant ? "" : this.assistantTextCheckpoints.restore(input.executionId); + const assistant = existingAssistant ?? assignedAssistant ?? this.stageRecoveredAssistant(input, text, recoveredNarrative.length); this.canonical.markUnresolvedCodexChildDeliveriesUnknown(input.executionId); const result = this.canonical.interruptUnfinishedExecution( input.executionId, @@ -188,6 +200,19 @@ export class CanonicalParentTurnWrite { return result; } + private assignedAssistantForLoss( + input: LostParentExecutionInput, + terminal: ReturnType["message"], + ) { + if (!input.assignedMessageId) return null; + const assigned = this.messages.findByIdInThreadIncludingInternal(input.threadId, input.assignedMessageId); + if (!assigned || assigned.role !== "assistant" || !assigned.is_internal + || terminal && terminal.id !== assigned.id) { + throw new Error(`Assigned assistant message was not staged: ${input.assignedMessageId}`); + } + return assigned; + } + /** Retire provisional text only after the canonical interruption and receipt commit. */ retireInterruptedText(executionId: string): void { if (!this.assistantTextCheckpoints.retire(executionId)) { @@ -232,6 +257,38 @@ export class CanonicalParentTurnWrite { return this.canonical.commit(input); } + /** Stage the exact live body on this writer connection for terminal reuse. */ + stageLiveAssistant(execution: ExecutionIdentity, input: DataOnlyParentLiveMessageInput): boolean { + if (!this.canStageLiveAssistant(execution, input)) return false; + const existing = this.messages.findByIdInThreadIncludingInternal(execution.threadId, input.messageId); + if (existing) return Boolean(existing.is_internal && this.matchesAssistantBody(existing, input)); + return this.createLiveAssistant(execution.threadId, input); + } + + private canStageLiveAssistant(execution: ExecutionIdentity, input: DataOnlyParentLiveMessageInput): boolean { + const turn = this.canonical.loadTurnByExecution(execution.executionId); + const checkpoint = this.canonical.loadCheckpoint(execution.executionId); + return Boolean(turn && turn.id === execution.turnId && turn.threadId === execution.threadId + && checkpoint?.turnId === execution.turnId && checkpoint.terminalOutcome === null) + && deriveTurnAssistantMessageId(execution.threadId, input.precedingMessageId) === input.messageId; + } + + private createLiveAssistant(threadId: string, input: DataOnlyParentLiveMessageInput): boolean { + const preceding = this.messages.findByIdInThreadIncludingInternal(threadId, input.precedingMessageId); + if (!preceding || preceding.sequence !== this.messages.getLatestSequenceIncludingInternal(threadId)) return false; + this.messages.createAssistantIdempotent({ + id: input.messageId, + threadId, + content: input.content, + sequence: preceding.sequence + 1, + model: input.model, + attachments: [...input.attachments], + isInternal: true, + }); + const staged = this.messages.findByIdInThreadIncludingInternal(threadId, input.messageId); + return Boolean(staged?.is_internal && this.matchesAssistantBody(staged, input)); + } + /** Stage an internal assistant and its terminal narrative without publishing them. */ stageTerminalProjection(input: DataOnlyParentTerminalProjectionInput): StagedParentTerminalProjection { if (!input.threadId || !input.executionId || !Number.isFinite(Date.parse(input.endedAt))) { @@ -317,6 +374,15 @@ export class CanonicalParentTurnWrite { } } + private matchesAssistantBody( + message: NonNullable>, + input: DataOnlyParentLiveMessageInput, + ): boolean { + return message.role === "assistant" && message.content === input.content + && message.model === input.model + && JSON.stringify(message.attachments ?? []) === JSON.stringify(input.attachments); + } + private projectionForMessage(message: NonNullable>): StagedParentTerminalProjection { const narrative = this.narrative.loadForMessages([message]); return { diff --git a/apps/server/src/features/agents/execution/execution-worker-handler.ts b/apps/server/src/features/agents/execution/execution-worker-handler.ts index da5574507..6487c26ca 100644 --- a/apps/server/src/features/agents/execution/execution-worker-handler.ts +++ b/apps/server/src/features/agents/execution/execution-worker-handler.ts @@ -2,6 +2,7 @@ import type { AgentEvent, TurnOutcome } from "@mcode/contracts"; import type { ProviderEventDraft } from "@mcode/providers"; import type { + DataOnlyParentLiveMessageInput, DataOnlyParentTerminalProjectionInput, DataOnlyParentTurnFinishInput, DataOnlyParentTurnStartInput, @@ -40,6 +41,7 @@ export type ExecutionWorkCommand = readonly narrative?: ParentNarrativeRecoveryCommit; readonly taskIntents?: readonly TaskToolWriteIntent[]; readonly systemIntents?: readonly CodexSystemWriterIntent[]; + readonly message?: DataOnlyParentLiveMessageInput; readonly publication: ExecutionLivePublicationIntent; } | { readonly kind: "checkpoint"; readonly phase: string; readonly nativeCursor: unknown | null } @@ -69,6 +71,7 @@ export interface ExecutionSemanticOperation { readonly narrative?: ParentNarrativeRecoveryCommit; readonly taskIntents?: readonly TaskToolWriteIntent[]; readonly systemIntents?: readonly CodexSystemWriterIntent[]; + readonly message?: DataOnlyParentLiveMessageInput; } | { readonly kind: "checkpoint"; readonly phase: string; readonly nativeCursor: unknown | null } | { readonly kind: "stop-requested"; readonly requestId: string; readonly lastAdmittedOrdinal: number } @@ -297,6 +300,7 @@ function liveEventMutation( ...(command.narrative ? { narrative: command.narrative } : {}), ...(command.taskIntents ? { taskIntents: command.taskIntents } : {}), ...(command.systemIntents ? { systemIntents: command.systemIntents } : {}), + ...(command.message ? { message: command.message } : {}), }; } From cc80907c89b5ce4f2f7f582ff51a659bd59a995a Mon Sep 17 00:00:00 2001 From: "(CJ) Chukwudi Nwobodo" <142016413+chuks-qua@users.noreply.github.com> Date: Thu, 24 Sep 2026 22:51:56 +0100 Subject: [PATCH 101/136] feat(server): own execution mailbox admission and stop --- .../__tests__/execution-mailbox-owner.test.ts | 72 ++++++++++++ .../execution/execution-mailbox-owner.ts | 104 ++++++++++++++++++ 2 files changed, 176 insertions(+) create mode 100644 apps/server/src/features/agents/execution/__tests__/execution-mailbox-owner.test.ts create mode 100644 apps/server/src/features/agents/execution/execution-mailbox-owner.ts diff --git a/apps/server/src/features/agents/execution/__tests__/execution-mailbox-owner.test.ts b/apps/server/src/features/agents/execution/__tests__/execution-mailbox-owner.test.ts new file mode 100644 index 000000000..215e9cc82 --- /dev/null +++ b/apps/server/src/features/agents/execution/__tests__/execution-mailbox-owner.test.ts @@ -0,0 +1,72 @@ +import { describe, expect, it } from "vitest"; + +import { ExecutionMailboxOwner } from "../execution-mailbox-owner.js"; +import { ExecutionMailboxScheduler, type ExecutionMailboxCommand } from "../execution-mailbox-scheduler.js"; +import type { ExecutionIdentity, ExecutionWorkerPort, ExecutionWorkerReply, ExecutionWorkerRequest } from "../execution-mailbox-protocol.js"; +import type { ExecutionWorkCommand, ExecutionWorkerResult } from "../execution-worker-handler.js"; + +type Command = ExecutionMailboxCommand; + +class WorkerPort implements ExecutionWorkerPort { + onmessage: ((event: MessageEvent>) => void) | null = null; + onerror: ((event: ErrorEvent) => void) | null = null; + onclose: (() => void) | null = null; + readonly requests: ExecutionWorkerRequest[] = []; + + postMessage(request: ExecutionWorkerRequest): void { this.requests.push(request); } + terminate(): void {} + + reply(index: number, result: ExecutionWorkerResult): void { + const request = this.requests[index]; + if (!request) throw new Error(`Missing request ${index}`); + this.onmessage?.(new MessageEvent("message", { data: { ...request, result } })); + } +} + +const execution: ExecutionIdentity = { threadId: "thread-1", turnId: "turn-1", executionId: "execution-1" }; +const parentTurn = { + thread: { id: execution.threadId, workspaceId: "workspace-1", providerId: "codex", createdAt: "2026-09-24T00:00:00.000Z" }, + turnId: execution.turnId, + executionId: execution.executionId, + permissionMode: "supervised" as const, + providerIdentities: [], + userMessage: { kind: "create" as const, messageId: "user-1", content: "Run", sequence: 1 }, +}; + +describe("ExecutionMailboxOwner", () => { + it("claims before durable start and admits Stop behind earlier commands", async () => { + const worker = new WorkerPort(); + const scheduler = new ExecutionMailboxScheduler({ + workerCount: 1, + limits: { + maxPending: 8, maxPendingBytes: 128_000, reservedControl: 2, reservedControlBytes: 16_000, + maxPerExecutionPending: 8, maxPerExecutionBytes: 128_000, + reservedPerExecutionControl: 2, reservedPerExecutionControlBytes: 16_000, + }, + createWorker: () => worker, + onWorkerLost: () => {}, + }); + const owner = new ExecutionMailboxOwner(scheduler); + const starting = owner.start({ execution, ownerEpoch: 1, providerId: "codex", parentTurn }); + expect(worker.requests[0]?.command.kind).toBe("start"); + expect(owner.current(execution.threadId)?.execution).toEqual(execution); + await expect(owner.start({ execution, ownerEpoch: 2, providerId: "codex", parentTurn })) + .rejects.toThrow("already has an execution owner"); + worker.reply(0, { kind: "committed", operationId: `${worker.requests[0]!.lease.leaseId}:1`, durableRevision: 1 }); + await starting; + + const checkpoint = owner.submit(execution, { kind: "checkpoint", phase: "running", nativeCursor: null }); + const stopped = owner.stop(execution, "stop-1"); + expect(worker.requests[1]?.command.kind).toBe("checkpoint"); + await expect(owner.submit(execution, { kind: "assistant-text", inputs: [] })) + .rejects.toThrow("stopping"); + expect(() => owner.submit({ ...execution, executionId: "stale" }, { kind: "checkpoint", phase: "running", nativeCursor: null })) + .toThrow("No matching execution owner"); + worker.reply(1, { kind: "committed", operationId: `${worker.requests[1]!.lease.leaseId}:2`, durableRevision: 1 }); + expect(worker.requests[2]).toMatchObject({ stopWatermark: 2, command: { kind: "stop", requestId: "stop-1" } }); + worker.reply(2, { kind: "committed", operationId: `${worker.requests[2]!.lease.leaseId}:3`, durableRevision: 1 }); + await checkpoint; + await stopped; + scheduler.shutdown(); + }); +}); diff --git a/apps/server/src/features/agents/execution/execution-mailbox-owner.ts b/apps/server/src/features/agents/execution/execution-mailbox-owner.ts new file mode 100644 index 000000000..9ad249d86 --- /dev/null +++ b/apps/server/src/features/agents/execution/execution-mailbox-owner.ts @@ -0,0 +1,104 @@ +import type { DataOnlyParentTurnStartInput } from "../canonical/canonical-parent-turn-write.js"; +import type { ExecutionIdentity, ExecutionLease, ExecutionMailboxCompletion } from "./execution-mailbox-protocol.js"; +import type { ExecutionMailboxScheduler } from "./execution-mailbox-scheduler.js"; +import type { ExecutionWorkCommand, ExecutionWorkerResult } from "./execution-worker-handler.js"; + +type Scheduler = ExecutionMailboxScheduler; + +interface OwnedExecution { + readonly execution: ExecutionIdentity; + readonly lease: ExecutionLease; +} + +/** Claims one mailbox before a parent turn can be written or sent to a provider. */ +export class ExecutionMailboxOwner { + private readonly active = new Map(); + + constructor(private readonly scheduler: Scheduler) {} + + /** Start the exact execution on its assigned worker and wait for its durable receipt. */ + async start(input: { + readonly execution: ExecutionIdentity; + readonly ownerEpoch: number; + readonly providerId: string; + readonly parentTurn: DataOnlyParentTurnStartInput; + }): Promise { + if (this.active.has(input.execution.threadId)) throw new Error("Thread already has an execution owner"); + const claim = this.scheduler.claim(input.execution, input.ownerEpoch); + if (claim.kind !== "claimed") throw new Error(`Execution claim failed: ${claim.kind}`); + const owner = { execution: input.execution, lease: claim.lease }; + this.active.set(input.execution.threadId, owner); + try { + const result = await this.submitOwned(owner, { kind: "start", providerId: input.providerId, input: input.parentTurn }); + if (result.kind !== "committed") throw new Error(`Execution start was not committed: ${input.execution.executionId}`); + } catch (error) { + // A lost worker is reconciled by its coordinator. A rejected start did not create worker state. + if (this.scheduler.release(owner.execution, owner.lease)) this.active.delete(input.execution.threadId); + throw error; + } + } + + /** Admit one ordered command only for the current thread execution and lease. */ + submit(execution: ExecutionIdentity, command: ExecutionWorkCommand): Promise { + return this.submitOwned(this.requireOwner(execution), command); + } + + /** Put Stop behind every already admitted event before awaiting provider teardown. */ + stop(execution: ExecutionIdentity, requestId: string): Promise { + const owner = this.requireOwner(execution); + return this.submitOwned(owner, { kind: "stop", requestId }); + } + + /** Release the slot only after finalization and its release command have settled. */ + async release(execution: ExecutionIdentity): Promise { + const owner = this.requireOwner(execution); + const result = await this.submitOwned(owner, { kind: "release" }); + if (result.kind !== "released" || !this.scheduler.release(owner.execution, owner.lease)) { + throw new Error(`Execution release was not acknowledged: ${execution.executionId}`); + } + this.active.delete(execution.threadId); + } + + /** Forget a worker-lost execution only after the recovery coordinator released its lease. */ + forgetRecovered(execution: ExecutionIdentity, lease: ExecutionLease): void { + const owner = this.active.get(execution.threadId); + if (owner && sameExecution(owner.execution, execution) && owner.lease.leaseId === lease.leaseId) { + this.active.delete(execution.threadId); + } + } + + /** Current exact owner, for provider ingress and Stop routing. */ + current(threadId: string): OwnedExecution | undefined { + return this.active.get(threadId); + } + + private requireOwner(execution: ExecutionIdentity): OwnedExecution { + const owner = this.active.get(execution.threadId); + if (!owner || !sameExecution(owner.execution, execution)) { + throw new Error(`No matching execution owner: ${execution.executionId}`); + } + return owner; + } + + private async submitOwned( + owner: OwnedExecution, + command: ExecutionWorkCommand | { readonly kind: "stop"; readonly requestId: string }, + ): Promise { + const byteLength = Buffer.byteLength(JSON.stringify({ execution: owner.execution, lease: owner.lease, command }), "utf8"); + const admitted = this.scheduler.submit({ ...owner, command, byteLength }); + if (admitted.kind !== "admitted") throw new Error(`Execution command admission failed: ${admitted.kind}`); + return requireReply(await admitted.completion); + } +} + +function requireReply(completion: ExecutionMailboxCompletion): ExecutionWorkerResult { + if (completion.kind !== "reply") throw new Error(`Execution worker ${completion.kind}`); + if (completion.result.kind === "rejected") { + throw new Error(`Execution worker rejected command: ${completion.result.reason}`); + } + return completion.result; +} + +function sameExecution(left: ExecutionIdentity, right: ExecutionIdentity): boolean { + return left.threadId === right.threadId && left.turnId === right.turnId && left.executionId === right.executionId; +} From e58cc1ab64240773bb97c3a874afee12eada5911 Mon Sep 17 00:00:00 2001 From: "(CJ) Chukwudi Nwobodo" <142016413+chuks-qua@users.noreply.github.com> Date: Thu, 24 Sep 2026 22:50:41 +0100 Subject: [PATCH 102/136] feat(server): commit live Codex plan projections with receipts --- .../canonical-plan-live-projection.test.ts | 169 ++++++++++++++++++ .../canonical-execution-semantic-writer.ts | 109 +++++++++-- .../canonical-execution-writer-port.ts | 7 + .../execution-plan-question-release.ts | 52 ++++++ .../execution/execution-worker-handler.ts | 27 ++- .../agents/planning/persistence/plan-repo.ts | 6 + 6 files changed, 356 insertions(+), 14 deletions(-) create mode 100644 apps/server/src/features/agents/canonical/__tests__/canonical-plan-live-projection.test.ts create mode 100644 apps/server/src/features/agents/canonical/execution-plan-question-release.ts diff --git a/apps/server/src/features/agents/canonical/__tests__/canonical-plan-live-projection.test.ts b/apps/server/src/features/agents/canonical/__tests__/canonical-plan-live-projection.test.ts new file mode 100644 index 000000000..a2ac63547 --- /dev/null +++ b/apps/server/src/features/agents/canonical/__tests__/canonical-plan-live-projection.test.ts @@ -0,0 +1,169 @@ +import "reflect-metadata"; +import * as NodeFS from "node:fs"; +import * as NodeOS from "node:os"; +import * as NodePath from "node:path"; +import type { Database } from "bun:sqlite"; +import { AgentEventType, type PlanQuestion } from "@mcode/contracts"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; + +import { openDatabase } from "../../../../runtime/persistence/sqlite/database.js"; +import { PlanRepo } from "../../planning/persistence/plan-repo.js"; +import { AgentEventPublicationRegistry } from "../../orchestration/agent-event-publication-registry.js"; +import { deriveTurnAssistantMessageId } from "../../turns/turn-assistant-message-id.js"; +import type { ExecutionSemanticOperation } from "../../execution/execution-worker-handler.js"; +import { CanonicalExecutionSemanticWriter } from "../canonical-execution-semantic-writer.js"; +import { CanonicalAgentWriterClient } from "../canonical-agent-writer-client.js"; +import { CanonicalExecutionWriterPort } from "../canonical-execution-writer-port.js"; +import { ExecutionLivePublicationRelease } from "../execution-live-publication-release.js"; +import { ExecutionPlanQuestionRelease } from "../execution-plan-question-release.js"; +import type { DataOnlyParentTurnStartInput } from "../canonical-parent-turn-write.js"; + +const THREAD_ID = "thread-plan"; +const TURN_ID = "turn-plan"; +const EXECUTION_ID = "00000000-0000-4000-8000-000000000091"; +const NOW = "2026-09-24T10:00:00.000Z"; +const execution = { threadId: THREAD_ID, turnId: TURN_ID, executionId: EXECUTION_ID } as const; +const lease = { ownerEpoch: 1, workerIndex: 0, workerGeneration: 1, leaseId: "lease-plan" } as const; +const questions = [{ id: "q1", category: "AUTH", question: "Which login?", options: [ + { id: "o1", title: "Passkey", description: "Use passkeys.", recommended: true }, + { id: "o2", title: "Password", description: "Use passwords." }, +] }]; +const planOutput = { title: "Login plan", contentMd: "# Login plan\n## Build\nUse passkeys.", + sectionsJson: '[{"id":"s1","title":"Build","level":2}]', changeSummary: null }; + +function seedThread(db: Database): void { + db.prepare("INSERT INTO workspaces (id, name, path, created_at, updated_at) VALUES (?, ?, ?, ?, ?)") + .run("workspace-plan", "Workspace", "C:/fixture", NOW, NOW); + db.prepare("INSERT INTO threads (id, workspace_id, title, branch, provider, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?)") + .run(THREAD_ID, "workspace-plan", "Thread", "main", "codex", NOW, NOW); +} + +function startInput(): DataOnlyParentTurnStartInput { + return { + thread: { id: THREAD_ID, workspaceId: "workspace-plan", providerId: "codex", createdAt: NOW }, + turnId: TURN_ID, executionId: EXECUTION_ID, permissionMode: "supervised", providerIdentities: [], + userMessage: { kind: "create", messageId: "user-plan", content: "Plan login", sequence: 1 }, + }; +} + +function operation(ordinal: number, mutation: ExecutionSemanticOperation["mutation"]): ExecutionSemanticOperation { + return { operationId: `${lease.leaseId}:${ordinal}`, execution, lease, ordinal, mutation }; +} + +describe("Codex live plan projections on the sole writer", () => { + let directory: string; + let path: string; + let db: Database; + let writer: CanonicalExecutionSemanticWriter; + + beforeEach(async () => { + directory = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "mcode-plan-projection-")); + path = NodePath.join(directory, "mcode.db"); + db = openDatabase({ dbPath: path }); + seedThread(db); + writer = new CanonicalExecutionSemanticWriter(db, () => {}); + expect((await writer.transact(operation(1, { kind: "begin", providerId: "codex", input: startInput() }))).kind) + .toBe("committed"); + }); + + afterEach(() => { + db.close(true); + NodeFS.rmSync(directory, { recursive: true, force: true }); + }); + + it("replays the question publication receipt after the writer restarts", async () => { + const block = `\`\`\`plan-questions\n${JSON.stringify(questions)}\n\`\`\``; + const event = { type: AgentEventType.TextDelta, threadId: THREAD_ID, turnExecutionId: EXECUTION_ID, + isFinalResponse: true, delta: block }; + const write = { ...operation(2, { kind: "live-event", text: { kind: "append", inputs: [ + { ...execution, sequence: 1, text: block }, + ] }, planQuestions: questions }), livePublication: [{ after: "writer" as const, event }] }; + const published: Array<{ threadId: string; questions: readonly PlanQuestion[] }> = []; + const release = new ExecutionPlanQuestionRelease((threadId, batch) => published.push({ threadId, questions: batch })); + expect(published).toEqual([]); + const receipt = await writer.transact(write); + expect(receipt).toMatchObject({ kind: "committed", planQuestions: { + publicationId: "lease-plan:2:plan-questions", threadId: THREAD_ID, questions, + }, livePublication: [{ event }] }); + release.release(write, receipt); + release.release(write, receipt); + expect(published).toEqual([{ threadId: THREAD_ID, questions }]); + db.close(true); + db = openDatabase({ dbPath: path }); + writer = new CanonicalExecutionSemanticWriter(db, () => {}); + expect(await writer.transact(write)).toEqual(receipt); + expect(db.prepare("SELECT COUNT(*) AS count FROM canonical_writer_operation_receipts WHERE execution_id = ? AND operation_id = ?") + .get(EXECUTION_ID, "lease-plan:2")).toEqual({ count: 1 }); + }); + + it("publishes questions through the worker port only after its receipt", async () => { + const block = `\`\`\`plan-questions\n${JSON.stringify(questions)}\n\`\`\``; + const event = { type: AgentEventType.TextDelta, threadId: THREAD_ID, turnExecutionId: EXECUTION_ID, + isFinalResponse: true, delta: block }; + const write = { ...operation(2, { kind: "live-event", text: { kind: "append", inputs: [ + { ...execution, sequence: 1, text: block }, + ] }, planQuestions: questions }), livePublication: [{ after: "writer" as const, event }] }; + const published: string[] = []; + const registry = new AgentEventPublicationRegistry(); + registry.bind((item) => published.push(`agent:${item.type}`)); + const client = new CanonicalAgentWriterClient(path); + const port = new CanonicalExecutionWriterPort(client, () => {}, + new ExecutionLivePublicationRelease(registry), + new ExecutionPlanQuestionRelease(() => published.push("plan.questions"))); + try { + expect(published).toEqual([]); + expect((await port.transact(write)).kind).toBe("committed"); + expect(published).toEqual(["agent:textDelta", "plan.questions"]); + expect((await port.transact(write)).kind).toBe("committed"); + expect(published).toEqual(["agent:textDelta", "plan.questions"]); + } finally { + await client.close(); + } + }); + + it("persists a plan with its staged assistant before acknowledging the message", async () => { + const messageId = deriveTurnAssistantMessageId(THREAD_ID, "user-plan"); + const content = "# Login plan\n## Build\nUse passkeys."; + const event = { type: AgentEventType.Message, threadId: THREAD_ID, turnExecutionId: EXECUTION_ID, + messageId, content, model: null, tokens: null }; + const write = { ...operation(2, { kind: "live-event", text: { kind: "unchanged" }, + message: { precedingMessageId: "user-plan", messageId, content, model: null, attachments: [] }, + planOutput, + }), livePublication: [{ after: "writer" as const, event }] }; + db.run("CREATE TRIGGER fail_plan_receipt BEFORE INSERT ON canonical_writer_operation_receipts WHEN NEW.kind = 'semantic:live-event' BEGIN SELECT RAISE(ABORT, 'plan receipt unavailable'); END"); + await expect(writer.transact(write)).rejects.toThrow("plan receipt unavailable"); + expect(new PlanRepo(db).listByThread(THREAD_ID)).toEqual([]); + expect(db.prepare("SELECT id FROM messages WHERE id = ?").get(messageId)).toBeNull(); + db.run("DROP TRIGGER fail_plan_receipt"); + + const receipt = await writer.transact(write); + expect(receipt).toMatchObject({ kind: "committed", planOutput: { + threadId: THREAD_ID, messageId, title: planOutput.title, version: 1, + }, livePublication: [{ event }] }); + expect(db.prepare("SELECT is_internal FROM messages WHERE id = ?").get(messageId)).toEqual({ is_internal: 1 }); + expect(new PlanRepo(db).getByMessageId(messageId)).toEqual(receipt.kind === "committed" ? receipt.planOutput : null); + db.close(true); + db = openDatabase({ dbPath: path }); + writer = new CanonicalExecutionSemanticWriter(db, () => {}); + expect(await writer.transact(write)).toEqual(receipt); + expect(new PlanRepo(db).listByThread(THREAD_ID)).toHaveLength(1); + expect(await writer.transact({ ...write, mutation: { ...write.mutation, planOutput: { + ...planOutput, title: "Different plan", + } } })).toEqual({ kind: "conflict", operationId: "lease-plan:2" }); + }); + + it("rejects detached or malformed plan data without advancing the execution", async () => { + const event = { type: AgentEventType.TextDelta, threadId: THREAD_ID, + turnExecutionId: EXECUTION_ID, isFinalResponse: true, delta: "text" }; + const base = { ...operation(2, { kind: "live-event", text: { kind: "append", inputs: [ + { ...execution, sequence: 1, text: "text" }, + ] } }), livePublication: [{ after: "writer" as const, event }] }; + expect(await writer.transact({ ...base, mutation: { ...base.mutation, + planOutput } })).toEqual({ kind: "conflict", operationId: "lease-plan:2" }); + expect(await writer.transact({ ...base, mutation: { ...base.mutation, + planQuestions: [{ ...questions[0], options: [] }] } })).toEqual({ kind: "conflict", operationId: "lease-plan:2" }); + expect(new PlanRepo(db).listByThread(THREAD_ID)).toEqual([]); + expect(db.prepare("SELECT COUNT(*) AS count FROM canonical_writer_operation_receipts WHERE execution_id = ? AND operation_id = ?") + .get(EXECUTION_ID, "lease-plan:2")).toEqual({ count: 0 }); + }); +}); diff --git a/apps/server/src/features/agents/canonical/canonical-execution-semantic-writer.ts b/apps/server/src/features/agents/canonical/canonical-execution-semantic-writer.ts index bf24715be..1a5371891 100644 --- a/apps/server/src/features/agents/canonical/canonical-execution-semantic-writer.ts +++ b/apps/server/src/features/agents/canonical/canonical-execution-semantic-writer.ts @@ -5,6 +5,9 @@ import { AgentEventType, CanonicalAgentEventEnvelopeSchema, ParentNarrativeRecoveryItemSchema, + PlanQuestionBatchSchema, + PlanRecordSchema, + PlanSectionNavSchema, ProviderIdSchema, TurnOutcomeSchema, type TurnOutcome, @@ -21,6 +24,7 @@ import type { ExecutionSemanticWriter, ExecutionWriteReceipt, ExecutionLivePublicationReceipt, + ExecutionPlanQuestionsReceipt, } from "../execution/execution-worker-handler.js"; import type { CanonicalAgentEventPublisher } from "./canonical-agent-boundary.js"; import { CanonicalAgentBoundary } from "./canonical-agent-boundary.js"; @@ -29,6 +33,7 @@ import { CanonicalCodexSystemErrorProjection, matchesCodexSystemIntents } from " import { CanonicalContextCompactionProjection } from "./canonical-context-compaction-projection.js"; import { CanonicalParentTurnWrite, type DataOnlyParentLiveMessageInput } from "./canonical-parent-turn-write.js"; import { TaskRepo } from "../orchestration/persistence/task-repo.js"; +import { PlanRepo } from "../planning/persistence/plan-repo.js"; import type { TaskToolWriteIntent } from "../tasks/task-tool-intent-reducer.js"; import { ParentAssistantTextCheckpointService, @@ -70,6 +75,13 @@ const taskIntentsSchema = z.array(z.discriminatedUnion("kind", [ group: z.string().max(128), patch: storedTaskSchema.pick({ status: true, content: true, activeForm: true }).partial() }).strict(), z.object({ kind: z.literal("remove-task"), id: z.string().min(1).max(256), group: z.string().max(128) }).strict(), ])).max(16); +const planOutputSchema = z.object({ + title: z.string().trim().min(1).max(512), + contentMd: z.string().min(1).max(256 * 1024), + sectionsJson: z.string().max(64 * 1024), + changeSummary: z.string().max(4096).nullable(), +}).strict(); +const planSectionsSchema = z.array(PlanSectionNavSchema()).max(128); const storedPublicationSequencesSchema = z.array(z.union([ z.number().int().positive().max(Number.MAX_SAFE_INTEGER), z.object({ executionId: z.string(), sequence: z.number().int().positive().max(Number.MAX_SAFE_INTEGER) }), @@ -130,6 +142,11 @@ const storedReceiptSchema = z.object({ committedItems: z.number().int().positive(), committedBytes: z.number().int().positive(), }).optional(), + planQuestions: z.object({ + publicationId: z.string(), threadId: z.string(), + questions: PlanQuestionBatchSchema().shape.questions, + }).optional(), + planOutput: PlanRecordSchema().optional(), }); const storedOperationSchema = z.object({ kind: z.string(), @@ -174,6 +191,7 @@ export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter private readonly contextCompaction: CanonicalContextCompactionProjection; private readonly systemProjection: CanonicalCodexSystemErrorProjection; private readonly tasks: TaskRepo; + private readonly plans: PlanRepo; private readonly assistantText: ParentAssistantTextCheckpointService; private readonly canonical: CanonicalAgentBoundary; private readonly findOperation: ReturnType; @@ -202,6 +220,7 @@ export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter this.contextCompaction = new CanonicalContextCompactionProjection(db); this.systemProjection = new CanonicalCodexSystemErrorProjection(db); this.tasks = new TaskRepo(db); + this.plans = new PlanRepo(db); this.assistantText = new ParentAssistantTextCheckpointService(db); this.findOperation = db.prepare("SELECT kind, input_hash, receipt_json FROM canonical_writer_operation_receipts WHERE execution_id = ? AND operation_id = ?"); this.listPublicationChunks = db.prepare("SELECT operation_id, kind, input_hash, receipt_json FROM canonical_writer_operation_receipts WHERE execution_id = ? AND operation_id > ? AND operation_id < ? ORDER BY operation_id LIMIT ?"); @@ -423,15 +442,10 @@ export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter const head = this.requireNextHead(operation); if (head.providerId !== "codex") throw new SemanticConflict(); this.requireUnfinishedCheckpoint(operation.execution); - this.stageLiveMessage(operation.execution, head, mutation.message); - const textResult = this.applyLiveText(mutation.text, operation.execution.executionId); - if (mutation.narrative) this.persistNarrativeDelta(mutation.narrative); - if (mutation.taskIntents) this.applyTaskIntents(operation.execution.threadId, mutation.taskIntents); - if (mutation.text.kind === "reclassify" && !this.assistantText.resetInTransaction(operation.execution.executionId)) { - throw new SemanticConflict(); - } + const { textResult, planQuestions, planOutput } = this.applyLiveFeatureEffects(operation, head); const livePublication = this.projectLiveSystem(operation); - const committedReceipt = committed(operation, head.durableRevision, undefined, undefined, textResult, livePublication); + const committedReceipt = committed(operation, head.durableRevision, undefined, undefined, textResult, + livePublication, planQuestions, planOutput); this.storeHead({ ...head, ordinal: operation.ordinal, ...(mutation.message ? { assignedMessageId: mutation.message.messageId } : {}) }); this.storeReceipt(operation, hash, committedReceipt); @@ -454,6 +468,28 @@ export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter return [{ publicationId: `${operation.operationId}:0`, after: result.after, event: result.event }]; } + private applyLiveFeatureEffects( + operation: ExecutionSemanticOperation, + head: SemanticHead, + ) { + const mutation = operation.mutation; + if (mutation.kind !== "live-event") throw new SemanticConflict(); + this.stageLiveMessage(operation.execution, head, mutation.message); + const textResult = this.applyLiveText(mutation.text, operation.execution.executionId); + if (mutation.narrative) this.persistNarrativeDelta(mutation.narrative); + if (mutation.taskIntents) this.applyTaskIntents(operation.execution.threadId, mutation.taskIntents); + const planOutput = this.persistLivePlanOutput(operation.execution.threadId, mutation); + if (mutation.text.kind === "reclassify" && !this.assistantText.resetInTransaction(operation.execution.executionId)) { + throw new SemanticConflict(); + } + const planQuestions = mutation.planQuestions ? { + publicationId: `${operation.operationId}:plan-questions`, + threadId: operation.execution.threadId, + questions: mutation.planQuestions, + } satisfies ExecutionPlanQuestionsReceipt : undefined; + return { textResult, planQuestions, planOutput }; + } + private stageLiveMessage( execution: ExecutionIdentity, head: SemanticHead, @@ -464,6 +500,15 @@ export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter if (!this.turns.stageLiveAssistant(execution, message)) throw new SemanticConflict(); } + private persistLivePlanOutput( + threadId: string, + mutation: Extract, + ) { + return mutation.planOutput && mutation.message + ? this.persistPlanOutput(threadId, mutation.message.messageId, mutation.planOutput) + : undefined; + } + private applyLiveText( text: Extract["text"], executionId: string, @@ -489,6 +534,16 @@ export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter } } + private persistPlanOutput( + threadId: string, + messageId: string, + output: NonNullable["planOutput"]>, + ) { + if (this.plans.getByMessageId(messageId)) throw new SemanticConflict(); + return this.plans.create(threadId, messageId, output.title, output.contentMd, + output.sectionsJson, output.changeSummary); + } + private requireUnfinishedCheckpoint(execution: ExecutionIdentity): void { const checkpoint = this.canonical.loadCheckpoint(execution.executionId); if (!checkpoint || checkpoint.threadId !== execution.threadId @@ -707,7 +762,7 @@ export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter } return receipt.operationId === operation.operationId && Number.isSafeInteger(receipt.durableRevision) ? committed(operation, receipt.durableRevision, receipt.providerCommit, receipt.providerEvents, - receipt.assistantTextCheckpoint, receipt.livePublication) : conflict(operation); + receipt.assistantTextCheckpoint, receipt.livePublication, receipt.planQuestions, receipt.planOutput) : conflict(operation); } private matchesLivePublicationReceipt( @@ -859,10 +914,19 @@ function validLiveEventInput(operation: ExecutionSemanticOperation): boolean { || !AgentEventSchema().safeParse(event).success || !validLiveEventAssociation(mutation, event, operation.execution)) return false; if (mutation.narrative && !validNarrativeDeltaInput(mutation.narrative, operation.execution)) return false; - if (!validLiveTaskIntents(mutation.taskIntents, event)) return false; + if (!validLiveFeatureIntents(mutation, event, operation.execution.threadId)) return false; return validLiveEventBudget(operation, mutation); } +function validLiveFeatureIntents( + mutation: Extract, + event: ExecutionLivePublicationIntent["event"], + threadId: string, +): boolean { + return validLiveTaskIntents(mutation.taskIntents, event) + && validLivePlanProjection(mutation, event, threadId); +} + function validLiveSystemMutation( mutation: Extract, event: ExecutionLivePublicationIntent["event"], @@ -873,6 +937,24 @@ function validLiveSystemMutation( && matchesCodexSystemIntents(event, mutation.systemIntents); } +function validLivePlanProjection( + mutation: Extract, + event: ExecutionLivePublicationIntent["event"], + threadId: string, +): boolean { + if (mutation.planQuestions !== undefined) { + if (event.type !== "textDelta" || mutation.planOutput !== undefined + || !PlanQuestionBatchSchema().safeParse({ threadId, questions: mutation.planQuestions }).success) return false; + } + if (mutation.planOutput === undefined) return true; + if (event.type !== "message" || !mutation.message || !planOutputSchema.safeParse(mutation.planOutput).success) return false; + try { + return planSectionsSchema.safeParse(JSON.parse(mutation.planOutput.sectionsJson)).success; + } catch { + return false; + } +} + function validLiveTaskIntents( intents: readonly TaskToolWriteIntent[] | undefined, event: ExecutionLivePublicationIntent["event"], @@ -896,7 +978,8 @@ function validLiveEventBudget( ? mutation.narrative.items.length + (mutation.narrative.discardedItemIds?.length ?? 0) : 0; const textRows = mutation.text.kind === "append" ? mutation.text.inputs.length : mutation.text.kind === "unchanged" ? 0 : 1; - const rows = textRows + narrativeRows + (mutation.taskIntents?.length ?? 0) + (mutation.message ? 1 : 0); + const rows = textRows + narrativeRows + (mutation.taskIntents?.length ?? 0) + + Number(Boolean(mutation.message)) + Number(Boolean(mutation.planOutput)); if (rows > ACTIVE_TURN_WRITE_BATCH_LIMITS.maxRows - 2) return false; return Buffer.byteLength(JSON.stringify(operation), "utf8") <= ACTIVE_TURN_WRITE_BATCH_LIMITS.maxBytes; } @@ -1082,6 +1165,8 @@ function committed( providerEvents?: readonly ProjectedCommittedProviderEvent[], assistantTextCheckpoint?: ParentAssistantTextCheckpointResult, livePublication: readonly ExecutionLivePublicationReceipt[] | undefined = livePublicationFor(operation), + planQuestions?: ExecutionPlanQuestionsReceipt, + planOutput?: ReturnType, ): Extract { return { kind: "committed", operationId: operation.operationId, durableRevision, @@ -1089,6 +1174,8 @@ function committed( ...(providerEvents ? { providerEvents } : {}), ...(assistantTextCheckpoint ? { assistantTextCheckpoint } : {}), ...(livePublication ? { livePublication } : {}), + ...(planQuestions ? { planQuestions } : {}), + ...(planOutput ? { planOutput } : {}), }; } diff --git a/apps/server/src/features/agents/canonical/canonical-execution-writer-port.ts b/apps/server/src/features/agents/canonical/canonical-execution-writer-port.ts index 77f777d41..07e965e58 100644 --- a/apps/server/src/features/agents/canonical/canonical-execution-writer-port.ts +++ b/apps/server/src/features/agents/canonical/canonical-execution-writer-port.ts @@ -7,6 +7,7 @@ import type { CanonicalAgentEventPublisher } from "./canonical-agent-boundary.js import { CanonicalAgentWriterClient } from "./canonical-agent-writer-client.js"; import type { LostExecutionInterruption } from "./canonical-execution-semantic-writer.js"; import type { ExecutionLivePublicationRelease } from "./execution-live-publication-release.js"; +import type { ExecutionPlanQuestionRelease } from "./execution-plan-question-release.js"; /** Bridges an execution worker to the sole SQLite writer and publishes only committed events. */ export class CanonicalExecutionWriterPort implements ExecutionSemanticWriter { @@ -14,12 +15,18 @@ export class CanonicalExecutionWriterPort implements ExecutionSemanticWriter { private readonly writer: CanonicalAgentWriterClient, private readonly publish: CanonicalAgentEventPublisher, private readonly livePublication?: ExecutionLivePublicationRelease, + private readonly planQuestions?: ExecutionPlanQuestionRelease, ) {} /** Resolve after the durable writer receipt and its canonical event publication. */ async transact(operation: ExecutionSemanticOperation): Promise { const receipt = await this.writer.transactSemantic(operation, this.publish); + if (receipt.kind === "committed" && receipt.planQuestions && !this.planQuestions) { + throw new Error("Plan-question publication is not bound"); + } + this.planQuestions?.validate(operation, receipt); this.livePublication?.release(operation, receipt); + this.planQuestions?.release(operation, receipt); return receipt; } diff --git a/apps/server/src/features/agents/canonical/execution-plan-question-release.ts b/apps/server/src/features/agents/canonical/execution-plan-question-release.ts new file mode 100644 index 000000000..55a6f757f --- /dev/null +++ b/apps/server/src/features/agents/canonical/execution-plan-question-release.ts @@ -0,0 +1,52 @@ +import type { PlanQuestion } from "@mcode/contracts"; +import { PlanQuestionBatchSchema } from "@mcode/contracts"; +import * as NodeUtil from "node:util"; + +import type { + ExecutionPlanQuestionsReceipt, ExecutionSemanticOperation, ExecutionWriteReceipt, +} from "../execution/execution-worker-handler.js"; + +const MAX_RELEASED_QUESTIONS = 8_192; + +/** Publishes a parsed plan-question batch only from its committed writer receipt. */ +export class ExecutionPlanQuestionRelease { + private readonly released = new Set(); + + constructor(private readonly publish: (threadId: string, questions: readonly PlanQuestion[]) => void) {} + + /** Check question data before another publication channel releases the event. */ + validate(operation: ExecutionSemanticOperation, receipt: ExecutionWriteReceipt): ExecutionPlanQuestionsReceipt | null { + if (receipt.kind !== "committed" || !receipt.planQuestions) return null; + const batch = receipt.planQuestions; + if (!matchesOperation(operation, receipt, batch) || !PlanQuestionBatchSchema().safeParse({ + threadId: batch.threadId, questions: batch.questions, + }).success || !NodeUtil.isDeepStrictEqual(batch.questions, operation.mutation.kind === "live-event" + ? operation.mutation.planQuestions : undefined)) { + throw new Error("Plan-question publication receipt is invalid"); + } + return batch; + } + + /** Publish once per host lifetime after the writer acknowledges the event. */ + release(operation: ExecutionSemanticOperation, receipt: ExecutionWriteReceipt): void { + const batch = this.validate(operation, receipt); + if (!batch) return; + const key = JSON.stringify([operation.execution.executionId, batch.publicationId]); + if (this.released.has(key)) return; + if (this.released.size >= MAX_RELEASED_QUESTIONS) throw new Error("Plan-question publication tracking is full"); + this.publish(batch.threadId, batch.questions); + this.released.add(key); + } +} + +function matchesOperation( + operation: ExecutionSemanticOperation, + receipt: Extract, + batch: ExecutionPlanQuestionsReceipt, +): boolean { + return receipt.operationId === operation.operationId && operation.mutation.kind === "live-event" + && Boolean(operation.mutation.planQuestions) + && operation.livePublication?.[0]?.event.type === "textDelta" + && batch.publicationId === `${operation.operationId}:plan-questions` + && batch.threadId === operation.execution.threadId; +} diff --git a/apps/server/src/features/agents/execution/execution-worker-handler.ts b/apps/server/src/features/agents/execution/execution-worker-handler.ts index 6487c26ca..d113b42a5 100644 --- a/apps/server/src/features/agents/execution/execution-worker-handler.ts +++ b/apps/server/src/features/agents/execution/execution-worker-handler.ts @@ -1,4 +1,4 @@ -import type { AgentEvent, TurnOutcome } from "@mcode/contracts"; +import type { AgentEvent, PlanQuestion, PlanRecord, TurnOutcome } from "@mcode/contracts"; import type { ProviderEventDraft } from "@mcode/providers"; import type { @@ -16,6 +16,7 @@ import type { } from "../turns/parent-assistant-text-checkpoint-service.js"; import type { ParentNarrativeRecoveryCommit } from "../turns/parent-turn-durability.js"; import type { TaskToolWriteIntent } from "../tasks/task-tool-intent-reducer.js"; +import type { PlanPersistenceReady } from "../planning/plan-execution-state.js"; import type { ExecutionIdentity, ExecutionLease, @@ -42,6 +43,8 @@ export type ExecutionWorkCommand = readonly taskIntents?: readonly TaskToolWriteIntent[]; readonly systemIntents?: readonly CodexSystemWriterIntent[]; readonly message?: DataOnlyParentLiveMessageInput; + readonly planQuestions?: readonly PlanQuestion[]; + readonly planOutput?: PlanPersistenceReady; readonly publication: ExecutionLivePublicationIntent; } | { readonly kind: "checkpoint"; readonly phase: string; readonly nativeCursor: unknown | null } @@ -72,6 +75,8 @@ export interface ExecutionSemanticOperation { readonly taskIntents?: readonly TaskToolWriteIntent[]; readonly systemIntents?: readonly CodexSystemWriterIntent[]; readonly message?: DataOnlyParentLiveMessageInput; + readonly planQuestions?: readonly PlanQuestion[]; + readonly planOutput?: PlanPersistenceReady; } | { readonly kind: "checkpoint"; readonly phase: string; readonly nativeCursor: unknown | null } | { readonly kind: "stop-requested"; readonly requestId: string; readonly lastAdmittedOrdinal: number } @@ -93,6 +98,13 @@ export interface ExecutionLivePublicationReceipt extends ExecutionLivePublicatio readonly publicationId: string; } +/** Plan questions may be pushed only after their text event's durable writer receipt. */ +export interface ExecutionPlanQuestionsReceipt { + readonly publicationId: string; + readonly threadId: string; + readonly questions: readonly PlanQuestion[]; +} + /** Provider-facing receipt values retained with the execution operation. */ export type ExecutionProviderCommitReceipt = Pick< CanonicalAgentCommitResult, @@ -110,7 +122,7 @@ export type ProjectedCommittedProviderEvent = Omit< /** A writer reply is valid only after the semantic operation commits durably. */ export type ExecutionWriteReceipt = - | { readonly kind: "committed"; readonly operationId: string; readonly durableRevision: number; readonly providerCommit?: ExecutionProviderCommitReceipt; readonly providerEvents?: readonly ProjectedCommittedProviderEvent[]; readonly assistantTextCheckpoint?: ParentAssistantTextCheckpointResult; readonly livePublication?: readonly ExecutionLivePublicationReceipt[] } + | { readonly kind: "committed"; readonly operationId: string; readonly durableRevision: number; readonly providerCommit?: ExecutionProviderCommitReceipt; readonly providerEvents?: readonly ProjectedCommittedProviderEvent[]; readonly assistantTextCheckpoint?: ParentAssistantTextCheckpointResult; readonly livePublication?: readonly ExecutionLivePublicationReceipt[]; readonly planQuestions?: ExecutionPlanQuestionsReceipt; readonly planOutput?: PlanRecord } | { readonly kind: "conflict"; readonly operationId: string; readonly recoveryState?: "not-started" | "already-terminal" }; /** @@ -124,7 +136,7 @@ export interface ExecutionSemanticWriter { /** A command result that never calls an uncommitted mutation successful. */ export type ExecutionWorkerResult = - | { readonly kind: "committed"; readonly operationId: string; readonly durableRevision: number; readonly providerCommit?: ExecutionProviderCommitReceipt; readonly providerEvents?: readonly ProjectedCommittedProviderEvent[]; readonly assistantTextCheckpoint?: ParentAssistantTextCheckpointResult; readonly livePublication?: readonly ExecutionLivePublicationReceipt[] } + | { readonly kind: "committed"; readonly operationId: string; readonly durableRevision: number; readonly providerCommit?: ExecutionProviderCommitReceipt; readonly providerEvents?: readonly ProjectedCommittedProviderEvent[]; readonly assistantTextCheckpoint?: ParentAssistantTextCheckpointResult; readonly livePublication?: readonly ExecutionLivePublicationReceipt[]; readonly planQuestions?: ExecutionPlanQuestionsReceipt; readonly planOutput?: PlanRecord } | { readonly kind: "released" } | { readonly kind: "rejected"; readonly reason: "no-execution" | "stale-execution" | "out-of-order" | "invalid-transition" | "invalid-event-routing" | "invalid-text-routing" | "invalid-narrative-routing" | "invalid-stop-watermark" | "writer-conflict" }; @@ -301,6 +313,8 @@ function liveEventMutation( ...(command.taskIntents ? { taskIntents: command.taskIntents } : {}), ...(command.systemIntents ? { systemIntents: command.systemIntents } : {}), ...(command.message ? { message: command.message } : {}), + ...(command.planQuestions ? { planQuestions: command.planQuestions } : {}), + ...(command.planOutput ? { planOutput: command.planOutput } : {}), }; } @@ -380,6 +394,7 @@ function validLiveEventRouting( execution: ExecutionIdentity, ): boolean { if (command.narrative !== undefined && command.narrative?.executionId !== execution.executionId) return false; + if (!validPlanMessageRouting(command)) return false; switch (command.text?.kind) { case "unchanged": case "reclassify": @@ -393,6 +408,10 @@ function validLiveEventRouting( } } +function validPlanMessageRouting(command: Extract): boolean { + return command.planOutput === undefined || command.message !== undefined; +} + function validStartInput( command: Extract, execution: ExecutionIdentity, @@ -460,6 +479,8 @@ function committedResult(receipt: Extract Date: Thu, 24 Sep 2026 22:54:24 +0100 Subject: [PATCH 103/136] feat(server): admit parent turns through execution owner --- .../turn-admission-dispatch-coordinator.ts | 38 ++++++++++++++----- 1 file changed, 28 insertions(+), 10 deletions(-) diff --git a/apps/server/src/features/agents/turns/turn-admission-dispatch-coordinator.ts b/apps/server/src/features/agents/turns/turn-admission-dispatch-coordinator.ts index 8558ddd02..883724a69 100644 --- a/apps/server/src/features/agents/turns/turn-admission-dispatch-coordinator.ts +++ b/apps/server/src/features/agents/turns/turn-admission-dispatch-coordinator.ts @@ -51,6 +51,7 @@ import { } from "./parent-turn-durability.js"; import { appendSelectedTextComments } from "./selected-text-comment-append.js"; import { ApprovalReviewPolicy, type ApprovalReviewDecision } from "./approval-review-policy.js"; +import type { ExecutionIdentity } from "../execution/execution-mailbox-protocol.js"; const FILE_INJECTION_SEPARATOR = "\n\n---\n"; @@ -125,6 +126,16 @@ export interface TurnRuntimeAdmissionAuthority { owns(lease: TurnRuntimeLease): boolean; } +/** Commits a new parent turn through its exact execution mailbox. */ +export interface TurnParentStartOwner { + start(input: { + readonly execution: ExecutionIdentity; + readonly ownerEpoch: number; + readonly providerId: string; + readonly parentTurn: DataOnlyParentTurnStartInput; + }): Promise; +} + /** The thread-control action selected during generic dispatch preparation. */ export type ThreadControlLeaseDirective = | { @@ -150,6 +161,7 @@ export interface PreparedTurnDispatch { readonly threadControl: ThreadControlLeaseDirective | null; readonly contextSeed: number; readonly contextWindow: number | null; + readonly workerOwned?: true; } /** The stable outcome of handling a complete send command. */ @@ -194,6 +206,7 @@ export class TurnAdmissionDispatchCoordinator { private readonly environment: WorkspaceEnvironmentService | undefined, private readonly files: FileService | undefined, private readonly platform: NodeJS.Platform, + private readonly parentStartOwner?: TurnParentStartOwner, ) {} /** Admit one command and return an immutable package for the runtime owner. */ @@ -422,7 +435,19 @@ export class TurnAdmissionDispatchCoordinator { runtime.activate(lease); const attachmentData = await this.persistAttachments(prepared); const sourceTurnId = prepared.command.sourceTurnId ?? NodeCrypto.randomUUID(); - const parentStartInput = this.startParentTurn(prepared, lease, sourceTurnId, attachmentData, review); + const parentStartInput = this.prepareParentTurnStartInput(prepared, lease, sourceTurnId, attachmentData, review); + if (this.parentStartOwner) { + await this.parentStartOwner.start({ + execution: { threadId: lease.threadId, turnId: sourceTurnId, executionId: lease.turnExecutionId }, + ownerEpoch: lease.generation, + providerId: prepared.providerId, + parentTurn: parentStartInput, + }); + if (parentStartInput.reopenThread) { + const reopened = this.threads.findById(lease.threadId); + if (reopened) broadcast("thread.lifecycleChanged", { thread: reopened }); + } + } else this.startParentTurn(parentStartInput); this.publishCommittedEffects(prepared, sourceTurnId); const wirePayload = this.buildWirePayload(prepared); const request = await this.buildTurnRequest(prepared, lease, sourceTurnId, attachmentData, cwd, wirePayload, review); @@ -438,6 +463,7 @@ export class TurnAdmissionDispatchCoordinator { threadControl: this.threadControlDirective(prepared, sourceTurnId), contextSeed: prepared.thread.last_context_tokens ?? 0, contextWindow: prepared.thread.context_window, + ...(this.parentStartOwner ? { workerOwned: true as const } : {}), }; } @@ -689,14 +715,7 @@ export class TurnAdmissionDispatchCoordinator { return prepared.automaticAttachments ?? this.persistCommandAttachments(prepared.command); } - private startParentTurn( - prepared: PreparedCommand, - lease: TurnRuntimeLease, - sourceTurnId: string, - attachments: PersistedAttachmentData, - review: ApprovalReviewDecision, - ): DataOnlyParentTurnStartInput { - const input = this.prepareParentTurnStartInput(prepared, lease, sourceTurnId, attachments, review); + private startParentTurn(input: DataOnlyParentTurnStartInput): void { const { userMessage, reopenThread, answeredPlanQuestionMessageId, ...start } = input; let reopenedThread: Exclude, null> | null = null; this.parentTurns.startParentTurn({ @@ -709,7 +728,6 @@ export class TurnAdmissionDispatchCoordinator { }, }); if (reopenedThread) broadcast("thread.lifecycleChanged", { thread: reopenedThread }); - return input; } private prepareParentTurnStartInput( From 577b9830fe51dd204a79e03f44df9ac1176bf6cf Mon Sep 17 00:00:00 2001 From: "(CJ) Chukwudi Nwobodo" <142016413+chuks-qua@users.noreply.github.com> Date: Fri, 25 Sep 2026 10:49:15 +0100 Subject: [PATCH 104/136] fix(providers): bind Codex callbacks to dispatch attempts --- .../__tests__/provider-event-ingress.test.ts | 24 ++- .../composition/provider-event-ingress.ts | 1 + .../provider-event-worker-protocol.ts | 1 + .../src/events/provider-runtime-event.ts | 1 + .../codex/codex-provider-first-turn.test.ts | 60 +++++- .../codex-provider-subagent-turn.test.ts | 14 +- .../src/private/codex/codex-provider.ts | 196 +++++++++++++----- 7 files changed, 239 insertions(+), 58 deletions(-) diff --git a/apps/server/src/features/providers/composition/__tests__/provider-event-ingress.test.ts b/apps/server/src/features/providers/composition/__tests__/provider-event-ingress.test.ts index e47cb267d..e6b28c6bd 100644 --- a/apps/server/src/features/providers/composition/__tests__/provider-event-ingress.test.ts +++ b/apps/server/src/features/providers/composition/__tests__/provider-event-ingress.test.ts @@ -4,6 +4,7 @@ import { describe, expect, it, vi } from "vitest"; import { container, Lifecycle } from "tsyringe"; import { AgentEventType, + ProviderRuntimeEventSchema, type CanonicalAgentEventEnvelope, type IAgentProvider, type ProviderId, @@ -30,8 +31,9 @@ import type { CodexCollaborationDurability } from "../../../agents/collaboration const EXECUTION_ID = "00000000-0000-4000-8000-000000000001"; -function runtimeEvent(delta: string, threadId = "thread-1"): ProviderRuntimeEvent { +function runtimeEvent(delta: string, threadId = "thread-1", deliveryAttempt?: number): ProviderRuntimeEvent { return { + ...(deliveryAttempt === undefined ? {} : { deliveryAttempt }), event: { type: AgentEventType.TextDelta, threadId, @@ -68,7 +70,7 @@ function oversizedToolResult(threadId: string): ProviderRuntimeEvent { }; } -function committedEnvelope(eventId: string, delta: string): CanonicalAgentEventEnvelope { +function committedEnvelope(eventId: string, delta: string, deliveryAttempt?: number, sourceProviderId: ProviderId = "claude"): CanonicalAgentEventEnvelope { return { eventId, routing: { @@ -77,7 +79,7 @@ function committedEnvelope(eventId: string, delta: string): CanonicalAgentEventE executionId: EXECUTION_ID, itemId: `item-${eventId}`, }, - sourceProviderId: "claude", + sourceProviderId, sourceIdentities: [], acceptedSequence: 1, durableRevision: 1, @@ -90,7 +92,7 @@ function committedEnvelope(eventId: string, delta: string): CanonicalAgentEventE turnId: "turn-1", kind: "system", providerIdentities: [], - payload: { projection: "providerRuntimeEvent", runtimeEvent: runtimeEvent(delta) }, + payload: { projection: "providerRuntimeEvent", runtimeEvent: runtimeEvent(delta, "thread-1", deliveryAttempt) }, createdAt: "2026-08-27T12:00:00.000Z", updatedAt: "2026-08-27T12:00:00.000Z", }, @@ -229,6 +231,20 @@ describe("ProviderEventIngress", () => { expect(received[0]?.canonicalReceipt).toBeUndefined(); }); + it("keeps private delivery attempt metadata through runtime and committed ingress", async () => { + const provider = createProvider("codex"); + const { ingress, received } = createIngress([provider]); + const runtime = ProviderRuntimeEventSchema().parse(runtimeEvent("live", "thread-1", 2)); + + (provider as unknown as NodeEvents.EventEmitter).emit("event", structuredClone(runtime)); + ingress.acceptCommitted([committedEnvelope("committed-attempt", "committed", 2, "codex")]); + await flushIngress(); + + expect(received.map((event) => event.deliveryAttempt)).toEqual([2, 2]); + expect(received.map((event) => event.event.type)).toEqual([AgentEventType.TextDelta, AgentEventType.TextDelta]); + expect(ProviderRuntimeEventSchema().safeParse({ ...runtime, deliveryAttempt: 0 }).success).toBe(false); + }); + it("keeps direct canonical commits and runtime events in arrival order", async () => { const cursor = createProvider("cursor"); const { ingress, received } = createIngress([createProvider("claude"), cursor]); diff --git a/apps/server/src/features/providers/composition/provider-event-ingress.ts b/apps/server/src/features/providers/composition/provider-event-ingress.ts index 0f5ea0d0e..9c9fa4df1 100644 --- a/apps/server/src/features/providers/composition/provider-event-ingress.ts +++ b/apps/server/src/features/providers/composition/provider-event-ingress.ts @@ -83,6 +83,7 @@ export interface ProviderEventIngressEvent { providerId: ProviderId; sourceKind: ProviderEventSourceKind; event: AgentEvent; + deliveryAttempt?: number; runtimeExtension?: ProviderRuntimeEvent["extension"]; canonicalReceipt?: CanonicalProviderEventReceipt; } diff --git a/apps/server/src/features/providers/composition/provider-event-worker-protocol.ts b/apps/server/src/features/providers/composition/provider-event-worker-protocol.ts index e21da498f..04f75f811 100644 --- a/apps/server/src/features/providers/composition/provider-event-worker-protocol.ts +++ b/apps/server/src/features/providers/composition/provider-event-worker-protocol.ts @@ -203,6 +203,7 @@ function runtimeIngressEvent( providerId, sourceKind, event: normalizeEvent(runtimeEvent.event), + ...(runtimeEvent.deliveryAttempt !== undefined ? { deliveryAttempt: runtimeEvent.deliveryAttempt } : {}), ...(runtimeEvent.extension ? { runtimeExtension: runtimeEvent.extension } : {}), }; } diff --git a/packages/contracts/src/events/provider-runtime-event.ts b/packages/contracts/src/events/provider-runtime-event.ts index cab378144..7a526c900 100644 --- a/packages/contracts/src/events/provider-runtime-event.ts +++ b/packages/contracts/src/events/provider-runtime-event.ts @@ -63,6 +63,7 @@ export const ProviderRuntimeExtensionSchema = lazySchema(() => z export const ProviderRuntimeEventSchema = lazySchema(() => z.object({ event: AgentEventSchema(), + deliveryAttempt: z.number().int().positive().optional(), extension: ProviderRuntimeExtensionSchema().optional(), }).strict(), ); diff --git a/packages/providers/src/__tests__/codex/codex-provider-first-turn.test.ts b/packages/providers/src/__tests__/codex/codex-provider-first-turn.test.ts index d029576a2..7cb192d16 100644 --- a/packages/providers/src/__tests__/codex/codex-provider-first-turn.test.ts +++ b/packages/providers/src/__tests__/codex/codex-provider-first-turn.test.ts @@ -81,7 +81,7 @@ vi.mock("../../private/codex/codex-app-server.js", async () => { import { BrowserAutomationSessionLease, CodexProvider, stubEnvService } from "./codex-provider-test-fixture.js"; import { AgentEventSchema, AgentEventType } from "@mcode/contracts"; -import type { ProviderRuntimeEvent, ProviderTurnDiffUpdate } from "@mcode/contracts"; +import type { ProviderFileMutationStart, ProviderRuntimeEvent, ProviderTurnDiffUpdate } from "@mcode/contracts"; const schemaValidExecutionId = "00000000-0000-4000-8000-000000000001"; @@ -217,6 +217,63 @@ describe("CodexProvider first turn on new session", () => { provider.shutdown(); }); + it("keeps repeated public item IDs on their native attempts and rejects old server callbacks", async () => { + const provider = makeProvider(); + const events: ProviderRuntimeEvent[] = []; + const mutations: ProviderFileMutationStart[] = []; + provider.on("event", (event: ProviderRuntimeEvent) => events.push(event)); + provider.on("file_mutation_start", (event: ProviderFileMutationStart) => mutations.push(event)); + sendTurnMock.mockResolvedValueOnce("native-attempt-1").mockResolvedValueOnce("native-attempt-2"); + const request = { + turnId: "mcode-turn", turnExecutionId: schemaValidExecutionId, deliveryAttempt: 1, + sessionId, workspaceId: "workspace-test", threadId, message: "edit", cwd: process.cwd(), + model: "gpt-5.4", interactionMode: "build" as const, providerOptions: {}, + permissionMode: "supervised" as const, approvalReviewMode: "automatic" as const, + }; + const notifyTool = (server: (typeof appServers)[number], turnId: string) => server.emit("notification", { + method: "item/started", + params: { + threadId: "sdk-thread-1", turnId, + item: { type: "fileChange", id: "same-native-item", changes: [{ path: "tracked.txt", kind: "edit" }] }, + }, + }); + const notifyUnknownChild = (server: (typeof appServers)[number]) => server.emit("notification", { + method: "item/started", + params: { + threadId: "unknown-child", + item: { type: "fileChange", id: "unbound-child-edit", changes: [{ path: "child.txt", kind: "edit" }] }, + }, + }); + + await provider.sendTurn(request); + await new Promise((resolve) => setImmediate(resolve)); + const firstServer = appServers[0]!; + notifyTool(firstServer, "native-attempt-1"); + notifyUnknownChild(firstServer); + await provider.discardSession(sessionId); + + await provider.sendTurn({ ...request, deliveryAttempt: 2, resumeFrom: undefined }); + await new Promise((resolve) => setImmediate(resolve)); + const retryServer = appServers[1]!; + notifyTool(firstServer, "native-attempt-1"); + notifyUnknownChild(firstServer); + notifyTool(retryServer, "native-attempt-2"); + notifyUnknownChild(retryServer); + retryServer.emit("notification", { + method: "turn/completed", + params: { threadId: "sdk-thread-1", turn: { id: "native-attempt-2", status: "completed", usage: {} } }, + }); + await new Promise((resolve) => setImmediate(resolve)); + + expect(events.filter((event) => event.event.type === AgentEventType.ToolUse + && event.event.toolCallId === "same-native-item").map((event) => event.deliveryAttempt)).toEqual([1, 2]); + expect(mutations.filter((event) => event.toolCallId === "unbound-child-edit") + .map((event) => event.deliveryAttempt)).toEqual([undefined, undefined]); + expect(events.find((event) => event.event.type === AgentEventType.Ended + && event.event.turnExecutionId === schemaValidExecutionId && event.deliveryAttempt === 2)).toBeDefined(); + await provider.shutdown(); + }); + it("passes loopback browser MCP config and a child-only bearer token", async () => { const lease = new BrowserAutomationSessionLease(); const inheritedBrowserToken = process.env.MCODE_BROWSER_MCP_TOKEN; @@ -917,6 +974,7 @@ describe("CodexProvider first turn on new session", () => { expect(interruptRejected).toBe(true); expect(server.isAlive).toBe(false); expect(events).toContainEqual({ + deliveryAttempt: 1, event: { type: AgentEventType.Ended, threadId, diff --git a/packages/providers/src/__tests__/codex/codex-provider-subagent-turn.test.ts b/packages/providers/src/__tests__/codex/codex-provider-subagent-turn.test.ts index 2d1e0fda0..2aff53038 100644 --- a/packages/providers/src/__tests__/codex/codex-provider-subagent-turn.test.ts +++ b/packages/providers/src/__tests__/codex/codex-provider-subagent-turn.test.ts @@ -134,9 +134,12 @@ describe("CodexProvider sub-agent turn lifecycle isolation", () => { const entry = await startSession(provider, "mcode-file-attempt", "file-attempt"); const state = entry as PoolEntry & { turnExecutionIdsByNativeTurn: Map; + turnDeliveryAttemptsByNativeTurn: Map; + nativeExecutionConflictKeys: Set; turnDiffRouting?: { turnId: string; turnExecutionId: string; deliveryAttempt: number }; }; state.turnExecutionIdsByNativeTurn.set("bound-native", "same-execution"); + state.turnDeliveryAttemptsByNativeTurn.set("bound-native", 1); state.turnDiffRouting = { turnId: "test-turn", turnExecutionId: "same-execution", deliveryAttempt: 2 }; const item = { type: "fileChange", id: "edit", changes: [{ path: "tracked.txt", kind: "edit" }] }; @@ -150,8 +153,12 @@ describe("CodexProvider sub-agent turn lifecycle isolation", () => { expect(mutations[0]?.turnExecutionId).toBeUndefined(); expect(mutations[0]?.deliveryAttempt).toBeUndefined(); expect(mutations[1]).toMatchObject({ - toolCallId: "edit-current", turnExecutionId: "same-execution", deliveryAttempt: 2, + toolCallId: "edit-current", turnExecutionId: "same-execution", deliveryAttempt: 1, }); + state.nativeExecutionConflictKeys.add("bound-native"); + entry.server.emit("notification", { method: "item/started", + params: { threadId: "early-child", turnId: "bound-native", item: { ...item, id: "edit-conflicted" } } }); + expect(mutations[2]?.deliveryAttempt).toBeUndefined(); }); it("assigns the active execution to main tool and text events before turn/start binds", async () => { @@ -493,6 +500,8 @@ describe("CodexProvider sub-agent turn lifecycle isolation", () => { turnBindingPhase: "idle" | "awaiting" | "bound"; currentNativeTurnId?: string; turnExecutionIdsByNativeTurn: Map; + turnDeliveryAttemptsByNativeTurn: Map; + turnDiffRouting?: { turnId: string; turnExecutionId: string; deliveryAttempt: number }; childExecutionGenerations: Map; nativeThreadExecutionIds: Map; pendingChildEvents: unknown[]; @@ -503,6 +512,8 @@ describe("CodexProvider sub-agent turn lifecycle isolation", () => { state.turnBindingPhase = "bound"; state.currentNativeTurnId = "main-native-turn"; state.turnExecutionIdsByNativeTurn.set("main-native-turn", state.currentTurnExecutionId); + state.turnDeliveryAttemptsByNativeTurn.set("main-native-turn", 2); + state.turnDiffRouting = { turnId: "test-turn", turnExecutionId: state.currentTurnExecutionId, deliveryAttempt: 2 }; entry.server.emit("notification", { method: "item/started", @@ -578,6 +589,7 @@ describe("CodexProvider sub-agent turn lifecycle isolation", () => { "child-two-layer-item", ]); expect(childEvents).toHaveLength(3); + expect(childEvents.every((runtimeEvent) => runtimeEvent.deliveryAttempt === undefined)).toBe(true); expect(childEvents.every((runtimeEvent) => ( runtimeEvent.event.turnExecutionId === "exec-mcode-child-two-layer-replay" diff --git a/packages/providers/src/private/codex/codex-provider.ts b/packages/providers/src/private/codex/codex-provider.ts index e0ef4169c..b51fd0eb1 100644 --- a/packages/providers/src/private/codex/codex-provider.ts +++ b/packages/providers/src/private/codex/codex-provider.ts @@ -329,6 +329,8 @@ interface CodexSessionState { pendingTurnStartNotification?: { nativeTurnId: string; executionId: string }; /** Immutable Mcode execution identity keyed by native Codex turn id. */ turnExecutionIdsByNativeTurn: Map; + /** Immutable dispatch attempt keyed by the same authoritative native turn id. */ + turnDeliveryAttemptsByNativeTurn: Map; /** Current generation execution identity keyed by authoritative child thread id. */ nativeThreadExecutionIds: Map; /** Execution currently owning the active main turn, retained through drains. */ @@ -338,7 +340,7 @@ interface CodexSessionState { /** Bounded conflict fingerprints already logged for this session. */ nativeExecutionConflictKeys: Set; /** Current generation for each authoritative child thread linkage. */ - childExecutionGenerations: Map; + childExecutionGenerations: Map; /** Monotonic child generation counter, bounded by the child map lifetime. */ nextChildGeneration: number; /** Child threads already queried for authoritative model metadata this session. */ @@ -669,6 +671,8 @@ export class CodexProvider extends NodeEvents.EventEmitter implements IAgentProv threadControlEligible: boolean; } >(); + /** Guards callbacks before pool registration and after a retry replaces the app-server. */ + private activeCodexServers = new Map(); /** Browser scope staged only until a fresh main app-server process starts. */ private pendingBrowserAccess = new Map { @@ -1410,13 +1419,10 @@ export class CodexProvider extends NodeEvents.EventEmitter implements IAgentProv const mapper = new CodexEventMapper(threadId, undefined, (event) => { const state = this.runtime.get(sessionId); const nativeTurnId = event.nativeTurnId; - const turnExecutionId = nativeTurnId && state?.mapper === mapper - && !state.nativeExecutionConflictKeys.has(nativeTurnId) - ? state.turnExecutionIdsByNativeTurn.get(nativeTurnId) : undefined; - const deliveryAttempt = turnExecutionId && state?.turnDiffRouting?.turnExecutionId === turnExecutionId - ? state.turnDiffRouting.deliveryAttempt : undefined; + const bound = state?.mapper === mapper ? this.nativeTurnAttempt(state, nativeTurnId) : undefined; this.emit("file_mutation_start", { - threadId: event.threadId, ...(deliveryAttempt ? { turnExecutionId, deliveryAttempt } : {}), + threadId: event.threadId, + ...(bound ? { turnExecutionId: bound.executionId, deliveryAttempt: bound.deliveryAttempt } : {}), toolCallId: event.toolCallId, toolName: event.toolName, toolInput: event.toolInput, } satisfies ProviderFileMutationStart); @@ -1428,6 +1434,7 @@ export class CodexProvider extends NodeEvents.EventEmitter implements IAgentProv private attachCodexServerEvents(context: CodexSpawnContext, server: CodexAppServer, mapper: CodexEventMapper): void { server.on("invalidNotification", () => { + if (this.activeCodexServers.get(context.sessionId) !== server) return; for (const event of mapper.mapNotification(undefined)) this.emitRuntimeEvent(event); }); server.on("notification", (notification: CodexNotification) => this.handleCodexServerNotification(context, server, mapper, notification)); @@ -1450,12 +1457,12 @@ export class CodexProvider extends NodeEvents.EventEmitter implements IAgentProv ): void { const rawNotification = notification; const entry = this.runtime.get(context.sessionId); + if (!this.isActiveCodexServer(context.sessionId, server, mapper, entry)) return; const mainNotification = isMainThreadNotification(server, rawNotification.params); const nativeThreadId = nativeThreadIdFromParams(rawNotification.params); const nativeTurnId = nativeTurnIdFromParams(rawNotification.params); - const knownExecutionId = nativeTurnId ? entry?.turnExecutionIdsByNativeTurn.get(nativeTurnId) : undefined; // Reject old main-turn events before they can reset the shared mapper for a new dispatch. - if (mainNotification && knownExecutionId && knownExecutionId !== entry?.currentTurnExecutionId) return; + if (mainNotification && !this.isCurrentCodexMainTurn(entry, nativeTurnId)) return; this.handleCodexUsageNotification(rawNotification, server, context.threadId); const replayThreadId = this.bindCodexTurnStarted(entry, rawNotification.method, mainNotification, nativeThreadId, nativeTurnId); const events = this.mapCodexNotificationEvents(context.threadId, notification, mapper, rawNotification); @@ -1466,7 +1473,24 @@ export class CodexProvider extends NodeEvents.EventEmitter implements IAgentProv this.deliverCodexNotificationEvents({ entry, sessionId: context.sessionId, mapper, mappedEvents: events, mainNotification, nativeThreadId, nativeTurnId, eventExecutionId: executionId, startupEventExecutionId: startupExecutionId }); if (entry && replayThreadId) this.replayPendingChildEvents(entry, context.sessionId, replayThreadId); this.fetchCompletedChildMetadata(context.sessionId, context.threadId, server, mapper, rawNotification.method, nativeThreadId, executionId); - this.handleDiscoveredCodexChildren({ entry, sessionId: context.sessionId, threadId: context.threadId, server, mapper, notification: rawNotification, nativeThreadId, eventExecutionId: executionId }); + this.handleDiscoveredCodexChildren({ entry, sessionId: context.sessionId, threadId: context.threadId, server, mapper, notification: rawNotification, nativeThreadId, nativeTurnId, eventExecutionId: executionId }); + } + + private isActiveCodexServer(sessionId: string, server: CodexAppServer, mapper: CodexEventMapper, entry: CodexSessionState | undefined): boolean { + return this.activeCodexServers.get(sessionId) === server + && (!entry || (entry.server === server && entry.mapper === mapper)); + } + + private isCurrentCodexMainTurn(entry: CodexSessionState | undefined, nativeTurnId: string | undefined): boolean { + if (!entry || !nativeTurnId) return true; + const knownExecutionId = entry.turnExecutionIdsByNativeTurn.get(nativeTurnId); + return (!knownExecutionId || knownExecutionId === entry.currentTurnExecutionId) + && this.matchesCurrentCodexAttempt(entry, nativeTurnId); + } + + private matchesCurrentCodexAttempt(entry: CodexSessionState, nativeTurnId: string | undefined): boolean { + const nativeAttempt = this.nativeTurnAttempt(entry, nativeTurnId); + return !nativeAttempt || nativeAttempt.deliveryAttempt === entry.turnDiffRouting?.deliveryAttempt; } /** Subscribe to complete native turn diffs without routing their bytes through renderer events. */ @@ -1499,10 +1523,13 @@ export class CodexProvider extends NodeEvents.EventEmitter implements IAgentProv mapper: CodexEventMapper, error: string, ): void { + if (this.activeCodexServers.get(context.sessionId) !== server) return; logger.error("CodexAppServer fatal", { sessionId: context.sessionId, error, breadcrumb: server.lastTransportBreadcrumb }); - const executionId = this.runtime.get(context.sessionId)?.activeParentTurnExecutionId ?? context.stagedExecutionId; + const entry = this.runtime.get(context.sessionId); + const executionId = entry?.activeParentTurnExecutionId ?? context.stagedExecutionId; for (const event of mapper.drainPendingAssistantBoundary(false)) { - this.emitRuntimeEvent(providerRuntimeEvent(executionId ? { ...event, turnExecutionId: executionId } : event)); + const runtimeEvent = providerRuntimeEvent(executionId ? { ...event, turnExecutionId: executionId } : event); + this.emitRuntimeEvent(entry ? this.withCodexTurnAttempt(entry, runtimeEvent, executionId) : runtimeEvent); } this.emitTurnFailure(context.threadId, error, undefined, true, executionId); if (this.runtime.get(context.sessionId)?.server === server) { @@ -1574,7 +1601,9 @@ export class CodexProvider extends NodeEvents.EventEmitter implements IAgentProv } private attachCodexThreadIdentity(context: CodexSpawnContext, server: CodexAppServer, mapper: CodexEventMapper): void { - server.on("threadIdChanged", (newThreadId: string) => this.recordCodexThreadIdentity(context, mapper, newThreadId)); + server.on("threadIdChanged", (newThreadId: string) => { + if (this.activeCodexServers.get(context.sessionId) === server) this.recordCodexThreadIdentity(context, mapper, newThreadId); + }); if (server.resumeFailed) { logger.warn("Codex session context lost; resume failed, started fresh thread", { sessionId: context.sessionId }); this.emitRuntimeEvent(providerRuntimeEvent({ type: AgentEventType.System, threadId: context.threadId, subtype: "context_lost" } satisfies AgentEvent)); @@ -1593,7 +1622,7 @@ export class CodexProvider extends NodeEvents.EventEmitter implements IAgentProv sessionId: context.sessionId, threadId: context.threadId, cwd: context.cwd, server, mapper, nextTurnExecutionId: this.pendingSpawnTurns.get(context.sessionId)?.turnExecutionId, lastUsedAt: Date.now(), sandboxMode: context.sandbox, runTurnSeq: 0, pendingTurnId: null, - turnBindingPhase: "idle", turnStartResponsePending: false, turnExecutionIdsByNativeTurn: new Map(), + turnBindingPhase: "idle", turnStartResponsePending: false, turnExecutionIdsByNativeTurn: new Map(), turnDeliveryAttemptsByNativeTurn: new Map(), nativeThreadExecutionIds: new Map(), nativeExecutionConflictKeys: new Set(), childExecutionGenerations: new Map(), turnDiffRevision: 0, nextChildGeneration: 0, pendingChildEvents: [], deliveredChildEventKeys: new Set(), workspaceId: context.browserAccess?.workspaceId ?? "unknown-workspace", @@ -1691,18 +1720,21 @@ export class CodexProvider extends NodeEvents.EventEmitter implements IAgentProv nativeTurnId: string | undefined, ): string | undefined { if (!state.activeParentTurnExecutionId) return undefined; - const executionId = nativeThreadId ? state.childExecutionGenerations.get(nativeThreadId)?.executionId : undefined; + const child = nativeThreadId ? state.childExecutionGenerations.get(nativeThreadId) : undefined; + const executionId = child?.executionId; const replayThreadId = nativeTurnId && executionId && nativeThreadId - ? this.bindChildNativeTurnStart(state, nativeThreadId, nativeTurnId, executionId) + ? this.bindChildNativeTurnStart(state, nativeThreadId, nativeTurnId, executionId, child.deliveryAttempt) : undefined; pruneExecutionMap(state.nativeThreadExecutionIds); pruneExecutionMap(state.turnExecutionIdsByNativeTurn); + this.pruneNativeTurnAttempts(state); return replayThreadId; } - private bindChildNativeTurnStart(state: CodexSessionState, nativeThreadId: string, nativeTurnId: string, executionId: string): string | undefined { + private bindChildNativeTurnStart(state: CodexSessionState, nativeThreadId: string, nativeTurnId: string, executionId: string, deliveryAttempt?: number): string | undefined { const assignment = assignNativeExecution(state.turnExecutionIdsByNativeTurn, nativeTurnId, executionId, state.nativeExecutionConflictKeys); - return assignment === "conflict" ? undefined : nativeThreadId; + return assignment === "conflict" || !this.rememberNativeTurnAttempt(state, nativeTurnId, executionId, deliveryAttempt) + ? undefined : nativeThreadId; } private mapCodexNotificationEvents( @@ -1728,7 +1760,32 @@ export class CodexProvider extends NodeEvents.EventEmitter implements IAgentProv ): void { if (!state || !nativeThreadId || mainNotification || !mapper.hasReceiverThread(nativeThreadId) || !state.activeParentTurnExecutionId) return; state.nativeThreadExecutionIds.set(nativeThreadId, state.activeParentTurnExecutionId); - if (nativeTurnId && method === "turn/started") state.turnExecutionIdsByNativeTurn.set(nativeTurnId, state.activeParentTurnExecutionId); + if (nativeTurnId && method === "turn/started") { + const assignment = assignNativeExecution(state.turnExecutionIdsByNativeTurn, nativeTurnId, state.activeParentTurnExecutionId, state.nativeExecutionConflictKeys); + if (assignment !== "conflict") this.rememberNativeTurnAttempt(state, nativeTurnId, state.activeParentTurnExecutionId, state.childExecutionGenerations.get(nativeThreadId)?.deliveryAttempt); + } + } + + private nativeTurnAttempt(state: CodexSessionState, nativeTurnId: string | undefined): { executionId: string; deliveryAttempt: number } | undefined { + if (!nativeTurnId || state.nativeExecutionConflictKeys.has(nativeTurnId)) return undefined; + const executionId = state.turnExecutionIdsByNativeTurn.get(nativeTurnId); + const deliveryAttempt = state.turnDeliveryAttemptsByNativeTurn.get(nativeTurnId); + return executionId && deliveryAttempt !== undefined ? { executionId, deliveryAttempt } : undefined; + } + + private rememberNativeTurnAttempt(state: CodexSessionState, nativeTurnId: string, executionId: string, deliveryAttempt: number | undefined): boolean { + if (deliveryAttempt === undefined) return true; + if (state.turnExecutionIdsByNativeTurn.get(nativeTurnId) !== executionId) return false; + const existing = state.turnDeliveryAttemptsByNativeTurn.get(nativeTurnId); + if (existing !== undefined && existing !== deliveryAttempt) return false; + state.turnDeliveryAttemptsByNativeTurn.set(nativeTurnId, deliveryAttempt); + return true; + } + + private pruneNativeTurnAttempts(state: CodexSessionState): void { + for (const nativeTurnId of state.turnDeliveryAttemptsByNativeTurn.keys()) { + if (!state.turnExecutionIdsByNativeTurn.has(nativeTurnId)) state.turnDeliveryAttemptsByNativeTurn.delete(nativeTurnId); + } } private codexNotificationExecutionId( @@ -1781,13 +1838,28 @@ export class CodexProvider extends NodeEvents.EventEmitter implements IAgentProv args: Omit[0], "mappedEvents">, event: ProviderRuntimeEvent, ): void { - const eventKey = childEventIdentity(event); + const boundEvent = this.withCodexDeliveryAttempt(args, event); + const eventKey = childEventIdentity(boundEvent); if (this.isDuplicateCodexChildEvent(args.entry, eventKey)) return; - if (this.shouldBufferCodexChildEvent(args, event)) { - if (args.entry && args.nativeThreadId) bufferPendingChildEvent(args.entry, event, args.nativeThreadId, args.nativeTurnId, eventKey); + if (this.shouldBufferCodexChildEvent(args, boundEvent)) { + if (args.entry && args.nativeThreadId) bufferPendingChildEvent(args.entry, boundEvent, args.nativeThreadId, args.nativeTurnId, eventKey); return; } - this.emitCodexNotificationEvent(args, event, eventKey); + this.emitCodexNotificationEvent(args, boundEvent, eventKey); + } + + private withCodexDeliveryAttempt( + args: Omit[0], "mappedEvents">, + event: ProviderRuntimeEvent, + ): ProviderRuntimeEvent { + if (!args.entry) return event; + // Replayed child events share the parent notification batch. Their own native turn is the only safe attempt source. + const nativeTurnId = event.extension?.child + ? event.extension.child.nativeTurnId + : args.nativeTurnId; + const bound = this.nativeTurnAttempt(args.entry, nativeTurnId); + if (!bound || (args.eventExecutionId && bound.executionId !== args.eventExecutionId)) return event; + return { ...event, deliveryAttempt: bound.deliveryAttempt }; } private isDuplicateCodexChildEvent(state: CodexSessionState | undefined, eventKey: string | undefined): boolean { @@ -1851,6 +1923,7 @@ export class CodexProvider extends NodeEvents.EventEmitter implements IAgentProv mapper: CodexEventMapper; notification: { method?: string; params?: Record }; nativeThreadId: string | undefined; + nativeTurnId: string | undefined; eventExecutionId: string | undefined; }): void { const childThreadId = nativeSubAgentThreadId(args.notification); @@ -1864,22 +1937,27 @@ export class CodexProvider extends NodeEvents.EventEmitter implements IAgentProv args: Parameters[0], childThreadId: string, ): void { - if (this.isCurrentCodexParentNotification(args)) this.registerCodexChildGeneration(args.entry as CodexSessionState, args.sessionId, childThreadId); + if (this.isCurrentCodexParentNotification(args)) this.registerCodexChildGeneration(args.entry as CodexSessionState, args.sessionId, childThreadId, args.nativeTurnId); this.fetchChildThreadMetadata(args.sessionId, args.threadId, args.server, args.mapper, childThreadId, args.eventExecutionId, true); } private isCurrentCodexParentNotification(args: Parameters[0]): boolean { const executionId = args.entry?.currentTurnExecutionId; if (!args.entry || !executionId || args.eventExecutionId !== executionId) return false; + if (!this.matchesCurrentCodexAttempt(args.entry, args.nativeTurnId)) return false; if (args.server.threadId && args.nativeThreadId === args.server.threadId) return true; return Boolean(args.nativeThreadId && args.entry.nativeThreadExecutionIds.get(args.nativeThreadId) === executionId); } - private registerCodexChildGeneration(state: CodexSessionState, sessionId: string, childThreadId: string): void { + private registerCodexChildGeneration(state: CodexSessionState, sessionId: string, childThreadId: string, parentNativeTurnId: string | undefined): void { const executionId = state.currentTurnExecutionId; if (!executionId) return; state.nextChildGeneration += 1; - state.childExecutionGenerations.set(childThreadId, { executionId, generation: state.nextChildGeneration }); + const parentAttempt = this.nativeTurnAttempt(state, parentNativeTurnId); + state.childExecutionGenerations.set(childThreadId, { + executionId, generation: state.nextChildGeneration, + ...(parentAttempt?.executionId === executionId ? { deliveryAttempt: parentAttempt.deliveryAttempt } : {}), + }); pruneChildGenerationMap(state.childExecutionGenerations); state.nativeThreadExecutionIds.set(childThreadId, executionId); pruneExecutionMap(state.nativeThreadExecutionIds); @@ -1917,9 +1995,11 @@ export class CodexProvider extends NodeEvents.EventEmitter implements IAgentProv private matchesChildEventGeneration( item: PendingChildEvent, - childMapping: { executionId: string; generation: number }, + childMapping: { executionId: string; generation: number; deliveryAttempt?: number }, executionId: string, ): boolean { + if (item.event.deliveryAttempt !== undefined && childMapping.deliveryAttempt !== undefined + && item.event.deliveryAttempt !== childMapping.deliveryAttempt) return false; if (item.childGeneration !== undefined) return item.childGeneration === childMapping.generation; return !item.executionIdAtBuffer || item.executionIdAtBuffer === executionId; } @@ -2016,9 +2096,8 @@ export class CodexProvider extends NodeEvents.EventEmitter implements IAgentProv async interrupt(state: CodexSessionState): Promise { const turnExecutionId = executionForDrain(state); for (const event of state.mapper.drainPendingAssistantBoundary(false)) { - this.emitRuntimeEvent(providerRuntimeEvent( - turnExecutionId ? { ...event, turnExecutionId } : event, - )); + const runtimeEvent = providerRuntimeEvent(turnExecutionId ? { ...event, turnExecutionId } : event); + this.emitRuntimeEvent(this.withCodexTurnAttempt(state, runtimeEvent, turnExecutionId)); } const nativeTurnId = state.currentNativeTurnId; if (nativeTurnId) { @@ -2026,24 +2105,24 @@ export class CodexProvider extends NodeEvents.EventEmitter implements IAgentProv await state.server.interruptTurnAndDrain(nativeTurnId); } catch (error) { if (turnExecutionId) { - this.emitRuntimeEvent(providerRuntimeEvent({ + this.emitCodexTurnEvent(state, { type: AgentEventType.Ended, threadId: state.threadId, turnExecutionId, reason: "provider_lost", - } satisfies AgentEvent)); + } satisfies AgentEvent, turnExecutionId); } throw error; } } else { await state.server.interruptTurn(); if (turnExecutionId) { - this.emitRuntimeEvent(providerRuntimeEvent({ + this.emitCodexTurnEvent(state, { type: AgentEventType.Ended, threadId: state.threadId, turnExecutionId, reason: "provider_lost", - } satisfies AgentEvent)); + } satisfies AgentEvent, turnExecutionId); } } state.pendingTurnStartNotification = undefined; @@ -2062,12 +2141,12 @@ export class CodexProvider extends NodeEvents.EventEmitter implements IAgentProv * Drives shutdown, eviction, and stale-discard, but not turn cancellation. */ async close(state: CodexSessionState): Promise { + if (this.activeCodexServers.get(state.sessionId) === state.server) this.activeCodexServers.delete(state.sessionId); await this.host.threadControl.close(state.sessionId); const turnExecutionId = executionForDrain(state); for (const event of state.mapper.drainPendingAssistantBoundary(false)) { - this.emitRuntimeEvent(providerRuntimeEvent( - turnExecutionId ? { ...event, turnExecutionId } : event, - )); + const runtimeEvent = providerRuntimeEvent(turnExecutionId ? { ...event, turnExecutionId } : event); + this.emitRuntimeEvent(this.withCodexTurnAttempt(state, runtimeEvent, turnExecutionId)); } this.liveSessionIds.delete(state.sessionId); state.pendingTurnStartNotification = undefined; @@ -2321,11 +2400,17 @@ export class CodexProvider extends NodeEvents.EventEmitter implements IAgentProv } private emitCodexPendingAssistantBoundary(entry: CodexSessionState, turnExecutionId: string): void { - for (const event of entry.mapper.drainPendingAssistantBoundary(false)) this.emitCodexTurnEvent(event, turnExecutionId); + for (const event of entry.mapper.drainPendingAssistantBoundary(false)) this.emitCodexTurnEvent(entry, event, turnExecutionId); + } + + private emitCodexTurnEvent(entry: CodexSessionState, event: AgentEvent, turnExecutionId: string): void { + this.emitRuntimeEvent(this.withCodexTurnAttempt(entry, providerRuntimeEvent({ ...event, turnExecutionId }), turnExecutionId)); } - private emitCodexTurnEvent(event: AgentEvent, turnExecutionId: string): void { - this.emitRuntimeEvent(providerRuntimeEvent({ ...event, turnExecutionId })); + private withCodexTurnAttempt(entry: CodexSessionState, event: ProviderRuntimeEvent, turnExecutionId: string | undefined): ProviderRuntimeEvent { + const nativeAttempt = this.nativeTurnAttempt(entry, entry.currentNativeTurnId); + return nativeAttempt && nativeAttempt.executionId === turnExecutionId + ? { ...event, deliveryAttempt: nativeAttempt.deliveryAttempt } : event; } private async waitForCodexTurn( @@ -2492,10 +2577,16 @@ export class CodexProvider extends NodeEvents.EventEmitter implements IAgentProv } const assignment = assignNativeExecution(entry.turnExecutionIdsByNativeTurn, turnId, turnExecutionId, entry.nativeExecutionConflictKeys); if (assignment === "conflict") throw new Error("Codex turn/start response reused native turn id"); + const deliveryAttempt = entry.turnDiffRouting?.turnExecutionId === turnExecutionId + ? entry.turnDiffRouting.deliveryAttempt : undefined; + if (!this.rememberNativeTurnAttempt(entry, turnId, turnExecutionId, deliveryAttempt)) { + throw new Error("Codex turn/start response reused native turn id across dispatch attempts"); + } entry.currentNativeTurnId = turnId; entry.pendingTurnId = turnId; entry.turnBindingPhase = "bound"; pruneExecutionMap(entry.turnExecutionIdsByNativeTurn); + this.pruneNativeTurnAttempts(entry); } private handleCodexTurnWaitError( @@ -2545,7 +2636,7 @@ export class CodexProvider extends NodeEvents.EventEmitter implements IAgentProv ): void { if (run.seq === run.entry.runTurnSeq) this.clearCodexTurnRunState(run.entry); if (!completion.serverDied && run.seq === run.entry.runTurnSeq) { - this.emitCodexTurnEvent(completion.deferredEnded ?? { + this.emitCodexTurnEvent(run.entry, completion.deferredEnded ?? { type: AgentEventType.Ended, threadId, turnExecutionId, @@ -2789,6 +2880,7 @@ export class CodexProvider extends NodeEvents.EventEmitter implements IAgentProv this.pendingSpawnTurns.clear(); this.pendingBrowserAccess.clear(); this.liveSessionIds.clear(); + this.activeCodexServers.clear(); logger.info("CodexProvider shutdown complete"); } } From 68b5dbae42554e263e94dafee530c5a170069e15 Mon Sep 17 00:00:00 2001 From: "(CJ) Chukwudi Nwobodo" <142016413+chuks-qua@users.noreply.github.com> Date: Fri, 25 Sep 2026 10:49:15 +0100 Subject: [PATCH 105/136] feat(server): route provider batches to execution owner --- .../__tests__/turn-event-pipeline.test.ts | 16 +- .../turns/provider-turn-event-application.ts | 5 + .../agents/turns/turn-event-pipeline.ts | 8 + .../claude-canonical-event-publisher.test.ts | 2 + .../__tests__/provider-event-ingress.test.ts | 35 +++- .../__tests__/provider-host-ports.test.ts | 188 +++++++++++++++++- .../canonical-live-event-publisher.ts | 2 + .../composition/provider-event-ingress.ts | 36 +++- .../composition/provider-host-ports.ts | 170 +++++++++++++--- packages/providers/src/host-ports.ts | 4 + .../cursor-canonical-event-publisher.test.ts | 4 + .../cursor-canonical-event-publisher.ts | 2 + 12 files changed, 436 insertions(+), 36 deletions(-) diff --git a/apps/server/src/features/agents/turns/__tests__/turn-event-pipeline.test.ts b/apps/server/src/features/agents/turns/__tests__/turn-event-pipeline.test.ts index c9b96df95..0b3be33b6 100644 --- a/apps/server/src/features/agents/turns/__tests__/turn-event-pipeline.test.ts +++ b/apps/server/src/features/agents/turns/__tests__/turn-event-pipeline.test.ts @@ -61,13 +61,16 @@ function createPipeline( pipeline: TurnEventPipeline; finalize: ReturnType; rejectForQueueCapacity: ReturnType; + publishCommitted: ReturnType; } { const lifecycle: TurnLifecycleControl = { normalize: (event) => event, finalize, }; + const publishCommitted = vi.fn(); const application: TurnEventApplication = { apply, + publishCommitted, observeFileMutation: vi.fn(), rejectForQueueCapacity, previousFileFinalization, @@ -75,10 +78,21 @@ function createPipeline( observeToolUse: vi.fn(), observeToolResult: vi.fn(), }; - return { pipeline: new TurnEventPipeline(lifecycle, application, undefined, ingressFence), finalize, rejectForQueueCapacity }; + return { pipeline: new TurnEventPipeline(lifecycle, application, undefined, ingressFence), finalize, rejectForQueueCapacity, publishCommitted }; } describe("TurnEventPipeline", () => { + it("publishes projected worker output without legacy event application", () => { + const apply = vi.fn(() => true); + const { pipeline, publishCommitted } = createPipeline(apply); + const input = { ...textDelta("durable"), sourceKind: "worker-commit" as const }; + + pipeline.handleProjectedCommitted(input); + + expect(publishCommitted).toHaveBeenCalledWith(input.event); + expect(apply).not.toHaveBeenCalled(); + }); + it("keeps canonical receipt provenance and FIFO order while an earlier checkpoint delays publication", () => { let checkpointReady = false; const received: Array<{ input: ProviderEventIngressEvent; event: AgentEvent }> = []; diff --git a/apps/server/src/features/agents/turns/provider-turn-event-application.ts b/apps/server/src/features/agents/turns/provider-turn-event-application.ts index 844120ca8..ce74dbe0c 100644 --- a/apps/server/src/features/agents/turns/provider-turn-event-application.ts +++ b/apps/server/src/features/agents/turns/provider-turn-event-application.ts @@ -112,6 +112,11 @@ export class ProviderTurnEventApplication implements TurnEventApplication { return this.applyPreparedEvent(input, event, publish); } + /** Publish an execution-writer event without repeating any turn persistence. */ + publishCommitted(event: AgentEvent): void { + this.publish(event); + } + /** Record a provider file mutation before its public event is available. */ observeFileMutation(event: import("@mcode/contracts").ProviderFileMutationStart): void { this.fileEffects.observeProviderMutation(event); diff --git a/apps/server/src/features/agents/turns/turn-event-pipeline.ts b/apps/server/src/features/agents/turns/turn-event-pipeline.ts index 6d1704b5d..58e5b0d8d 100644 --- a/apps/server/src/features/agents/turns/turn-event-pipeline.ts +++ b/apps/server/src/features/agents/turns/turn-event-pipeline.ts @@ -41,6 +41,8 @@ export interface TurnLifecycleControl { export interface TurnEventApplication { /** Apply a validated event after the pipeline admits it in per-turn order. */ apply(input: ProviderEventIngressEvent, event: AgentEvent, publish: boolean): boolean | Promise; + /** Publish an event whose complete semantic work already committed on the execution writer. */ + publishCommitted(event: AgentEvent): void; /** Record a provider file mutation before its corresponding public event arrives. */ observeFileMutation(event: ProviderFileMutationStart): void; /** Abort one turn only when its completion cannot be compacted into the bounded queue. */ @@ -102,6 +104,12 @@ export class TurnEventPipeline implements ProviderEventIngressConsumer { this.drain(event.threadId); } + /** Publish committed worker output without passing it through legacy persistence. */ + handleProjectedCommitted(input: ProviderEventIngressEvent): void { + const event = this.lifecycle.normalize(input.event); + if (event) this.application.publishCommitted(event); + } + /** Stop only the affected turn when provider ingress cannot retain one of its events. */ handleProviderIngressOverflow(input: ProviderEventIngressEvent): void { this.application.rejectForQueueCapacity(input.event); diff --git a/apps/server/src/features/providers/adapters/claude/__tests__/claude-canonical-event-publisher.test.ts b/apps/server/src/features/providers/adapters/claude/__tests__/claude-canonical-event-publisher.test.ts index 960e314f4..732554da8 100644 --- a/apps/server/src/features/providers/adapters/claude/__tests__/claude-canonical-event-publisher.test.ts +++ b/apps/server/src/features/providers/adapters/claude/__tests__/claude-canonical-event-publisher.test.ts @@ -46,6 +46,8 @@ describe("ClaudeCanonicalEventPublisher", () => { threadId: routing.threadId, turnId: routing.turnId, executionId: routing.executionId, + batchId: `claude:${routing.executionId}:attempt:1:event:1`, + deliveryAttempt: 1, events: [expect.objectContaining({ eventId: `claude:${routing.executionId}:attempt:1:event:1`, sourceProviderId: "claude", diff --git a/apps/server/src/features/providers/composition/__tests__/provider-event-ingress.test.ts b/apps/server/src/features/providers/composition/__tests__/provider-event-ingress.test.ts index e6b28c6bd..c27491498 100644 --- a/apps/server/src/features/providers/composition/__tests__/provider-event-ingress.test.ts +++ b/apps/server/src/features/providers/composition/__tests__/provider-event-ingress.test.ts @@ -28,6 +28,7 @@ import { } from "../provider-event-worker-protocol.js"; import { CodexCollaborationEventAdapter } from "../../../agents/collaboration/adapters/codex-collaboration-event-adapter.js"; import type { CodexCollaborationDurability } from "../../../agents/collaboration/codex-collaboration-durability.js"; +import type { ProjectedCommittedProviderEvent } from "../../../agents/execution/execution-worker-handler.js"; const EXECUTION_ID = "00000000-0000-4000-8000-000000000001"; @@ -100,6 +101,20 @@ function committedEnvelope(eventId: string, delta: string, deliveryAttempt?: num }; } +function projectedEvent(eventId: string, delta: string): ProjectedCommittedProviderEvent { + return { + providerId: "claude", + sourceKind: "canonical-commit", + event: runtimeEvent(delta).event, + canonicalReceipt: { + eventId, + acceptedSequence: 1, + durableRevision: 1, + serverTimestamps: { acceptedAt: "2026-08-27T12:00:00.000Z" }, + }, + }; +} + function createProvider(id: ProviderId): IAgentProvider { return Object.assign(new NodeEvents.EventEmitter(), { id }) as unknown as IAgentProvider; } @@ -172,6 +187,7 @@ function createIngress( ) { const diagnostics: ProviderEventIngressDiagnostic[] = []; const received: ProviderEventIngressEvent[] = []; + const projected: ProviderEventIngressEvent[] = []; const overflowed: ProviderEventIngressEvent[] = []; const registry = { resolveAll: () => providers } as never; const ingress = new ProviderEventIngress( @@ -181,10 +197,11 @@ function createIngress( ); ingress.start(registry, { handleProviderEvent: (event) => received.push(event), + handleProjectedCommitted: (event) => projected.push(event), handleProviderFileMutation: vi.fn(), handleProviderIngressOverflow: (event) => overflowed.push(event), }); - return { diagnostics, ingress, overflowed, provider: providers[0], received }; + return { diagnostics, ingress, overflowed, projected, provider: providers[0], received }; } async function flushIngress(): Promise { @@ -277,6 +294,22 @@ describe("ProviderEventIngress", () => { expect(received).toHaveLength(1); }); + it("queues projected writer events in order without calling the legacy consumer", async () => { + const { ingress, projected, received } = createIngress(); + const first = projectedEvent("worker-first", "first"); + const second = projectedEvent("worker-second", "second"); + + ingress.acceptProjectedCommitted([first, second]); + expect(projected).toEqual([]); + await ingress.waitForThread("thread-1"); + + expect(projected.map((item) => item.canonicalReceipt?.eventId)).toEqual(["worker-first", "worker-second"]); + expect(received).toEqual([]); + ingress.acceptProjectedCommitted([first]); + await ingress.waitForThread("thread-1"); + expect(projected).toHaveLength(2); + }); + it("does not invoke the consumer on the provider callback stack", async () => { const { provider, received } = createIngress(); diff --git a/apps/server/src/features/providers/composition/__tests__/provider-host-ports.test.ts b/apps/server/src/features/providers/composition/__tests__/provider-host-ports.test.ts index 4e94cef32..2bb9d276b 100644 --- a/apps/server/src/features/providers/composition/__tests__/provider-host-ports.test.ts +++ b/apps/server/src/features/providers/composition/__tests__/provider-host-ports.test.ts @@ -1,7 +1,12 @@ import { describe, expect, it, vi } from "vitest"; -import type { CanonicalAgentEventEnvelope } from "@mcode/contracts"; +import { ProviderIdSchema, type CanonicalAgentEventEnvelope } from "@mcode/contracts"; +import type { ProviderEventBatch } from "@mcode/providers"; -import { createProviderHostPorts } from "../provider-host-ports.js"; +import { + createProviderHostPorts, + type ProviderEventOwnershipRoute, + type WorkerOwnedProviderEventResult, +} from "../provider-host-ports.js"; const EXECUTION_ID = "00000000-0000-4000-8000-000000000001"; @@ -35,6 +40,65 @@ function committedRuntimeEnvelope(): CanonicalAgentEventEnvelope { }; } +function ownedBatch(providerId: "claude" | "cursor", deliveryAttempt: number, sequence: number): ProviderEventBatch { + const eventId = `${providerId}:${EXECUTION_ID}:attempt:${deliveryAttempt}:event:${sequence}`; + const itemId = `${providerId}:${EXECUTION_ID}:attempt:${deliveryAttempt}:item:${sequence}`; + const envelope = committedRuntimeEnvelope(); + if (envelope.payload.type !== "item.recorded") throw new Error("Expected recorded item fixture"); + return { + threadId: "thread-1", + turnId: "turn-1", + executionId: EXECUTION_ID, + batchId: eventId, + deliveryAttempt, + phase: "running", + events: [{ + eventId, + routing: { ...envelope.routing, itemId }, + sourceProviderId: providerId, + sourceIdentities: [], + sourceSequence: sequence, + payload: { + type: "item.recorded", + item: { ...envelope.payload.item, id: itemId }, + }, + }], + }; +} + +function workerCommit(batch: ProviderEventBatch, sequence: number): WorkerOwnedProviderEventResult { + const [draft] = batch.events; + if (!draft || !batch.batchId || !batch.deliveryAttempt) throw new Error("Expected owned batch fixture"); + return { + batchId: batch.batchId, + deliveryAttempt: batch.deliveryAttempt, + commit: { + outcome: "committed", + conversationRevision: sequence, + rosterRevision: 0, + acceptedThrough: sequence, + durableThrough: sequence, + eventCount: 1, + }, + providerEvents: [{ + providerId: ProviderIdSchema.parse(draft.sourceProviderId), + sourceKind: "canonical-commit", + event: { + type: "textDelta", + threadId: batch.threadId, + turnExecutionId: batch.executionId, + delta: `output ${sequence}`, + }, + canonicalReceipt: { + eventId: draft.eventId, + acceptedSequence: sequence, + durableRevision: sequence, + serverTimestamps: { acceptedAt: "2026-08-27T12:00:00.000Z" }, + }, + }], + }; +} + describe("createProviderHostPorts", () => { it("adapts the server browser grant to the Provider contract", () => { const ports = createProviderHostPorts({ @@ -143,4 +207,124 @@ describe("createProviderHostPorts", () => { await expect(ports.events.submit(batch)).rejects.toThrow("commit failed"); expect(acceptCommitted).not.toHaveBeenCalled(); }); + + it.each(["claude", "cursor"] as const)("delivers %s worker batches only after ordered commit replies", async (providerId) => { + const commit = vi.fn(); + const accepted: string[] = []; + let acknowledge: ((value: WorkerOwnedProviderEventResult) => void) | undefined; + const firstReply = new Promise((resolve) => { acknowledge = resolve; }); + const submit = vi.fn() + .mockImplementationOnce(() => firstReply) + .mockImplementation((batch: ProviderEventBatch) => Promise.resolve(workerCommit(batch, 2))); + const route: ProviderEventOwnershipRoute = { + kind: "worker", + threadId: "thread-1", + turnId: "turn-1", + executionId: EXECUTION_ID, + deliveryAttempt: 2, + submit, + }; + const ports = createProviderHostPorts({ + events: { commit }, + ingress: { acceptProjectedCommitted: (events: WorkerOwnedProviderEventResult["providerEvents"]) => + accepted.push(...events.map((event) => event.canonicalReceipt.eventId)) }, + eventOwnership: { resolve: () => route }, + } as never); + const first = ownedBatch(providerId, 2, 1); + const second = ownedBatch(providerId, 2, 2); + + const pending = ports.events.submit(first); + expect(commit).not.toHaveBeenCalled(); + expect(accepted).toEqual([]); + acknowledge?.(workerCommit(first, 1)); + await expect(pending).resolves.toMatchObject({ + commit: { outcome: "committed", acceptedThrough: 1 }, + delivery: { ingress: "queued" }, + }); + await ports.events.submit(second); + expect(accepted).toEqual([first.events[0]?.eventId, second.events[0]?.eventId]); + expect(submit).toHaveBeenCalledTimes(2); + expect(commit).not.toHaveBeenCalled(); + }); + + it("rejects a stale attempt or mismatched receipt without falling back to the legacy writer", async () => { + const batch = ownedBatch("claude", 1, 1); + const commit = vi.fn(); + const acceptProjectedCommitted = vi.fn(); + const submit = vi.fn(() => Promise.resolve({ ...workerCommit(batch, 1), deliveryAttempt: 2 })); + const route: ProviderEventOwnershipRoute = { + kind: "worker", + threadId: batch.threadId, + turnId: batch.turnId, + executionId: batch.executionId, + deliveryAttempt: 2, + submit, + }; + const ports = createProviderHostPorts({ + events: { commit }, + ingress: { acceptProjectedCommitted }, + eventOwnership: { resolve: () => route }, + } as never); + + await expect(ports.events.submit(batch)).rejects.toThrow("does not match execution ownership"); + expect(submit).not.toHaveBeenCalled(); + route.deliveryAttempt = 1; + await expect(ports.events.submit(batch)).rejects.toThrow("receipt does not match submitted batch"); + batch.batchId = undefined; + await expect(ports.events.submit(batch)).rejects.toThrow("does not match execution ownership"); + expect(commit).not.toHaveBeenCalled(); + expect(acceptProjectedCommitted).not.toHaveBeenCalled(); + }); + + it("propagates a worker commit conflict without publishing or using the legacy writer", async () => { + const batch = ownedBatch("cursor", 1, 1); + const commit = vi.fn(); + const acceptProjectedCommitted = vi.fn(); + const result = workerCommit(batch, 1); + const ports = createProviderHostPorts({ + events: { commit }, + ingress: { acceptProjectedCommitted }, + eventOwnership: { resolve: () => ({ + kind: "worker", + threadId: batch.threadId, + turnId: batch.turnId, + executionId: batch.executionId, + deliveryAttempt: 1, + submit: () => Promise.resolve({ + ...result, + commit: { ...result.commit, outcome: "conflict" }, + providerEvents: [], + }), + }) }, + } as never); + + await expect(ports.events.submit(batch)).rejects.toThrow("Canonical worker batch failed: conflict"); + expect(commit).not.toHaveBeenCalled(); + expect(acceptProjectedCommitted).not.toHaveBeenCalled(); + }); + + it("propagates worker failure and rejected ownership without a legacy commit", async () => { + const batch = ownedBatch("cursor", 1, 1); + const commit = vi.fn(); + const submit = vi.fn(() => Promise.reject(new Error("worker unavailable"))); + let route: ProviderEventOwnershipRoute = { + kind: "worker", + threadId: batch.threadId, + turnId: batch.turnId, + executionId: batch.executionId, + deliveryAttempt: 1, + submit, + }; + const ports = createProviderHostPorts({ + events: { commit }, + ingress: { acceptProjectedCommitted: vi.fn() }, + eventOwnership: { resolve: () => route }, + } as never); + + await expect(ports.events.submit(batch)).rejects.toThrow("worker unavailable"); + route = { kind: "rejected" }; + await expect(ports.events.submit(batch)).rejects.toThrow("no longer admitted"); + expect(submit).toHaveBeenCalledTimes(1); + expect(commit).not.toHaveBeenCalled(); + }); }); diff --git a/apps/server/src/features/providers/composition/canonical-live-event-publisher.ts b/apps/server/src/features/providers/composition/canonical-live-event-publisher.ts index 91651f3a7..8b87b2105 100644 --- a/apps/server/src/features/providers/composition/canonical-live-event-publisher.ts +++ b/apps/server/src/features/providers/composition/canonical-live-event-publisher.ts @@ -55,6 +55,8 @@ export class CanonicalLiveEventPublisher { threadId: routing.threadId, turnId: routing.turnId, executionId: routing.executionId, + batchId: draft.eventId, + deliveryAttempt: routing.deliveryAttempt, phase: "running", events: [draft], }); diff --git a/apps/server/src/features/providers/composition/provider-event-ingress.ts b/apps/server/src/features/providers/composition/provider-event-ingress.ts index 9c9fa4df1..82b436031 100644 --- a/apps/server/src/features/providers/composition/provider-event-ingress.ts +++ b/apps/server/src/features/providers/composition/provider-event-ingress.ts @@ -32,6 +32,7 @@ import { } from "./provider-event-worker-protocol.js"; import { normalizeProviderError } from "./provider-error-normalize.js"; import { eventApplyType, serverWorkTrace } from "../../agents/diagnostics/server-work-trace.js"; +import type { ProjectedCommittedProviderEvent } from "../../agents/execution/execution-worker-handler.js"; const MAX_PENDING_NON_TERMINAL_EVENTS = 8_192; const MAX_PENDING_NON_TERMINAL_EVENTS_PER_THREAD = 2_048; @@ -67,7 +68,7 @@ const MAX_OVERFLOWED_TURNS = 16_384; export const PROVIDER_EVENT_INGRESS_DIAGNOSTIC_SINK = Symbol("ProviderEventIngressDiagnosticSink"); /** Identifies the transport path that delivered one provider runtime event. */ -export type ProviderEventSourceKind = "provider-runtime" | "canonical-commit"; +export type ProviderEventSourceKind = "provider-runtime" | "canonical-commit" | "worker-commit"; /** Durable canonical metadata that must remain attached to a projected live event. */ export interface CanonicalProviderEventReceipt { @@ -91,6 +92,8 @@ export interface ProviderEventIngressEvent { /** Downstream contract for applying accepted provider events in the turn pipeline. */ export interface ProviderEventIngressConsumer { handleProviderEvent(event: ProviderEventIngressEvent): void; + /** Publish a writer-owned event without running the legacy turn write pipeline. */ + handleProjectedCommitted?(event: ProviderEventIngressEvent): void; handleProviderFileMutation(event: ProviderFileMutationStart): void; handleProviderTurnDiff(event: ProviderTurnDiffUpdate): void; /** Stop one turn when ingress cannot retain one of its provider events. */ @@ -264,6 +267,24 @@ export class ProviderEventIngress { for (const envelope of envelopes) this.acceptCanonicalEnvelope(envelope); } + /** Deliver writer-projected events after their durable reply without reapplying old turn writes. */ + acceptProjectedCommitted(events: readonly ProjectedCommittedProviderEvent[]): void { + if (this.stopped || !this.consumer?.handleProjectedCommitted) { + throw new Error("Worker-owned provider publication is unavailable"); + } + for (const event of events) { + const eventId = event.canonicalReceipt.eventId; + if (this.seenCanonicalEventIds.has(eventId) || this.pendingCanonicalEventIds.has(eventId)) { + this.report({ reason: "duplicate-event", sourceKind: "worker-commit", eventId }); + continue; + } + if (!this.enqueue({ ...event, sourceKind: "worker-commit" })) { + throw new Error("Worker-owned provider publication queue is full"); + } + this.rememberCanonicalEvent(eventId); + } + } + private acceptCanonicalEnvelope(envelope: unknown): void { const task: ProviderEventWorkerTask = { kind: "canonical-commit", envelope }; const eventId = canonicalEventIdentity(task); @@ -397,6 +418,10 @@ export class ProviderEventIngress { queueCapacity: "global" | "thread" | "global-bytes" | "thread-bytes" | "terminal", ): false { this.rejectedQueueEvents += 1; + if (event.sourceKind === "worker-commit") { + this.reportQueueDiagnostic("queue-capacity", event, queue?.length ?? 0, queueCapacity); + return false; + } if (!this.rememberOverflowedTurn(event)) return false; this.reportQueueDiagnostic("queue-capacity", event, queue?.length ?? 0, queueCapacity); this.discardPendingExecution(event); @@ -460,13 +485,18 @@ export class ProviderEventIngress { serverWorkTrace.record("mailbox-wait", queued.event.event.threadId, queued.event.event.turnExecutionId, NodePerfHooks.performance.now() - queued.traceQueuedAt); serverWorkTrace.measure("event-apply", queued.event.event.threadId, - queued.event.event.turnExecutionId, () => this.consumer?.handleProviderEvent(queued.event), + queued.event.event.turnExecutionId, () => this.deliverQueuedEvent(queued.event), eventApplyType(queued.event.event.type)); - } else this.consumer.handleProviderEvent(queued.event); + } else this.deliverQueuedEvent(queued.event); } if (this.pendingEventCount > 0) this.scheduleYieldedDrain(); } + private deliverQueuedEvent(event: ProviderEventIngressEvent): void { + if (event.sourceKind === "worker-commit") this.consumer?.handleProjectedCommitted?.(event); + else this.consumer?.handleProviderEvent(event); + } + private takeNextPendingEvent(): QueuedProviderEvent | undefined { while (this.readyThreadIds.length > 0) { const threadId = this.readyThreadIds.shift(); diff --git a/apps/server/src/features/providers/composition/provider-host-ports.ts b/apps/server/src/features/providers/composition/provider-host-ports.ts index 890f578d9..aee2205bf 100644 --- a/apps/server/src/features/providers/composition/provider-host-ports.ts +++ b/apps/server/src/features/providers/composition/provider-host-ports.ts @@ -1,4 +1,10 @@ -import type { ProviderHostPorts } from "@mcode/providers"; +import type { + ProviderEventBatch, + ProviderEventCommitReceipt, + ProviderHostPorts, + ProviderEventSubmissionReceipt, +} from "@mcode/providers"; +import type { CanonicalAgentEventEnvelope } from "@mcode/contracts"; import type { HostRuntime } from "@mcode/shared/node/host-runtime"; import type { JobObject } from "../../../runtime/process/containment/job-object.js"; import type { EnvService } from "../../../runtime/environment/env-service.js"; @@ -8,6 +14,39 @@ import type { BrowserAutomationSessionLease } from "../../browser-automation/ind import type { InternalThreadControlMcpRuntime } from "../../thread-control/index.js"; import { killProcessTree } from "../../../runtime/process/containment/process-kill.js"; import type { ProviderEventIngress } from "./provider-event-ingress.js"; +import type { ProjectedCommittedProviderEvent } from "../../agents/execution/execution-worker-handler.js"; + +/** A provider batch with the stable identity required by a worker owner. */ +export type WorkerOwnedProviderEventBatch = ProviderEventBatch & { + batchId: string; + deliveryAttempt: number; +}; + +/** One acknowledged worker commit and its projected live events, bound to the batch. */ +export interface WorkerOwnedProviderEventResult { + batchId: string; + deliveryAttempt: number; + commit: ProviderEventCommitReceipt; + providerEvents: readonly ProjectedCommittedProviderEvent[]; +} + +/** One durable route decision for an execution. Retired executions must resolve to rejected. */ +export type ProviderEventOwnershipRoute = + | { kind: "legacy" } + | { kind: "rejected" } + | { + kind: "worker"; + threadId: string; + turnId: string; + executionId: string; + deliveryAttempt: number; + submit(batch: WorkerOwnedProviderEventBatch): Promise; + }; + +/** Selects one commit owner for every batch of an exact execution. */ +export interface ProviderEventOwnership { + resolve(executionId: string): ProviderEventOwnershipRoute; +} /** Server services used to compose the narrow Provider host-port boundary. */ export interface ProviderHostPortDependencies { @@ -19,6 +58,7 @@ export interface ProviderHostPortDependencies { grants: ScopedPreGrantService; events: CanonicalAgentBoundary; ingress: ProviderEventIngress; + eventOwnership?: ProviderEventOwnership; } /** Adapts server-owned services to the only host operations exposed to Providers. */ @@ -82,38 +122,110 @@ export function createProviderHostPorts( consume: (request) => dependencies.grants.tryConsume(request), }, events: { - submit: async (batch) => { - const result = dependencies.events.commit({ - threadId: batch.threadId, - turnId: batch.turnId, - executionId: batch.executionId, - phase: batch.phase, - nativeCursor: batch.nativeCursor, - events: batch.events, - }); - if (result.outcome === "committed" && result.events.length > 0) { - dependencies.ingress.acceptCommitted(result.events); - } - return { - commit: { - outcome: providerCommitOutcome(result.outcome), - conversationRevision: result.conversationRevision, - rosterRevision: result.rosterRevision, - acceptedThrough: result.acceptedThrough, - durableThrough: result.durableThrough, - eventCount: result.events.length, - }, - delivery: { - ingress: result.outcome === "committed" && result.events.length > 0 - ? "queued" - : "not-required", - }, - }; - }, + submit: (batch) => submitProviderEvents(dependencies, batch), }, }; } +async function submitProviderEvents( + dependencies: ProviderHostPortDependencies, + batch: ProviderEventBatch, +): Promise { + const route: ProviderEventOwnershipRoute = dependencies.eventOwnership?.resolve(batch.executionId) + ?? { kind: "legacy" }; + if (route.kind === "rejected") throw new Error("Canonical event execution is no longer admitted"); + if (route.kind === "worker") return await submitWorkerEvents(dependencies, route, batch); + return submitLegacyEvents(dependencies, batch); +} + +async function submitWorkerEvents( + dependencies: ProviderHostPortDependencies, + route: Extract, + batch: ProviderEventBatch, +): Promise { + const ownedBatch = checkedWorkerBatch(batch, route); + const result = await route.submit(ownedBatch); + if (!matchesWorkerReceipt(ownedBatch, result)) { + throw new Error("Canonical worker receipt does not match submitted batch"); + } + if (result.commit.outcome === "conflict" || result.commit.outcome === "ingest-overflow") { + throw new Error(`Canonical worker batch failed: ${result.commit.outcome}`); + } + if (result.commit.outcome === "committed" && result.providerEvents.length > 0) { + dependencies.ingress.acceptProjectedCommitted(result.providerEvents); + } + return { + commit: result.commit, + delivery: { ingress: result.commit.outcome === "committed" && result.providerEvents.length > 0 + ? "queued" : "not-required" }, + }; +} + +function submitLegacyEvents( + dependencies: ProviderHostPortDependencies, + batch: ProviderEventBatch, +): ProviderEventSubmissionReceipt { + const result = dependencies.events.commit({ + threadId: batch.threadId, + turnId: batch.turnId, + executionId: batch.executionId, + phase: batch.phase, + nativeCursor: batch.nativeCursor, + events: batch.events, + }); + return deliverCommitted(dependencies.ingress, { + outcome: providerCommitOutcome(result.outcome), + conversationRevision: result.conversationRevision, + rosterRevision: result.rosterRevision, + acceptedThrough: result.acceptedThrough, + durableThrough: result.durableThrough, + eventCount: result.events.length, + }, result.events); +} + +function deliverCommitted( + ingress: ProviderEventIngress, + commit: ProviderEventCommitReceipt, + events: readonly CanonicalAgentEventEnvelope[], +): ProviderEventSubmissionReceipt { + const handoff = commit.outcome === "committed" && events.length > 0; + if (handoff) ingress.acceptCommitted(events); + return { commit, delivery: { ingress: handoff ? "queued" : "not-required" } }; +} + +function matchesWorkerReceipt( + batch: WorkerOwnedProviderEventBatch, + result: WorkerOwnedProviderEventResult, +): boolean { + if (result.batchId !== batch.batchId || result.deliveryAttempt !== batch.deliveryAttempt) return false; + if (result.commit.outcome !== "committed") return result.providerEvents.length === 0; + const submittedIds = new Set(batch.events.map((event) => event.eventId)); + return result.providerEvents.length <= result.commit.eventCount + && result.providerEvents.every((event) => submittedIds.has(event.canonicalReceipt.eventId)); +} + +function checkedWorkerBatch( + batch: ProviderEventBatch, + route: Extract, +): WorkerOwnedProviderEventBatch { + if (!batch.batchId || !Number.isSafeInteger(batch.deliveryAttempt) || (batch.deliveryAttempt ?? 0) < 1 + || batch.deliveryAttempt !== route.deliveryAttempt + || !matchesExecution(batch, route) + || batch.events.length === 0 + || !batch.events.every((event) => matchesExecution(event.routing, route))) { + throw new Error("Canonical worker batch does not match execution ownership"); + } + return { ...batch, batchId: batch.batchId, deliveryAttempt: batch.deliveryAttempt }; +} + +function matchesExecution( + routing: { threadId: string; turnId?: string; executionId: string }, + route: Pick, +): boolean { + return routing.threadId === route.threadId && routing.turnId === route.turnId + && routing.executionId === route.executionId; +} + function providerCommitOutcome( outcome: "committed" | "duplicate" | "conflict" | "terminal-outcome-confirmed" | "ingest-overflow", ): "committed" | "duplicate" | "conflict" | "ingest-overflow" { diff --git a/packages/providers/src/host-ports.ts b/packages/providers/src/host-ports.ts index 5fa605369..03e7320f5 100644 --- a/packages/providers/src/host-ports.ts +++ b/packages/providers/src/host-ports.ts @@ -124,6 +124,10 @@ export interface ProviderEventBatch { threadId: string; turnId: string; executionId: string; + /** Stable across retries of this submission when an execution has a worker owner. */ + batchId?: string; + /** Provider delivery attempt that produced this batch. */ + deliveryAttempt?: number; phase: string; nativeCursor?: unknown; events: readonly ProviderEventDraft[]; diff --git a/packages/providers/src/private/cursor/__tests__/cursor-canonical-event-publisher.test.ts b/packages/providers/src/private/cursor/__tests__/cursor-canonical-event-publisher.test.ts index 726ee0a68..968d1f56e 100644 --- a/packages/providers/src/private/cursor/__tests__/cursor-canonical-event-publisher.test.ts +++ b/packages/providers/src/private/cursor/__tests__/cursor-canonical-event-publisher.test.ts @@ -48,6 +48,8 @@ describe("CursorCanonicalEventPublisher", () => { threadId: routing.threadId, turnId: routing.turnId, executionId: routing.executionId, + batchId: `cursor:${routing.executionId}:attempt:1:event:1`, + deliveryAttempt: 1, phase: "running", events: [{ eventId: `cursor:${routing.executionId}:attempt:1:event:1`, @@ -65,6 +67,8 @@ describe("CursorCanonicalEventPublisher", () => { }); AgentEventRoutingSchema.parse(textBatch.events[0]?.routing); expect(terminalBatch).toMatchObject({ + batchId: `cursor:${routing.executionId}:attempt:1:event:2`, + deliveryAttempt: 1, events: [{ eventId: `cursor:${routing.executionId}:attempt:1:event:2`, sourceSequence: 2, diff --git a/packages/providers/src/private/cursor/cursor-canonical-event-publisher.ts b/packages/providers/src/private/cursor/cursor-canonical-event-publisher.ts index ad8fe4aa4..9ccbe4175 100644 --- a/packages/providers/src/private/cursor/cursor-canonical-event-publisher.ts +++ b/packages/providers/src/private/cursor/cursor-canonical-event-publisher.ts @@ -50,6 +50,8 @@ export class CursorCanonicalEventPublisher { threadId: routing.threadId, turnId: routing.turnId, executionId: routing.executionId, + batchId: draft.eventId, + deliveryAttempt: routing.deliveryAttempt, phase: "running", events: [draft], }); From 9d588144cf6c4b25149b104ff67c943ab3e62ac2 Mon Sep 17 00:00:00 2001 From: "(CJ) Chukwudi Nwobodo" <142016413+chuks-qua@users.noreply.github.com> Date: Fri, 25 Sep 2026 10:49:15 +0100 Subject: [PATCH 106/136] fix(server): roll back rejected child projections --- ...anonical-execution-semantic-writer.test.ts | 8 +++-- .../canonical-committed-provider-projector.ts | 8 +++-- .../canonical-execution-semantic-writer.ts | 30 ++++++++++++++++++- 3 files changed, 40 insertions(+), 6 deletions(-) diff --git a/apps/server/src/features/agents/canonical/__tests__/canonical-execution-semantic-writer.test.ts b/apps/server/src/features/agents/canonical/__tests__/canonical-execution-semantic-writer.test.ts index 22c1af269..fcb61d3d8 100644 --- a/apps/server/src/features/agents/canonical/__tests__/canonical-execution-semantic-writer.test.ts +++ b/apps/server/src/features/agents/canonical/__tests__/canonical-execution-semantic-writer.test.ts @@ -3,7 +3,7 @@ import * as NodeFS from "node:fs"; import * as NodeOS from "node:os"; import * as NodePath from "node:path"; import type { Database } from "bun:sqlite"; -import { AgentEventType, type AgentEvent } from "@mcode/contracts"; +import { AgentEventType, type AgentEvent, type ProviderRuntimeExtension } from "@mcode/contracts"; import { afterEach, beforeEach, describe, expect, it } from "vitest"; import { openDatabase } from "../../../../runtime/persistence/sqlite/database.js"; @@ -67,7 +67,7 @@ function event() { }; } -function runtimeEvent(sequence: number, agentEvent: AgentEvent): Extract< +function runtimeEvent(sequence: number, agentEvent: AgentEvent, extension?: ProviderRuntimeExtension): Extract< ExecutionSemanticOperation["mutation"], { kind: "append-events" } >["events"][number] { const itemId = `runtime-${sequence}`; @@ -80,7 +80,9 @@ function runtimeEvent(sequence: number, agentEvent: AgentEvent): Extract< payload: { type: "item.recorded", item: { id: itemId, threadId: THREAD_ID, turnId: TURN_ID, kind: "system", providerIdentities: [], - payload: { projection: "providerRuntimeEvent", runtimeEvent: { event: agentEvent } }, + payload: { projection: "providerRuntimeEvent", runtimeEvent: { + event: agentEvent, ...(extension ? { extension } : {}), + } }, createdAt: NOW, updatedAt: NOW, } }, }; diff --git a/apps/server/src/features/agents/canonical/canonical-committed-provider-projector.ts b/apps/server/src/features/agents/canonical/canonical-committed-provider-projector.ts index 37a7cdae3..eae4d84cc 100644 --- a/apps/server/src/features/agents/canonical/canonical-committed-provider-projector.ts +++ b/apps/server/src/features/agents/canonical/canonical-committed-provider-projector.ts @@ -4,12 +4,16 @@ import { CodexCollaborationEventAdapter } from "../collaboration/adapters/codex- import type { CodexCollaborationDurability } from "../collaboration/codex-collaboration-durability.js"; import type { ProjectedCommittedProviderEvent } from "../execution/execution-worker-handler.js"; import { processProviderEventWorkerTask } from "../../providers/composition/provider-event-worker-protocol.js"; +import type { ProviderEventProjection } from "../../providers/composition/provider-event-adapter.js"; + +/** Runs one Codex projection in a savepoint and retains only its complete effects. */ +export type CommitCodexProjection = (project: () => ProviderEventProjection) => ProviderEventProjection; /** Interprets committed runtime envelopes beside the SQLite connection that owns their effects. */ export class CanonicalCommittedProviderProjector { private readonly codex: CodexCollaborationEventAdapter; - constructor(durability: CodexCollaborationDurability) { + constructor(durability: CodexCollaborationDurability, private readonly commitCodex: CommitCodexProjection) { this.codex = new CodexCollaborationEventAdapter(durability); } @@ -24,7 +28,7 @@ export class CanonicalCommittedProviderProjector { throw new Error(`Committed provider envelope cannot be interpreted: ${envelope.eventId}`); } const interpretation = outcome.event.providerId === "codex" - ? this.codex.project(outcome.event) + ? this.commitCodex(() => this.codex.project(outcome.event)) : { status: "forward" as const, event: outcome.event.event }; if (interpretation.status !== "forward") continue; projected.push({ diff --git a/apps/server/src/features/agents/canonical/canonical-execution-semantic-writer.ts b/apps/server/src/features/agents/canonical/canonical-execution-semantic-writer.ts index 1a5371891..ad622693f 100644 --- a/apps/server/src/features/agents/canonical/canonical-execution-semantic-writer.ts +++ b/apps/server/src/features/agents/canonical/canonical-execution-semantic-writer.ts @@ -29,6 +29,7 @@ import type { import type { CanonicalAgentEventPublisher } from "./canonical-agent-boundary.js"; import { CanonicalAgentBoundary } from "./canonical-agent-boundary.js"; import { CanonicalCommittedProviderProjector } from "./canonical-committed-provider-projector.js"; +import type { ProviderEventProjection } from "../../providers/composition/provider-event-adapter.js"; import { CanonicalCodexSystemErrorProjection, matchesCodexSystemIntents } from "./canonical-codex-system-error-projection.js"; import { CanonicalContextCompactionProjection } from "./canonical-context-compaction-projection.js"; import { CanonicalParentTurnWrite, type DataOnlyParentLiveMessageInput } from "./canonical-parent-turn-write.js"; @@ -52,6 +53,12 @@ const MAX_BUFFERED_PUBLICATION_EVENTS = 2_048; const MAX_LIVE_PUBLICATION_EVENTS = 64; const MAX_LIVE_PUBLICATION_BYTES = 256 * 1024; const MAX_LIVE_RECEIPT_BYTES = 512 * 1024; + +class RejectedCodexProjection extends Error { + constructor(readonly diagnostic: Extract["diagnostic"]) { + super("Codex projection rejected"); + } +} const LIVE_RECOVERY_KINDS: ReadonlySet = new Set([ "append-assistant-text", "narrative-delta", "live-event", ]); @@ -216,7 +223,7 @@ export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter this.bufferPublication(events); }); this.canonical = codexBoundary; - this.providerProjector = new CanonicalCommittedProviderProjector(codexBoundary); + this.providerProjector = new CanonicalCommittedProviderProjector(codexBoundary, (project) => this.commitCodexProjection(project)); this.contextCompaction = new CanonicalContextCompactionProjection(db); this.systemProjection = new CanonicalCodexSystemErrorProjection(db); this.tasks = new TaskRepo(db); @@ -400,6 +407,27 @@ export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter })()); } + private commitCodexProjection(project: () => ProviderEventProjection): ProviderEventProjection { + const pending = this.bufferedPublication; + if (!pending) throw new Error("Codex projection requires a semantic transaction"); + const pendingLength = pending.length; + const pendingCount = this.bufferedPublicationCount; + try { + return this.db.transaction(() => { + const result = project(); + if (result.status === "rejected") throw new RejectedCodexProjection(result.diagnostic); + return result; + })(); + } catch (error) { + pending.length = pendingLength; + this.bufferedPublicationCount = pendingCount; + if (!(error instanceof RejectedCodexProjection)) throw error; + // The adapter's diagnostic was part of the rejected savepoint. + this.canonical.recordCodexChildRoutingDiagnostic(error.diagnostic); + return { status: "rejected", diagnostic: error.diagnostic }; + } + } + private appendAssistantText(operation: ExecutionSemanticOperation, hash: string): ExecutionWriteReceipt { const mutation = operation.mutation; if (mutation.kind !== "append-assistant-text") return conflict(operation); From f7f9be80c57df08ba4a351fae73f78f692872142 Mon Sep 17 00:00:00 2001 From: "(CJ) Chukwudi Nwobodo" <142016413+chuks-qua@users.noreply.github.com> Date: Fri, 25 Sep 2026 10:50:09 +0100 Subject: [PATCH 107/136] feat(server): persist sealed execution file evidence at terminal --- .../canonical-parent-turn-write.test.ts | 51 ++++++++++ .../canonical/canonical-parent-turn-write.ts | 98 +++++++++++++++++-- .../turns/__tests__/turn-file-tracker.test.ts | 55 +++++++++++ .../turns/turn-execution-file-evidence.ts | 94 ++++++++++++++++++ .../agents/turns/turn-file-tracker.ts | 37 ++++++- 5 files changed, 326 insertions(+), 9 deletions(-) create mode 100644 apps/server/src/features/agents/turns/turn-execution-file-evidence.ts diff --git a/apps/server/src/features/agents/canonical/__tests__/canonical-parent-turn-write.test.ts b/apps/server/src/features/agents/canonical/__tests__/canonical-parent-turn-write.test.ts index ebae45a8a..774a61312 100644 --- a/apps/server/src/features/agents/canonical/__tests__/canonical-parent-turn-write.test.ts +++ b/apps/server/src/features/agents/canonical/__tests__/canonical-parent-turn-write.test.ts @@ -11,6 +11,7 @@ import { MessageRepo } from "../../conversation/persistence/message-repo.js"; import { PlanQuestionAnswersRepo } from "../../planning/persistence/plan-question-answers-repo.js"; import { ToolCallRecordRepo } from "../../tools/persistence/tool-call-record-repo.js"; import { deriveTurnAssistantMessageId } from "../../turns/turn-assistant-message-id.js"; +import type { PreparedExecutionFileEvidence } from "../../turns/turn-execution-file-evidence.js"; import { CanonicalParentTurnWrite, type DataOnlyParentTerminalProjectionInput, @@ -301,6 +302,56 @@ describe("CanonicalParentTurnWrite", () => { expect(db.prepare("SELECT count(*) AS count FROM turn_diff_snapshots").get()).toEqual({ count: 1 }); }); + it("commits exact-execution file evidence with the terminal assistant or rolls it all back", async () => { + writer.start(startInput()); + const messageId = deriveTurnAssistantMessageId(THREAD_ID, "user-1"); + const input = terminalProjectionInput(); + input.assistant.messageId = messageId; + const staged = writer.stageTerminalProjection(input); + const patch = "diff --git a/a.txt b/a.txt\nindex 1..2\n--- a/a.txt\n+++ b/a.txt\n@@ -1 +1 @@\n-old\n+new\n"; + const fileEffects = { + revision: 1, fileCount: 1, additions: 1, deletions: 1, + effects: [{ path: "a.txt", kind: "edited" as const, scope: "workspace" as const, + additions: 1, deletions: 1, binary: false, toolCallIds: ["tool-1"] }], + }; + const fileEvidence: PreparedExecutionFileEvidence = { + threadId: THREAD_ID, turnId: TURN_ID, executionId: EXECUTION_ID, deliveryAttempt: 2, + fileEffects, filesChanged: ["a.txt"], + snapshot: { + threadId: THREAD_ID, executionId: EXECUTION_ID, refBefore: "a".repeat(40), + refAfter: "b".repeat(40), filesChanged: ["a.txt"], fileEffects, worktreePath: null, + }, + selectedTurnDiff: { thread_id: THREAD_ID, source: "native", patch, revision: 2 }, + }; + const finish: DataOnlyParentTurnFinishInput = { + ...finishInput(new MessageRepo(db).findByIdInThreadIncludingInternal(THREAD_ID, messageId)!), + projection: { kind: "writer-staged", messageId }, deliveryAttempt: 2, fileEvidence, + }; + expect(() => writer.finish({ ...finish, fileEvidence: { ...fileEvidence, executionId: "old-execution" } })) + .toThrow("Invalid execution file evidence"); + expect(() => writer.finish({ ...finish, deliveryAttempt: 1 })).toThrow("Invalid execution file evidence"); + db.run("CREATE TRIGGER fail_snapshot BEFORE INSERT ON turn_snapshots BEGIN SELECT RAISE(ABORT, 'snapshot unavailable'); END"); + await expect(writer.finish(structuredClone(finish))).rejects.toThrow("snapshot unavailable"); + expect(db.prepare("SELECT terminal_outcome FROM canonical_agent_ingest_checkpoints WHERE execution_id = ?").get(EXECUTION_ID)) + .toEqual({ terminal_outcome: null }); + expect(db.prepare("SELECT is_internal FROM messages WHERE id = ?").get(messageId)).toEqual({ is_internal: 1 }); + expect(db.prepare("SELECT COUNT(*) AS count FROM turn_diff_snapshots").get()).toEqual({ count: 0 }); + db.run("DROP TRIGGER fail_snapshot"); + expect((await writer.finish(finish)).outcome).toBe("committed"); + expect(db.prepare("SELECT message_id, thread_id, files_changed, ref_before, ref_after FROM turn_snapshots WHERE message_id = ?") + .get(messageId)).toEqual({ + message_id: messageId, thread_id: THREAD_ID, files_changed: '["a.txt"]', + ref_before: "a".repeat(40), ref_after: "b".repeat(40), + }); + expect(db.prepare("SELECT source, patch FROM turn_diff_snapshots WHERE message_id = ?").get(messageId)) + .toEqual({ source: "native", patch }); + expect(db.prepare("SELECT has_file_changes FROM threads WHERE id = ?").get(THREAD_ID)) + .toEqual({ has_file_changes: 1 }); + expect((await writer.finish({ ...finish, projection: staged.projection })).outcome).toBe("terminal-outcome-confirmed"); + expect(db.prepare("SELECT COUNT(*) AS count FROM turn_snapshots").get()).toEqual({ count: 1 }); + expect(db.prepare("SELECT COUNT(*) AS count FROM turn_diff_snapshots").get()).toEqual({ count: 1 }); + }); + it("refuses to reuse a public assistant row as a staged turn projection", () => { writer.start(startInput()); const input = terminalProjectionInput(); diff --git a/apps/server/src/features/agents/canonical/canonical-parent-turn-write.ts b/apps/server/src/features/agents/canonical/canonical-parent-turn-write.ts index a938865e9..0dc3b021b 100644 --- a/apps/server/src/features/agents/canonical/canonical-parent-turn-write.ts +++ b/apps/server/src/features/agents/canonical/canonical-parent-turn-write.ts @@ -2,6 +2,7 @@ import type { Database } from "bun:sqlite"; import * as NodeCrypto from "node:crypto"; import { ParentNarrativeRecoveryItemSchema, + TurnFileEffectSummarySchema, type ParentNarrativeRecoveryItem, type StoredAttachment, type TurnOutcome, @@ -19,6 +20,8 @@ import type { ExecutionIdentity } from "../execution/execution-mailbox-protocol. import { ParentAssistantTextCheckpointService } from "../turns/parent-assistant-text-checkpoint-service.js"; import { TURN_DIFF_MAX_BYTES, parseTurnDiff } from "../turns/turn-diff-patch.js"; import { TurnDiffRepo } from "../turns/persistence/turn-diff-repo.js"; +import { TurnSnapshotRepo } from "../turns/persistence/turn-snapshot-repo.js"; +import type { PreparedExecutionFileEvidence } from "../turns/turn-execution-file-evidence.js"; import type { SelectedTurnDiff } from "../turns/turn-diff-service.js"; import type { ParentTurnFinishInput, @@ -98,7 +101,9 @@ export interface DataOnlyParentEventInput extends Omit { projection: ParentTurnProjection | { readonly kind: "writer-staged"; readonly messageId?: string }; + deliveryAttempt?: number; selectedTurnDiff?: SelectedTurnDiff; + fileEvidence?: PreparedExecutionFileEvidence; } /** Cloneable terminal data whose compatibility rows are staged on the writer connection. */ @@ -148,6 +153,8 @@ export class CanonicalParentTurnWrite { private readonly stagedAssistant: ReturnType; private readonly assistantTextCheckpoints: ParentAssistantTextCheckpointService; private readonly turnDiffs: TurnDiffRepo; + private readonly turnSnapshots: TurnSnapshotRepo; + private readonly markThreadFilesChanged: ReturnType; constructor(db: Database, publish: CanonicalAgentEventPublisher) { this.db = db; @@ -165,6 +172,8 @@ export class CanonicalParentTurnWrite { this.stagedAssistant = db.prepare("SELECT role, content FROM messages WHERE id = ? AND thread_id = ?"); this.assistantTextCheckpoints = new ParentAssistantTextCheckpointService(db); this.turnDiffs = new TurnDiffRepo(db); + this.turnSnapshots = new TurnSnapshotRepo(db); + this.markThreadFilesChanged = db.prepare("UPDATE threads SET has_file_changes = 1 WHERE id = ? AND has_file_changes = 0"); } /** Import fsynced text before the interruption transaction reads its durable prefix. */ @@ -401,10 +410,8 @@ export class CanonicalParentTurnWrite { ? this.loadStagedTerminalProjection(input.threadId, input.executionId, input.projection.messageId) : input.projection; this.assertStagedAssistant(input.threadId, staged); - if (input.selectedTurnDiff && (input.outcome !== "completed" || !staged.message - || !validSelectedTurnDiff(input.selectedTurnDiff, input.threadId))) { - throw new Error("Invalid selected turn diff for terminal assistant"); - } + const fileEvidence = input.fileEvidence; + const selectedTurnDiff = selectTerminalDiff(input, staged.message !== null); const projection: ParentTurnProjection = { message: staged.message ? { @@ -423,8 +430,23 @@ export class CanonicalParentTurnWrite { if (!projection.message) return; this.messages.setAssistantOutcome(projection.message.id, input.outcome, input.executionId); this.messages.publishAssistant(projection.message.id); - if (input.selectedTurnDiff) this.turnDiffs.create({ - id: NodeCrypto.randomUUID(), message_id: projection.message.id, ...input.selectedTurnDiff, + if (fileEvidence?.snapshot) { + const snapshot = fileEvidence.snapshot; + this.turnSnapshots.create({ + messageId: projection.message.id, + threadId: snapshot.threadId, + refBefore: snapshot.refBefore, + refAfter: snapshot.refAfter, + filesChanged: [...snapshot.filesChanged], + fileEffects: snapshot.fileEffects, + worktreePath: null, + }); + if (fileEvidence.fileEffects.fileCount > 0 || fileEvidence.filesChanged.length > 0) { + this.markThreadFilesChanged.run(input.threadId); + } + } + if (selectedTurnDiff) this.turnDiffs.create({ + id: NodeCrypto.randomUUID(), message_id: projection.message.id, ...selectedTurnDiff, }); }, }, onBatchWrite); @@ -454,6 +476,70 @@ export class CanonicalParentTurnWrite { } } +function selectTerminalDiff(input: DataOnlyParentTurnFinishInput, hasAssistant: boolean): SelectedTurnDiff | null | undefined { + if (input.fileEvidence) assertExecutionFileEvidence(input, hasAssistant); + const selected = input.fileEvidence?.selectedTurnDiff ?? input.selectedTurnDiff; + if (selected && (input.outcome !== "completed" || !hasAssistant + || !validSelectedTurnDiff(selected, input.threadId))) { + throw new Error("Invalid selected turn diff for terminal assistant"); + } + return selected; +} + +function assertExecutionFileEvidence(input: DataOnlyParentTurnFinishInput, hasAssistant: boolean): void { + const evidence = input.fileEvidence; + if (!evidence || !hasAssistant || input.selectedTurnDiff + || !validExecutionFileEvidence(evidence, input.threadId, input.turnId, + input.executionId, input.deliveryAttempt)) { + throw new Error("Invalid execution file evidence for terminal assistant"); + } +} + +function validExecutionFileEvidence( + evidence: PreparedExecutionFileEvidence, + threadId: string, + turnId: string, + executionId: string, + deliveryAttempt: number | undefined, +): boolean { + if (!sameTerminalFileOwner(evidence, threadId, turnId, executionId, deliveryAttempt) + || !TurnFileEffectSummarySchema().safeParse(evidence.fileEffects).success + || evidence.fileEffects.fileCount !== evidence.fileEffects.effects.length) return false; + const paths = evidence.fileEffects.effects.filter((effect) => effect.scope === "workspace") + .map((effect) => effect.path); + if (JSON.stringify(evidence.filesChanged) !== JSON.stringify(paths)) return false; + const snapshot = evidence.snapshot; + if (!snapshot) return evidence.fileEffects.fileCount === 0 && paths.length === 0; + return validExecutionSnapshot(snapshot, evidence, paths); +} + +function sameTerminalFileOwner( + evidence: PreparedExecutionFileEvidence, + threadId: string, + turnId: string, + executionId: string, + deliveryAttempt: number | undefined, +): boolean { + return Number.isSafeInteger(deliveryAttempt) && deliveryAttempt !== undefined && deliveryAttempt >= 1 + && evidence.threadId === threadId && evidence.turnId === turnId + && evidence.executionId === executionId && evidence.deliveryAttempt === deliveryAttempt; +} + +function validExecutionSnapshot( + snapshot: NonNullable, + evidence: PreparedExecutionFileEvidence, + paths: readonly string[], +): boolean { + return snapshot.threadId === evidence.threadId && snapshot.executionId === evidence.executionId + && snapshot.worktreePath === null && isSnapshotRef(snapshot.refBefore) && isSnapshotRef(snapshot.refAfter) + && JSON.stringify(snapshot.filesChanged) === JSON.stringify(paths) + && JSON.stringify(snapshot.fileEffects) === JSON.stringify(evidence.fileEffects); +} + +function isSnapshotRef(ref: string): boolean { + return ref === "" || /^[0-9a-f]{40}$|^[0-9a-f]{64}$/.test(ref); +} + function validSelectedTurnDiff(selected: SelectedTurnDiff, threadId: string): boolean { if (selected.thread_id !== threadId || !Number.isSafeInteger(selected.revision) || selected.revision < 0) return false; if (selected.source === "git") return selected.patch === null; diff --git a/apps/server/src/features/agents/turns/__tests__/turn-file-tracker.test.ts b/apps/server/src/features/agents/turns/__tests__/turn-file-tracker.test.ts index 025fa80b0..ebaf6c62a 100644 --- a/apps/server/src/features/agents/turns/__tests__/turn-file-tracker.test.ts +++ b/apps/server/src/features/agents/turns/__tests__/turn-file-tracker.test.ts @@ -1,10 +1,14 @@ +import "reflect-metadata"; import { afterEach, describe, expect, it } from "vitest"; import * as NodeFSPromises from "node:fs/promises"; import * as NodePath from "node:path"; import * as NodeOS from "node:os"; import { AgentEventType, type TurnFileEffectSummary } from "@mcode/contracts"; import { TurnFileTracker } from "../turn-file-tracker.js"; +import { prepareExecutionFileEvidence } from "../turn-execution-file-evidence.js"; import { parseTurnDiff } from "../turn-diff-patch.js"; +import { SnapshotService } from "../../../projects/diffs/snapshots/snapshot-service.js"; +import { RealGitExecutor } from "../../../projects/git/execution/real-git-executor.js"; import { createCursorAcpTurnState, mapCursorAcpSessionNotification, @@ -132,6 +136,57 @@ describe("TurnFileTracker", () => { expect(await worker.finalizeTurn("t")).toMatchObject({ fileCount: 1, additions: 2, deletions: 1 }); }); + it("seals only the exact execution generation before a resumed turn reuses a tool ID", async () => { + const root = await tempDir("mcode-sealed-file-generation-"); + const path = NodePath.join(root, "tracked.txt"); + await NodeFSPromises.writeFile(path, "before\n"); + const tracker = trackerWithBaseline({}, []); + const first = tracker.beginExecutionTurn({ + threadId: "t", executionId: "execution-1", cwd: root, baselineRef: null, + }); + const event = { threadId: "t", toolCallId: "edit", toolName: "Edit", toolInput: { file_path: "tracked.txt" } }; + const capture = tracker.captureToolUseObservation(event.threadId, event.toolCallId, event.toolName, event.toolInput); + if (!capture) throw new Error("Expected first execution capture"); + await NodeFSPromises.writeFile(path, "after\n"); + expect(await tracker.observeCapturedToolUse(event, capture)).toBe(true); + await tracker.observeToolResult("t", "edit"); + const frozen = await tracker.finalEvidenceForExecution(first); + expect(frozen?.summary).toMatchObject({ fileCount: 1 }); + expect(await tracker.finalEvidenceForExecution(first)).toEqual(frozen); + expect(tracker.captureToolUseObservation(event.threadId, event.toolCallId, event.toolName, event.toolInput)).toBeNull(); + expect(await tracker.observeCapturedToolUse(event, capture)).toBe(false); + + const second = tracker.beginExecutionTurn({ + threadId: "t", executionId: "execution-2", cwd: root, baselineRef: null, + }); + expect(await tracker.finalEvidenceForExecution({ ...first, generationToken: second.generationToken })).toBeNull(); + expect((await tracker.finalEvidenceForExecution(second))?.summary.fileCount).toBe(0); + tracker.clearTurn("t", first.generation); + expect(await tracker.finalEvidenceForExecution(first)).toBeNull(); + }); + + it("rejects native diff evidence from another delivery attempt before terminal preparation", async () => { + const root = await tempDir("mcode-file-diff-attempt-"); + const tracker = trackerWithBaseline({}, []); + const handoff = tracker.beginExecutionTurn({ + threadId: "t", executionId: "execution-1", cwd: root, baselineRef: null, + }); + const input = { + handoff, turnId: "turn-1", deliveryAttempt: 2, outcome: "completed" as const, + tracker, snapshots: new SnapshotService(new RealGitExecutor()), + nativeDiff: { + threadId: "t", turnId: "turn-1", turnExecutionId: "execution-1", deliveryAttempt: 1, + revision: 1, evidence: null, rejected: false, + }, + }; + await expect(prepareExecutionFileEvidence(input)).rejects.toThrow("Native diff does not belong"); + const prepared = await prepareExecutionFileEvidence({ ...input, nativeDiff: null }); + expect(prepared).toMatchObject({ + threadId: "t", executionId: "execution-1", filesChanged: [], snapshot: null, selectedTurnDiff: null, + }); + expect(() => structuredClone(prepared)).not.toThrow(); + }); + it("rejects a handoff or capture for another execution or root", async () => { const root = await tempDir("mcode-transferred-file-stale-"); const otherRoot = await tempDir("mcode-transferred-file-other-root-"); diff --git a/apps/server/src/features/agents/turns/turn-execution-file-evidence.ts b/apps/server/src/features/agents/turns/turn-execution-file-evidence.ts new file mode 100644 index 000000000..5cadf5f0f --- /dev/null +++ b/apps/server/src/features/agents/turns/turn-execution-file-evidence.ts @@ -0,0 +1,94 @@ +import type { TurnFileEffectSummary, TurnOutcome } from "@mcode/contracts"; + +import type { SnapshotService } from "../../projects/diffs/snapshots/snapshot-service.js"; +import type { CreateTurnSnapshotInput } from "./persistence/turn-snapshot-repo.js"; +import type { FileTurnHandoff, TurnFileTracker } from "./turn-file-tracker.js"; +import { + selectTurnDiffSettlement, + type PreparedTurnDiffEvidence, + type SelectedTurnDiff, +} from "./turn-diff-service.js"; + +/** Cloneable snapshot data whose message ID is assigned by the terminal writer. */ +export interface ExecutionTurnSnapshot extends Omit { + readonly executionId: string; +} + +/** File data retained through a failed writer attempt and retired only after its receipt. */ +export interface PreparedExecutionFileEvidence { + readonly threadId: string; + readonly turnId: string; + readonly executionId: string; + readonly deliveryAttempt: number; + readonly fileEffects: TurnFileEffectSummary; + readonly filesChanged: readonly string[]; + readonly snapshot: ExecutionTurnSnapshot | null; + readonly selectedTurnDiff: SelectedTurnDiff | null; +} + +/** Settle one sealed file generation before sending terminal data to the sole writer. */ +export async function prepareExecutionFileEvidence(input: { + readonly handoff: FileTurnHandoff; + readonly turnId: string; + readonly deliveryAttempt: number; + readonly outcome: TurnOutcome; + readonly nativeDiff: PreparedTurnDiffEvidence | null; + readonly tracker: TurnFileTracker; + readonly snapshots: SnapshotService; +}): Promise { + const { handoff, tracker, snapshots } = input; + const evidence = await tracker.finalEvidenceForExecution(handoff); + if (!evidence) throw new Error(`File evidence generation no longer belongs to execution ${handoff.executionId}`); + assertNativeDiffOwner(input); + + const fileEffects = evidence.summary; + const { snapshot, filesChanged } = await prepareSnapshot(handoff, evidence.baselineRef, fileEffects, snapshots); + return { + threadId: handoff.threadId, + turnId: input.turnId, + executionId: handoff.executionId, + deliveryAttempt: input.deliveryAttempt, + fileEffects, + filesChanged, + snapshot, + selectedTurnDiff: input.nativeDiff + ? selectTurnDiffSettlement(input.nativeDiff, input.outcome, fileEffects, evidence.reconstructionPatch) + : null, + }; +} + +function assertNativeDiffOwner(input: { + readonly nativeDiff: PreparedTurnDiffEvidence | null; + readonly handoff: FileTurnHandoff; + readonly turnId: string; + readonly deliveryAttempt: number; +}): void { + const diff = input.nativeDiff; + if (!diff) return; + if (diff.threadId !== input.handoff.threadId || diff.turnId !== input.turnId + || diff.turnExecutionId !== input.handoff.executionId + || diff.deliveryAttempt !== input.deliveryAttempt) { + throw new Error(`Native diff does not belong to execution ${input.handoff.executionId}`); + } +} + +async function prepareSnapshot( + handoff: FileTurnHandoff, + refBefore: string | null, + fileEffects: TurnFileEffectSummary, + snapshots: SnapshotService, +): Promise<{ snapshot: ExecutionTurnSnapshot | null; filesChanged: string[] }> { + const refAfter = refBefore ? await snapshots.captureRef(handoff.cwd) : null; + const filesChanged = fileEffects.effects.filter((effect) => effect.scope === "workspace") + .map((effect) => effect.path); + if (fileEffects.fileCount === 0 && !(refBefore && refAfter)) return { snapshot: null, filesChanged }; + return { filesChanged, snapshot: { + threadId: handoff.threadId, + executionId: handoff.executionId, + refBefore: refBefore ?? "", + refAfter: refAfter ?? "", + filesChanged, + fileEffects, + worktreePath: null, + } }; +} diff --git a/apps/server/src/features/agents/turns/turn-file-tracker.ts b/apps/server/src/features/agents/turns/turn-file-tracker.ts index dcfdd25d8..5b70e6bb2 100644 --- a/apps/server/src/features/agents/turns/turn-file-tracker.ts +++ b/apps/server/src/features/agents/turns/turn-file-tracker.ts @@ -84,6 +84,7 @@ interface TrackedPath { } interface TurnState { + sealed: boolean; reconstructionRejected?: boolean; executionId: string | null; generation: number; @@ -129,6 +130,13 @@ const fileTurnHandoffSchema = z.object({ export type FileTurnHandoff = Readonly>; type FileTurnStart = Omit & { readonly executionId: string | null }; +/** Frozen file evidence for one execution after its queued observations settle. */ +export interface ExecutionFileEvidence { + readonly baselineRef: string | null; + readonly summary: TurnFileEffectSummary; + readonly reconstructionPatch?: string; +} + /** Identity supplied by the scheduler, independently of the handoff payload. */ export interface ExpectedFileExecution { readonly threadId: string; @@ -246,7 +254,7 @@ export class TurnFileTracker { const generation = this.currentGeneration.get(threadId); if (generation === undefined) return null; const turn = this.getTurn(threadId, generation); - if (!turn) return null; + if (!turn || turn.sealed) return null; const candidates = boundedMutationCandidates(toolName, toolInput); if (candidates.length === 0) return null; const observations = this.synchronousObservations(turn, candidates).map(freezeCandidateObservation); @@ -266,7 +274,7 @@ export class TurnFileTracker { ): Promise { const { threadId, toolCallId, toolName, toolInput } = event; const turn = this.getTurn(threadId); - if (!turn || !capturedMatchesTurn(captured, turn, threadId, toolCallId)) return Promise.resolve(false); + if (!turn || turn.sealed || !capturedMatchesTurn(captured, turn, threadId, toolCallId)) return Promise.resolve(false); const generation = turn.generation; const boundedCandidates = boundedMutationCandidates(toolName, toolInput); if (boundedCandidates.length === 0 || captured.observations.length !== boundedCandidates.length @@ -316,11 +324,33 @@ export class TurnFileTracker { return turn.summary; } + /** Read an execution's settled evidence only while its original handoff still owns the generation. */ + async finalEvidenceForExecution(handoff: FileTurnHandoff): Promise { + const parsed = fileTurnHandoffSchema.safeParse(handoff); + if (!parsed.success) return null; + const turn = this.getTurn(parsed.data.threadId, parsed.data.generation); + if (!turn || !sameFileTurnHandoff(turn, parsed.data)) return null; + turn.sealed = true; + await turn.chain; + if (this.getTurn(parsed.data.threadId, parsed.data.generation) !== turn + || !sameFileTurnHandoff(turn, parsed.data)) return null; + return { + baselineRef: turn.baselineRef, + summary: structuredClone(turn.summary), + reconstructionPatch: this.reconstructionPatchForTurn(turn), + }; + } + /** Reconstruct a bounded patch only from complete explicit file-tool evidence. */ async reconstructionPatch(threadId: string, generation?: number): Promise { const turn = this.getTurn(threadId, generation); if (!turn || turn.reconstructionRejected) return undefined; await turn.chain; + return this.reconstructionPatchForTurn(turn); + } + + private reconstructionPatchForTurn(turn: TurnState): string | undefined { + if (turn.reconstructionRejected) return undefined; const tracked = [...turn.tracked.values()].filter((entry) => entry.scope === "workspace" && entry.effectCandidate); if (tracked.length !== turn.summary.fileCount || tracked.some((entry) => entry.evidenceRejected || entry.evidenceBefore === undefined || entry.evidenceAfter === undefined || entry.oldPath !== undefined)) return undefined; const patches = tracked.map((entry) => createTextPatch(entry.displayPath.replaceAll("\\", "/"), entry.evidenceBefore!, entry.evidenceAfter!, entry.effectCandidate!.effect.kind === "added" ? "added" : entry.effectCandidate!.effect.kind === "removed" ? "removed" : "edited")); @@ -346,7 +376,7 @@ export class TurnFileTracker { generation?: number, ): Promise { const turn = this.getTurn(threadId, generation); - if (!turn) return Promise.resolve(); + if (!turn || turn.sealed) return Promise.resolve(); const operation = turn.chain.then(() => work(turn)); turn.chain = operation.catch(() => undefined); return operation; @@ -784,6 +814,7 @@ function boundedMutationCandidates(toolName: string, toolInput: Record Date: Fri, 25 Sep 2026 10:52:43 +0100 Subject: [PATCH 108/136] feat(server): assign durable per-thread live publication IDs --- apps/server/drizzle/0066_fat_amazoness.sql | 5 + apps/server/drizzle/meta/0066_snapshot.json | 5027 +++++++++++++++++ apps/server/drizzle/meta/_journal.json | 7 + .../transport/__tests__/push.test.ts | 16 + .../canonical-agent-writer-client.test.ts | 4 +- ...anonical-execution-semantic-writer.test.ts | 61 +- .../canonical-execution-writer-port.test.ts | 55 +- .../canonical-execution-semantic-writer.ts | 95 +- .../execution-live-publication-release.ts | 44 +- .../provider-event-publication.test.ts | 19 + .../events/provider-event-publication.ts | 3 + .../execution-worker-loss-coordinator.test.ts | 10 +- .../start-agent-orchestration.test.ts | 17 + .../agent-event-publication-service.ts | 2 + .../src/runtime/persistence/sqlite/schema.ts | 10 + .../__tests__/agent-event-branches.test.ts | 70 + .../stable-agent-event-publications.test.ts | 73 + .../thread-store/agent-event-preflight.ts | 2 + .../stable-agent-event-publications.ts | 44 + apps/web/src/stores/threadStore.ts | 2 + .../src/events/__tests__/agent-event.test.ts | 10 + packages/contracts/src/events/agent-event.ts | 2 + 22 files changed, 5515 insertions(+), 63 deletions(-) create mode 100644 apps/server/drizzle/0066_fat_amazoness.sql create mode 100644 apps/server/drizzle/meta/0066_snapshot.json create mode 100644 apps/web/src/stores/thread-store/__tests__/stable-agent-event-publications.test.ts create mode 100644 apps/web/src/stores/thread-store/stable-agent-event-publications.ts diff --git a/apps/server/drizzle/0066_fat_amazoness.sql b/apps/server/drizzle/0066_fat_amazoness.sql new file mode 100644 index 000000000..c22cb0107 --- /dev/null +++ b/apps/server/drizzle/0066_fat_amazoness.sql @@ -0,0 +1,5 @@ +CREATE TABLE `canonical_writer_live_publication_heads` ( + `thread_id` text PRIMARY KEY NOT NULL, + `last_sequence` integer NOT NULL, + FOREIGN KEY (`thread_id`) REFERENCES `threads`(`id`) ON UPDATE no action ON DELETE cascade +); diff --git a/apps/server/drizzle/meta/0066_snapshot.json b/apps/server/drizzle/meta/0066_snapshot.json new file mode 100644 index 000000000..192a5b933 --- /dev/null +++ b/apps/server/drizzle/meta/0066_snapshot.json @@ -0,0 +1,5027 @@ +{ + "version": "6", + "dialect": "sqlite", + "id": "171945ab-d900-46c6-9acd-e89704d225f9", + "prevId": "d6ce0589-8bee-4f7f-b188-033550ae9cf0", + "tables": { + "canonical_agent_events": { + "name": "canonical_agent_events", + "columns": { + "event_id": { + "name": "event_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "thread_id": { + "name": "thread_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "turn_id": { + "name": "turn_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "execution_id": { + "name": "execution_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "accepted_sequence": { + "name": "accepted_sequence", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "durable_revision": { + "name": "durable_revision", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "roster_revision": { + "name": "roster_revision", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "envelope_json": { + "name": "envelope_json", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "accepted_at": { + "name": "accepted_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "persisted_at": { + "name": "persisted_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_canonical_agent_events_execution_sequence": { + "name": "idx_canonical_agent_events_execution_sequence", + "columns": [ + "execution_id", + "accepted_sequence" + ], + "isUnique": true + }, + "idx_canonical_agent_events_thread_revision": { + "name": "idx_canonical_agent_events_thread_revision", + "columns": [ + "thread_id", + "durable_revision" + ], + "isUnique": false + } + }, + "foreignKeys": { + "canonical_agent_events_thread_id_canonical_agent_threads_id_fk": { + "name": "canonical_agent_events_thread_id_canonical_agent_threads_id_fk", + "tableFrom": "canonical_agent_events", + "tableTo": "canonical_agent_threads", + "columnsFrom": [ + "thread_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "canonical_agent_ingest_checkpoints": { + "name": "canonical_agent_ingest_checkpoints", + "columns": { + "execution_id": { + "name": "execution_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "thread_id": { + "name": "thread_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "turn_id": { + "name": "turn_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "last_accepted_sequence": { + "name": "last_accepted_sequence", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "last_durable_sequence": { + "name": "last_durable_sequence", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "native_cursor_json": { + "name": "native_cursor_json", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "phase": { + "name": "phase", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "terminal_outcome": { + "name": "terminal_outcome", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "recovery_incident_id": { + "name": "recovery_incident_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_canonical_agent_checkpoints_thread": { + "name": "idx_canonical_agent_checkpoints_thread", + "columns": [ + "thread_id", + "updated_at" + ], + "isUnique": false + }, + "idx_canonical_agent_checkpoints_recovery_incident": { + "name": "idx_canonical_agent_checkpoints_recovery_incident", + "columns": [ + "recovery_incident_id", + "updated_at" + ], + "isUnique": false + } + }, + "foreignKeys": { + "canonical_agent_ingest_checkpoints_thread_id_canonical_agent_threads_id_fk": { + "name": "canonical_agent_ingest_checkpoints_thread_id_canonical_agent_threads_id_fk", + "tableFrom": "canonical_agent_ingest_checkpoints", + "tableTo": "canonical_agent_threads", + "columnsFrom": [ + "thread_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "canonical_agent_ingest_checkpoints_turn_id_canonical_agent_turns_id_fk": { + "name": "canonical_agent_ingest_checkpoints_turn_id_canonical_agent_turns_id_fk", + "tableFrom": "canonical_agent_ingest_checkpoints", + "tableTo": "canonical_agent_turns", + "columnsFrom": [ + "turn_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "canonical_agent_items": { + "name": "canonical_agent_items", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "thread_id": { + "name": "thread_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "turn_id": { + "name": "turn_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "parent_item_id": { + "name": "parent_item_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "provider_identities_json": { + "name": "provider_identities_json", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'[]'" + }, + "payload_json": { + "name": "payload_json", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_canonical_agent_items_turn": { + "name": "idx_canonical_agent_items_turn", + "columns": [ + "turn_id", + "created_at" + ], + "isUnique": false + }, + "idx_canonical_agent_items_thread": { + "name": "idx_canonical_agent_items_thread", + "columns": [ + "thread_id", + "created_at" + ], + "isUnique": false + }, + "idx_canonical_agent_items_created_id": { + "name": "idx_canonical_agent_items_created_id", + "columns": [ + "created_at", + "id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "canonical_agent_items_thread_id_canonical_agent_threads_id_fk": { + "name": "canonical_agent_items_thread_id_canonical_agent_threads_id_fk", + "tableFrom": "canonical_agent_items", + "tableTo": "canonical_agent_threads", + "columnsFrom": [ + "thread_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "canonical_agent_items_turn_id_canonical_agent_turns_id_fk": { + "name": "canonical_agent_items_turn_id_canonical_agent_turns_id_fk", + "tableFrom": "canonical_agent_items", + "tableTo": "canonical_agent_turns", + "columnsFrom": [ + "turn_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "canonical_agent_threads": { + "name": "canonical_agent_threads", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "parent_thread_id": { + "name": "parent_thread_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "root_thread_id": { + "name": "root_thread_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "owning_parent_thread_id": { + "name": "owning_parent_thread_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "provider_identities_json": { + "name": "provider_identities_json", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'[]'" + }, + "activity_state": { + "name": "activity_state", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "conversation_revision": { + "name": "conversation_revision", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "roster_revision": { + "name": "roster_revision", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_canonical_agent_threads_workspace": { + "name": "idx_canonical_agent_threads_workspace", + "columns": [ + "workspace_id" + ], + "isUnique": false + }, + "idx_canonical_agent_threads_root": { + "name": "idx_canonical_agent_threads_root", + "columns": [ + "root_thread_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "canonical_agent_threads_id_threads_id_fk": { + "name": "canonical_agent_threads_id_threads_id_fk", + "tableFrom": "canonical_agent_threads", + "tableTo": "threads", + "columnsFrom": [ + "id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "canonical_agent_threads_workspace_id_workspaces_id_fk": { + "name": "canonical_agent_threads_workspace_id_workspaces_id_fk", + "tableFrom": "canonical_agent_threads", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "canonical_agent_turns": { + "name": "canonical_agent_turns", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "thread_id": { + "name": "thread_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "execution_id": { + "name": "execution_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "trigger_json": { + "name": "trigger_json", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "permission_mode": { + "name": "permission_mode", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "approval_review_mode": { + "name": "approval_review_mode", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'manual'" + }, + "approval_review_reason": { + "name": "approval_review_reason", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'manual-requested'" + }, + "provider_identities_json": { + "name": "provider_identities_json", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'[]'" + }, + "started_at": { + "name": "started_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "ended_at": { + "name": "ended_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_canonical_agent_turns_execution": { + "name": "idx_canonical_agent_turns_execution", + "columns": [ + "execution_id" + ], + "isUnique": true + }, + "idx_canonical_agent_turns_thread": { + "name": "idx_canonical_agent_turns_thread", + "columns": [ + "thread_id", + "created_at" + ], + "isUnique": false + } + }, + "foreignKeys": { + "canonical_agent_turns_thread_id_canonical_agent_threads_id_fk": { + "name": "canonical_agent_turns_thread_id_canonical_agent_threads_id_fk", + "tableFrom": "canonical_agent_turns", + "tableTo": "canonical_agent_threads", + "columnsFrom": [ + "thread_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "canonical_collaboration_actions": { + "name": "canonical_collaboration_actions", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source_thread_id": { + "name": "source_thread_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source_turn_id": { + "name": "source_turn_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source_item_id": { + "name": "source_item_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "target_thread_id": { + "name": "target_thread_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "target_turn_id": { + "name": "target_turn_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "delivery_unknown": { + "name": "delivery_unknown", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "message": { + "name": "message", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "provider_identities_json": { + "name": "provider_identities_json", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'[]'" + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_canonical_collaboration_source": { + "name": "idx_canonical_collaboration_source", + "columns": [ + "source_thread_id", + "created_at" + ], + "isUnique": false + }, + "idx_canonical_collaboration_target": { + "name": "idx_canonical_collaboration_target", + "columns": [ + "target_thread_id", + "created_at" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "canonical_conversation_display_mappings": { + "name": "canonical_conversation_display_mappings", + "columns": { + "source_item_id": { + "name": "source_item_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "target_kind": { + "name": "target_kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "target_id": { + "name": "target_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source_updated_at": { + "name": "source_updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + } + }, + "indexes": { + "idx_canonical_conversation_display_mappings_target": { + "name": "idx_canonical_conversation_display_mappings_target", + "columns": [ + "target_kind", + "target_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "canonical_conversation_display_mappings_source_item_id_canonical_agent_items_id_fk": { + "name": "canonical_conversation_display_mappings_source_item_id_canonical_agent_items_id_fk", + "tableFrom": "canonical_conversation_display_mappings", + "tableTo": "canonical_agent_items", + "columnsFrom": [ + "source_item_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "canonical_legacy_message_provenance": { + "name": "canonical_legacy_message_provenance", + "columns": { + "message_id": { + "name": "message_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "migration_version": { + "name": "migration_version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "mapping_status": { + "name": "mapping_status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "canonical_thread_id": { + "name": "canonical_thread_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "canonical_turn_id": { + "name": "canonical_turn_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "canonical_item_id": { + "name": "canonical_item_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "reason": { + "name": "reason", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + } + }, + "indexes": { + "idx_canonical_legacy_message_mapping": { + "name": "idx_canonical_legacy_message_mapping", + "columns": [ + "mapping_status", + "message_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "canonical_legacy_message_provenance_message_id_messages_id_fk": { + "name": "canonical_legacy_message_provenance_message_id_messages_id_fk", + "tableFrom": "canonical_legacy_message_provenance", + "tableTo": "messages", + "columnsFrom": [ + "message_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "canonical_legacy_migration_checkpoints": { + "name": "canonical_legacy_migration_checkpoints", + "columns": { + "version": { + "name": "version", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'pending'" + }, + "migrated_messages": { + "name": "migrated_messages", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "ambiguous_messages": { + "name": "ambiguous_messages", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + }, + "completed_at": { + "name": "completed_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "canonical_writer_live_publication_heads": { + "name": "canonical_writer_live_publication_heads", + "columns": { + "thread_id": { + "name": "thread_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "last_sequence": { + "name": "last_sequence", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": { + "canonical_writer_live_publication_heads_thread_id_threads_id_fk": { + "name": "canonical_writer_live_publication_heads_thread_id_threads_id_fk", + "tableFrom": "canonical_writer_live_publication_heads", + "tableTo": "threads", + "columnsFrom": [ + "thread_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "canonical_writer_operation_receipts": { + "name": "canonical_writer_operation_receipts", + "columns": { + "execution_id": { + "name": "execution_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "operation_id": { + "name": "operation_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "input_hash": { + "name": "input_hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "receipt_json": { + "name": "receipt_json", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + } + }, + "indexes": {}, + "foreignKeys": { + "canonical_writer_operation_receipts_execution_id_canonical_agent_turns_execution_id_fk": { + "name": "canonical_writer_operation_receipts_execution_id_canonical_agent_turns_execution_id_fk", + "tableFrom": "canonical_writer_operation_receipts", + "tableTo": "canonical_agent_turns", + "columnsFrom": [ + "execution_id" + ], + "columnsTo": [ + "execution_id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "canonical_writer_operation_receipts_execution_id_operation_id_pk": { + "columns": [ + "execution_id", + "operation_id" + ], + "name": "canonical_writer_operation_receipts_execution_id_operation_id_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "cleanup_jobs": { + "name": "cleanup_jobs", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "thread_id": { + "name": "thread_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "workspace_path": { + "name": "workspace_path", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "worktree_path": { + "name": "worktree_path", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "branch": { + "name": "branch", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'explicit'" + }, + "attempts": { + "name": "attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "next_retry_at": { + "name": "next_retry_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "last_error": { + "name": "last_error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "cleanup_jobs_thread_id_unique": { + "name": "cleanup_jobs_thread_id_unique", + "columns": [ + "thread_id" + ], + "isUnique": true + }, + "idx_cleanup_jobs_retry": { + "name": "idx_cleanup_jobs_retry", + "columns": [ + "next_retry_at", + "attempts", + "created_at" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "conversation_display_materialization_state": { + "name": "conversation_display_materialization_state", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "last_source_created_at": { + "name": "last_source_created_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_source_id": { + "name": "last_source_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "completed": { + "name": "completed", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "diff_summaries": { + "name": "diff_summaries", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "thread_id": { + "name": "thread_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "turn_count": { + "name": "turn_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "last_turn_id": { + "name": "last_turn_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "model": { + "name": "model", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + } + }, + "indexes": { + "idx_diff_summaries_thread": { + "name": "idx_diff_summaries_thread", + "columns": [ + "thread_id" + ], + "isUnique": true + } + }, + "foreignKeys": { + "diff_summaries_thread_id_threads_id_fk": { + "name": "diff_summaries_thread_id_threads_id_fk", + "tableFrom": "diff_summaries", + "tableTo": "threads", + "columnsFrom": [ + "thread_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "external_thread_control_deliveries": { + "name": "external_thread_control_deliveries", + "columns": { + "pairing_id": { + "name": "pairing_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "authority_epoch": { + "name": "authority_epoch", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "delivery_id": { + "name": "delivery_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "fingerprint": { + "name": "fingerprint", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'in_flight'" + }, + "result_json": { + "name": "result_json", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + }, + "expires_at": { + "name": "expires_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_external_thread_control_delivery_identity": { + "name": "idx_external_thread_control_delivery_identity", + "columns": [ + "pairing_id", + "authority_epoch", + "delivery_id" + ], + "isUnique": true + }, + "idx_external_thread_control_delivery_retention": { + "name": "idx_external_thread_control_delivery_retention", + "columns": [ + "pairing_id", + "status", + "expires_at" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "external_thread_control_pairings": { + "name": "external_thread_control_pairings", + "columns": { + "pairing_id": { + "name": "pairing_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "integration_id": { + "name": "integration_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "credential_hash": { + "name": "credential_hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "workspace_ids_json": { + "name": "workspace_ids_json", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'[]'" + }, + "scopes_json": { + "name": "scopes_json", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'[]'" + }, + "calls_per_minute": { + "name": "calls_per_minute", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "max_active_threads": { + "name": "max_active_threads", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'active'" + }, + "authority_epoch": { + "name": "authority_epoch", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + }, + "replaced_by_pairing_id": { + "name": "replaced_by_pairing_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "replaces_pairing_id": { + "name": "replaces_pairing_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "external_thread_control_pairings_credential_hash_unique": { + "name": "external_thread_control_pairings_credential_hash_unique", + "columns": [ + "credential_hash" + ], + "isUnique": true + }, + "idx_external_thread_control_pairings_integration": { + "name": "idx_external_thread_control_pairings_integration", + "columns": [ + "integration_id", + "status" + ], + "isUnique": false + }, + "idx_external_thread_control_active_integration": { + "name": "idx_external_thread_control_active_integration", + "columns": [ + "integration_id" + ], + "isUnique": true, + "where": "status = 'active'" + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "hook_executions": { + "name": "hook_executions", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "message_id": { + "name": "message_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "hook_name": { + "name": "hook_name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "tool_name": { + "name": "tool_name", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "phase": { + "name": "phase", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "payload": { + "name": "payload", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'{}'" + }, + "duration_ms": { + "name": "duration_ms", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "did_block": { + "name": "did_block", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "started_at": { + "name": "started_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + }, + "ended_at": { + "name": "ended_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "sort_order": { + "name": "sort_order", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + } + }, + "indexes": { + "idx_hook_executions_message_sort_order_id": { + "name": "idx_hook_executions_message_sort_order_id", + "columns": [ + "message_id", + "sort_order", + "id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "hook_executions_message_id_messages_id_fk": { + "name": "hook_executions_message_id_messages_id_fk", + "tableFrom": "hook_executions", + "tableTo": "messages", + "columnsFrom": [ + "message_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "messages": { + "name": "messages", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "thread_id": { + "name": "thread_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "tool_calls": { + "name": "tool_calls", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "files_changed": { + "name": "files_changed", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "cost_usd": { + "name": "cost_usd", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "tokens_used": { + "name": "tokens_used", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "timestamp": { + "name": "timestamp", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + }, + "sequence": { + "name": "sequence", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "attachments": { + "name": "attachments", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "preview_annotations": { + "name": "preview_annotations", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "mentions": { + "name": "mentions", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "selected_text_comments": { + "name": "selected_text_comments", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "reply_to_message_id": { + "name": "reply_to_message_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "quoted_text": { + "name": "quoted_text", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "model": { + "name": "model", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "origin_type": { + "name": "origin_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'legacy'" + }, + "source_thread_id": { + "name": "source_thread_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "source_turn_id": { + "name": "source_turn_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "source_provider_id": { + "name": "source_provider_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "legacy_provenance": { + "name": "legacy_provenance", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "parent_agent_provenance": { + "name": "parent_agent_provenance", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "is_internal": { + "name": "is_internal", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "outcome": { + "name": "outcome", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "outcome_execution_id": { + "name": "outcome_execution_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "system_notice": { + "name": "system_notice", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "idx_messages_thread": { + "name": "idx_messages_thread", + "columns": [ + "thread_id" + ], + "isUnique": false + }, + "idx_messages_sequence": { + "name": "idx_messages_sequence", + "columns": [ + "thread_id", + "sequence" + ], + "isUnique": false + }, + "idx_messages_thread_sequence_id": { + "name": "idx_messages_thread_sequence_id", + "columns": [ + "thread_id", + "sequence", + "id" + ], + "isUnique": false + }, + "idx_messages_notice_session_sequence": { + "name": "idx_messages_notice_session_sequence", + "columns": [ + "thread_id", + "json_extract(\"system_notice\", '$.sessionId')", + "\"sequence\" desc" + ], + "isUnique": false + } + }, + "foreignKeys": { + "messages_thread_id_threads_id_fk": { + "name": "messages_thread_id_threads_id_fk", + "tableFrom": "messages", + "tableTo": "threads", + "columnsFrom": [ + "thread_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "messages_reply_to_message_id_messages_id_fk": { + "name": "messages_reply_to_message_id_messages_id_fk", + "tableFrom": "messages", + "tableTo": "messages", + "columnsFrom": [ + "reply_to_message_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "parent_assistant_text_checkpoint_chunks": { + "name": "parent_assistant_text_checkpoint_chunks", + "columns": { + "execution_id": { + "name": "execution_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "first_sequence": { + "name": "first_sequence", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "last_sequence": { + "name": "last_sequence", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "text": { + "name": "text", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "byte_length": { + "name": "byte_length", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_parent_assistant_text_checkpoint_chunks_sequence": { + "name": "idx_parent_assistant_text_checkpoint_chunks_sequence", + "columns": [ + "execution_id", + "first_sequence" + ], + "isUnique": true + } + }, + "foreignKeys": { + "parent_assistant_text_checkpoint_chunks_execution_id_parent_assistant_text_checkpoints_execution_id_fk": { + "name": "parent_assistant_text_checkpoint_chunks_execution_id_parent_assistant_text_checkpoints_execution_id_fk", + "tableFrom": "parent_assistant_text_checkpoint_chunks", + "tableTo": "parent_assistant_text_checkpoints", + "columnsFrom": [ + "execution_id" + ], + "columnsTo": [ + "execution_id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "parent_assistant_text_checkpoints": { + "name": "parent_assistant_text_checkpoints", + "columns": { + "execution_id": { + "name": "execution_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "thread_id": { + "name": "thread_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "turn_id": { + "name": "turn_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "last_sequence": { + "name": "last_sequence", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "retained_bytes": { + "name": "retained_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "retained_chunks": { + "name": "retained_chunks", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_parent_assistant_text_checkpoints_thread": { + "name": "idx_parent_assistant_text_checkpoints_thread", + "columns": [ + "thread_id", + "updated_at" + ], + "isUnique": false + } + }, + "foreignKeys": { + "parent_assistant_text_checkpoints_thread_id_canonical_agent_threads_id_fk": { + "name": "parent_assistant_text_checkpoints_thread_id_canonical_agent_threads_id_fk", + "tableFrom": "parent_assistant_text_checkpoints", + "tableTo": "canonical_agent_threads", + "columnsFrom": [ + "thread_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "parent_assistant_text_checkpoints_turn_id_canonical_agent_turns_id_fk": { + "name": "parent_assistant_text_checkpoints_turn_id_canonical_agent_turns_id_fk", + "tableFrom": "parent_assistant_text_checkpoints", + "tableTo": "canonical_agent_turns", + "columnsFrom": [ + "turn_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "plan_question_answers": { + "name": "plan_question_answers", + "columns": { + "assistant_message_id": { + "name": "assistant_message_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "thread_id": { + "name": "thread_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "answered_at": { + "name": "answered_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + } + }, + "indexes": { + "idx_plan_question_answers_thread_answered_at": { + "name": "idx_plan_question_answers_thread_answered_at", + "columns": [ + "thread_id", + "answered_at" + ], + "isUnique": false + } + }, + "foreignKeys": { + "plan_question_answers_assistant_message_id_messages_id_fk": { + "name": "plan_question_answers_assistant_message_id_messages_id_fk", + "tableFrom": "plan_question_answers", + "tableTo": "messages", + "columnsFrom": [ + "assistant_message_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "plan_question_answers_thread_id_threads_id_fk": { + "name": "plan_question_answers_thread_id_threads_id_fk", + "tableFrom": "plan_question_answers", + "tableTo": "threads", + "columnsFrom": [ + "thread_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "plans": { + "name": "plans", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "thread_id": { + "name": "thread_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "message_id": { + "name": "message_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "version": { + "name": "version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 1 + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "content_md": { + "name": "content_md", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "sections_json": { + "name": "sections_json", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "change_summary": { + "name": "change_summary", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'draft'" + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + } + }, + "indexes": { + "idx_plans_thread": { + "name": "idx_plans_thread", + "columns": [ + "thread_id" + ], + "isUnique": false + }, + "idx_plans_thread_version": { + "name": "idx_plans_thread_version", + "columns": [ + "thread_id", + "version" + ], + "isUnique": false + } + }, + "foreignKeys": { + "plans_thread_id_threads_id_fk": { + "name": "plans_thread_id_threads_id_fk", + "tableFrom": "plans", + "tableTo": "threads", + "columnsFrom": [ + "thread_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "plans_message_id_messages_id_fk": { + "name": "plans_message_id_messages_id_fk", + "tableFrom": "plans", + "tableTo": "messages", + "columnsFrom": [ + "message_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "project_action_runs": { + "name": "project_action_runs", + "columns": { + "thread_id": { + "name": "thread_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "action_id": { + "name": "action_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "run_id": { + "name": "run_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "revision": { + "name": "revision", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "terminal_session_id": { + "name": "terminal_session_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "action_name": { + "name": "action_name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "snapshot_json": { + "name": "snapshot_json", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "started_at": { + "name": "started_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "finished_at": { + "name": "finished_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "exit_code": { + "name": "exit_code", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "transcript": { + "name": "transcript", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "transcript_truncated": { + "name": "transcript_truncated", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + } + }, + "indexes": { + "idx_project_action_runs_slot": { + "name": "idx_project_action_runs_slot", + "columns": [ + "thread_id", + "action_id" + ], + "isUnique": true + }, + "idx_project_action_runs_thread": { + "name": "idx_project_action_runs_thread", + "columns": [ + "thread_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "project_action_runs_thread_id_threads_id_fk": { + "name": "project_action_runs_thread_id_threads_id_fk", + "tableFrom": "project_action_runs", + "tableTo": "threads", + "columnsFrom": [ + "thread_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "project_action_runs_workspace_id_workspaces_id_fk": { + "name": "project_action_runs_workspace_id_workspaces_id_fk", + "tableFrom": "project_action_runs", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "provider_catalog_snapshots": { + "name": "provider_catalog_snapshots", + "columns": { + "context_key": { + "name": "context_key", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "cwd": { + "name": "cwd", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "snapshot_json": { + "name": "snapshot_json", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + } + }, + "indexes": { + "idx_provider_catalog_snapshots_workspace": { + "name": "idx_provider_catalog_snapshots_workspace", + "columns": [ + "workspace_id" + ], + "isUnique": false + }, + "idx_provider_catalog_snapshots_provider": { + "name": "idx_provider_catalog_snapshots_provider", + "columns": [ + "provider_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "provider_catalog_snapshots_workspace_id_workspaces_id_fk": { + "name": "provider_catalog_snapshots_workspace_id_workspaces_id_fk", + "tableFrom": "provider_catalog_snapshots", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "provider_model_cache": { + "name": "provider_model_cache", + "columns": { + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "models_json": { + "name": "models_json", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "fetched_at": { + "name": "fetched_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + }, + "model_count": { + "name": "model_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "pull_request_review_links": { + "name": "pull_request_review_links", + "columns": { + "worktree_id": { + "name": "worktree_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "repository_node_id": { + "name": "repository_node_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "pull_request_number": { + "name": "pull_request_number", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "pr_url": { + "name": "pr_url", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "pr_state": { + "name": "pr_state", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "worktree_path": { + "name": "worktree_path", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "worktree_managed": { + "name": "worktree_managed", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 1 + }, + "head_repository_node_id": { + "name": "head_repository_node_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "head_repository_owner": { + "name": "head_repository_owner", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "head_repository_name": { + "name": "head_repository_name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "head_ref": { + "name": "head_ref", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "head_oid": { + "name": "head_oid", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "local_branch": { + "name": "local_branch", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "push_remote": { + "name": "push_remote", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "push_ref": { + "name": "push_ref", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "managed_remote_name": { + "name": "managed_remote_name", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "primary_thread_id": { + "name": "primary_thread_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + } + }, + "indexes": { + "idx_pull_request_review_links_identity": { + "name": "idx_pull_request_review_links_identity", + "columns": [ + "provider", + "repository_node_id", + "pull_request_number" + ], + "isUnique": true + }, + "idx_pull_request_review_links_primary_thread": { + "name": "idx_pull_request_review_links_primary_thread", + "columns": [ + "primary_thread_id" + ], + "isUnique": true + }, + "idx_pull_request_review_links_workspace": { + "name": "idx_pull_request_review_links_workspace", + "columns": [ + "workspace_id" + ], + "isUnique": false + }, + "idx_pull_request_review_links_worktree_path": { + "name": "idx_pull_request_review_links_worktree_path", + "columns": [ + "worktree_path" + ], + "isUnique": false + } + }, + "foreignKeys": { + "pull_request_review_links_workspace_id_workspaces_id_fk": { + "name": "pull_request_review_links_workspace_id_workspaces_id_fk", + "tableFrom": "pull_request_review_links", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "pull_request_review_links_primary_thread_id_threads_id_fk": { + "name": "pull_request_review_links_primary_thread_id_threads_id_fk", + "tableFrom": "pull_request_review_links", + "tableTo": "threads", + "columnsFrom": [ + "primary_thread_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "terminal_cleanup_ledger": { + "name": "terminal_cleanup_ledger", + "columns": { + "session_id": { + "name": "session_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "host_generation": { + "name": "host_generation", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "root_pid": { + "name": "root_pid", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "process_group_id": { + "name": "process_group_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "containment": { + "name": "containment", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "thought_segments": { + "name": "thought_segments", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "message_id": { + "name": "message_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "text": { + "name": "text", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "started_at": { + "name": "started_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + }, + "ended_at": { + "name": "ended_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "sort_order": { + "name": "sort_order", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "is_final_response": { + "name": "is_final_response", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + } + }, + "indexes": { + "idx_thought_segments_message_final_sort_order_id": { + "name": "idx_thought_segments_message_final_sort_order_id", + "columns": [ + "message_id", + "is_final_response", + "sort_order", + "id" + ], + "isUnique": false + }, + "idx_thought_segments_final_message_sort_order_id": { + "name": "idx_thought_segments_final_message_sort_order_id", + "columns": [ + "message_id", + "sort_order", + "id" + ], + "isUnique": false, + "where": "\"thought_segments\".\"is_final_response\" <> 0" + } + }, + "foreignKeys": { + "thought_segments_message_id_messages_id_fk": { + "name": "thought_segments_message_id_messages_id_fk", + "tableFrom": "thought_segments", + "tableTo": "messages", + "columnsFrom": [ + "message_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "thread_control_approvals": { + "name": "thread_control_approvals", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "thread_id": { + "name": "thread_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "prompt": { + "name": "prompt", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "execution_json": { + "name": "execution_json", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "placement_json": { + "name": "placement_json", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "turn_id": { + "name": "turn_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "caller_id": { + "name": "caller_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "source_thread_id": { + "name": "source_thread_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "source_turn_id": { + "name": "source_turn_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "source_provider_id": { + "name": "source_provider_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "operation": { + "name": "operation", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'thread_create_batch'" + }, + "operation_phase": { + "name": "operation_phase", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'pre_provision'" + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'pending'" + }, + "processing_started_at": { + "name": "processing_started_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + }, + "resolved_at": { + "name": "resolved_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "idx_thread_control_approvals_thread": { + "name": "idx_thread_control_approvals_thread", + "columns": [ + "thread_id", + "status" + ], + "isUnique": false + } + }, + "foreignKeys": { + "thread_control_approvals_thread_id_threads_id_fk": { + "name": "thread_control_approvals_thread_id_threads_id_fk", + "tableFrom": "thread_control_approvals", + "tableTo": "threads", + "columnsFrom": [ + "thread_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "thread_control_approvals_workspace_id_workspaces_id_fk": { + "name": "thread_control_approvals_workspace_id_workspaces_id_fk", + "tableFrom": "thread_control_approvals", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "thread_control_audit": { + "name": "thread_control_audit", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "caller_id": { + "name": "caller_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source_thread_id": { + "name": "source_thread_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "thread_id": { + "name": "thread_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "operation": { + "name": "operation", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "outcome": { + "name": "outcome", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + } + }, + "indexes": { + "idx_thread_control_audit_thread": { + "name": "idx_thread_control_audit_thread", + "columns": [ + "thread_id", + "created_at" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "thread_startups": { + "name": "thread_startups", + "columns": { + "startup_id": { + "name": "startup_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "phase": { + "name": "phase", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "steps_json": { + "name": "steps_json", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "transcript_json": { + "name": "transcript_json", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "cancellation": { + "name": "cancellation", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'none'" + }, + "revision": { + "name": "revision", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "request_fingerprint": { + "name": "request_fingerprint", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "thread_id": { + "name": "thread_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "error_json": { + "name": "error_json", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "block_json": { + "name": "block_json", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + } + }, + "indexes": { + "idx_thread_startups_workspace_updated": { + "name": "idx_thread_startups_workspace_updated", + "columns": [ + "workspace_id", + "\"updated_at\" desc" + ], + "isUnique": false + }, + "idx_thread_startups_state": { + "name": "idx_thread_startups_state", + "columns": [ + "state" + ], + "isUnique": false + } + }, + "foreignKeys": { + "thread_startups_workspace_id_workspaces_id_fk": { + "name": "thread_startups_workspace_id_workspaces_id_fk", + "tableFrom": "thread_startups", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "thread_startups_thread_id_threads_id_fk": { + "name": "thread_startups_thread_id_threads_id_fk", + "tableFrom": "thread_startups", + "tableTo": "threads", + "columnsFrom": [ + "thread_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "thread_tasks": { + "name": "thread_tasks", + "columns": { + "thread_id": { + "name": "thread_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "tasks_json": { + "name": "tasks_json", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + } + }, + "indexes": {}, + "foreignKeys": { + "thread_tasks_thread_id_threads_id_fk": { + "name": "thread_tasks_thread_id_threads_id_fk", + "tableFrom": "thread_tasks", + "tableTo": "threads", + "columnsFrom": [ + "thread_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "threads": { + "name": "threads", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'active'" + }, + "mode": { + "name": "mode", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'direct'" + }, + "worktree_path": { + "name": "worktree_path", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "branch": { + "name": "branch", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "checkout_state": { + "name": "checkout_state", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'named'" + }, + "base_branch": { + "name": "base_branch", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "issue_number": { + "name": "issue_number", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "pr_number": { + "name": "pr_number", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "pr_status": { + "name": "pr_status", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + }, + "deleted_at": { + "name": "deleted_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "user_completed_at": { + "name": "user_completed_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "scheduled_deletion_at": { + "name": "scheduled_deletion_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "cleanup_state": { + "name": "cleanup_state", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "cleanup_reason": { + "name": "cleanup_reason", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "model": { + "name": "model", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "worktree_managed": { + "name": "worktree_managed", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 1 + }, + "sdk_session_id": { + "name": "sdk_session_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_context_tokens": { + "name": "last_context_tokens", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "context_window": { + "name": "context_window", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'claude'" + }, + "reasoning_level": { + "name": "reasoning_level", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "interaction_mode": { + "name": "interaction_mode", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "orchestration_mode": { + "name": "orchestration_mode", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "permission_mode": { + "name": "permission_mode", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "parent_thread_id": { + "name": "parent_thread_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "forked_from_message_id": { + "name": "forked_from_message_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "delegation_coordinator_thread_id": { + "name": "delegation_coordinator_thread_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "delegation_creator_turn_id": { + "name": "delegation_creator_turn_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "delegation_creator_tool_call_id": { + "name": "delegation_creator_tool_call_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "delegation_creation_kind": { + "name": "delegation_creation_kind", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_by_integration_id": { + "name": "created_by_integration_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_compact_summary": { + "name": "last_compact_summary", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "copilot_agent": { + "name": "copilot_agent", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "devin_mode": { + "name": "devin_mode", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "context_window_mode": { + "name": "context_window_mode", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "thinking": { + "name": "thinking", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "codex_fast_mode": { + "name": "codex_fast_mode", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "default_open_in_app": { + "name": "default_open_in_app", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "has_file_changes": { + "name": "has_file_changes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "current_notice_session_id": { + "name": "current_notice_session_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "idx_threads_workspace": { + "name": "idx_threads_workspace", + "columns": [ + "workspace_id" + ], + "isUnique": false + }, + "idx_threads_workspace_deleted": { + "name": "idx_threads_workspace_deleted", + "columns": [ + "workspace_id", + "deleted_at" + ], + "isUnique": false + }, + "idx_threads_workspace_completed": { + "name": "idx_threads_workspace_completed", + "columns": [ + "workspace_id", + "user_completed_at" + ], + "isUnique": false + }, + "idx_threads_cleanup_due": { + "name": "idx_threads_cleanup_due", + "columns": [ + "cleanup_state", + "scheduled_deletion_at" + ], + "isUnique": false + }, + "idx_threads_workspace_recency": { + "name": "idx_threads_workspace_recency", + "columns": [ + "workspace_id", + "\"updated_at\" desc" + ], + "isUnique": false + }, + "idx_threads_status": { + "name": "idx_threads_status", + "columns": [ + "status" + ], + "isUnique": false + }, + "idx_threads_parent_thread_id": { + "name": "idx_threads_parent_thread_id", + "columns": [ + "parent_thread_id" + ], + "isUnique": false + }, + "idx_threads_forked_from_message_id": { + "name": "idx_threads_forked_from_message_id", + "columns": [ + "forked_from_message_id" + ], + "isUnique": false + }, + "idx_threads_delegation_coordinator": { + "name": "idx_threads_delegation_coordinator", + "columns": [ + "delegation_coordinator_thread_id" + ], + "isUnique": false + }, + "idx_threads_created_by_integration": { + "name": "idx_threads_created_by_integration", + "columns": [ + "created_by_integration_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "threads_workspace_id_workspaces_id_fk": { + "name": "threads_workspace_id_workspaces_id_fk", + "tableFrom": "threads", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "threads_delegation_coordinator_thread_id_threads_id_fk": { + "name": "threads_delegation_coordinator_thread_id_threads_id_fk", + "tableFrom": "threads", + "tableTo": "threads", + "columnsFrom": [ + "delegation_coordinator_thread_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "tool_call_records": { + "name": "tool_call_records", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "message_id": { + "name": "message_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "parent_tool_call_id": { + "name": "parent_tool_call_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "tool_name": { + "name": "tool_name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "display_name": { + "name": "display_name", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "provider_agent_key": { + "name": "provider_agent_key", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "subagent_identity_key": { + "name": "subagent_identity_key", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "subagent_provider_name": { + "name": "subagent_provider_name", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "subagent_prompt": { + "name": "subagent_prompt", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "subagent_type": { + "name": "subagent_type", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "subagent_agent_id": { + "name": "subagent_agent_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "subagent_duration_ms": { + "name": "subagent_duration_ms", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "model": { + "name": "model", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "reasoning_effort": { + "name": "reasoning_effort", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "input_summary": { + "name": "input_summary", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "output_summary": { + "name": "output_summary", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "output_truncated": { + "name": "output_truncated", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "output_total_bytes": { + "name": "output_total_bytes", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "output_artifact_path": { + "name": "output_artifact_path", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "exit_code": { + "name": "exit_code", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'running'" + }, + "started_at": { + "name": "started_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + }, + "completed_at": { + "name": "completed_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "sort_order": { + "name": "sort_order", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + } + }, + "indexes": { + "idx_tool_call_records_message_sort_order_id": { + "name": "idx_tool_call_records_message_sort_order_id", + "columns": [ + "message_id", + "sort_order", + "id" + ], + "isUnique": false + }, + "idx_tool_call_records_parent": { + "name": "idx_tool_call_records_parent", + "columns": [ + "parent_tool_call_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "tool_call_records_message_id_messages_id_fk": { + "name": "tool_call_records_message_id_messages_id_fk", + "tableFrom": "tool_call_records", + "tableTo": "messages", + "columnsFrom": [ + "message_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "turn_diff_snapshots": { + "name": "turn_diff_snapshots", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "message_id": { + "name": "message_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "thread_id": { + "name": "thread_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source": { + "name": "source", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "fidelity": { + "name": "fidelity", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "patch": { + "name": "patch", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "revision": { + "name": "revision", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + } + }, + "indexes": { + "idx_turn_diff_snapshots_message": { + "name": "idx_turn_diff_snapshots_message", + "columns": [ + "message_id" + ], + "isUnique": true + }, + "idx_turn_diff_snapshots_thread": { + "name": "idx_turn_diff_snapshots_thread", + "columns": [ + "thread_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "turn_diff_snapshots_message_id_messages_id_fk": { + "name": "turn_diff_snapshots_message_id_messages_id_fk", + "tableFrom": "turn_diff_snapshots", + "tableTo": "messages", + "columnsFrom": [ + "message_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "turn_diff_snapshots_thread_id_threads_id_fk": { + "name": "turn_diff_snapshots_thread_id_threads_id_fk", + "tableFrom": "turn_diff_snapshots", + "tableTo": "threads", + "columnsFrom": [ + "thread_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "turn_snapshots": { + "name": "turn_snapshots", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "message_id": { + "name": "message_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "thread_id": { + "name": "thread_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ref_before": { + "name": "ref_before", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ref_after": { + "name": "ref_after", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "files_changed": { + "name": "files_changed", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'[]'" + }, + "file_effects": { + "name": "file_effects", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'{\"revision\":0,\"fileCount\":0,\"additions\":0,\"deletions\":0,\"effects\":[]}'" + }, + "worktree_path": { + "name": "worktree_path", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + } + }, + "indexes": { + "idx_turn_snapshots_message": { + "name": "idx_turn_snapshots_message", + "columns": [ + "message_id" + ], + "isUnique": false + }, + "idx_turn_snapshots_thread": { + "name": "idx_turn_snapshots_thread", + "columns": [ + "thread_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "turn_snapshots_message_id_messages_id_fk": { + "name": "turn_snapshots_message_id_messages_id_fk", + "tableFrom": "turn_snapshots", + "tableTo": "messages", + "columnsFrom": [ + "message_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "turn_snapshots_thread_id_threads_id_fk": { + "name": "turn_snapshots_thread_id_threads_id_fk", + "tableFrom": "turn_snapshots", + "tableTo": "threads", + "columnsFrom": [ + "thread_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "workspace_environment_automatic_setup_attempts": { + "name": "workspace_environment_automatic_setup_attempts", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "thread_id": { + "name": "thread_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "reason": { + "name": "reason", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "launch_snapshot_json": { + "name": "launch_snapshot_json", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "outcome": { + "name": "outcome", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + }, + "started_at": { + "name": "started_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "finished_at": { + "name": "finished_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "exit_code": { + "name": "exit_code", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "output": { + "name": "output", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "output_truncated": { + "name": "output_truncated", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + } + }, + "indexes": { + "idx_workspace_environment_automatic_setup_attempts_thread": { + "name": "idx_workspace_environment_automatic_setup_attempts_thread", + "columns": [ + "thread_id", + "\"created_at\" desc" + ], + "isUnique": false + } + }, + "foreignKeys": { + "workspace_environment_automatic_setup_attempts_thread_id_threads_id_fk": { + "name": "workspace_environment_automatic_setup_attempts_thread_id_threads_id_fk", + "tableFrom": "workspace_environment_automatic_setup_attempts", + "tableTo": "threads", + "columnsFrom": [ + "thread_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "workspace_environment_command_approvals": { + "name": "workspace_environment_command_approvals", + "columns": { + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "command_id": { + "name": "command_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "fingerprint": { + "name": "fingerprint", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "approved_at": { + "name": "approved_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + } + }, + "indexes": { + "idx_workspace_environment_command_approvals_command": { + "name": "idx_workspace_environment_command_approvals_command", + "columns": [ + "workspace_id", + "command_id" + ], + "isUnique": true + } + }, + "foreignKeys": { + "workspace_environment_command_approvals_workspace_id_workspaces_id_fk": { + "name": "workspace_environment_command_approvals_workspace_id_workspaces_id_fk", + "tableFrom": "workspace_environment_command_approvals", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "workspace_environment_queued_turns": { + "name": "workspace_environment_queued_turns", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "thread_id": { + "name": "thread_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "message_id": { + "name": "message_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "queue_position": { + "name": "queue_position", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "submission_json": { + "name": "submission_json", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + }, + "released_at": { + "name": "released_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "dispatching_at": { + "name": "dispatching_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "dispatched_at": { + "name": "dispatched_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "idx_workspace_environment_queued_turns_state": { + "name": "idx_workspace_environment_queued_turns_state", + "columns": [ + "state", + "created_at" + ], + "isUnique": false + }, + "idx_workspace_environment_queued_turns_thread_state_position": { + "name": "idx_workspace_environment_queued_turns_thread_state_position", + "columns": [ + "thread_id", + "state", + "queue_position" + ], + "isUnique": false + } + }, + "foreignKeys": { + "workspace_environment_queued_turns_thread_id_threads_id_fk": { + "name": "workspace_environment_queued_turns_thread_id_threads_id_fk", + "tableFrom": "workspace_environment_queued_turns", + "tableTo": "threads", + "columnsFrom": [ + "thread_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "workspace_environment_setup_gates": { + "name": "workspace_environment_setup_gates", + "columns": { + "thread_id": { + "name": "thread_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "attempt_id": { + "name": "attempt_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + } + }, + "indexes": {}, + "foreignKeys": { + "workspace_environment_setup_gates_thread_id_threads_id_fk": { + "name": "workspace_environment_setup_gates_thread_id_threads_id_fk", + "tableFrom": "workspace_environment_setup_gates", + "tableTo": "threads", + "columnsFrom": [ + "thread_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "workspace_environment_storage_settings": { + "name": "workspace_environment_storage_settings", + "columns": { + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "storage_mode": { + "name": "storage_mode", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + } + }, + "indexes": {}, + "foreignKeys": { + "workspace_environment_storage_settings_workspace_id_workspaces_id_fk": { + "name": "workspace_environment_storage_settings_workspace_id_workspaces_id_fk", + "tableFrom": "workspace_environment_storage_settings", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "workspace_terminal_preferences": { + "name": "workspace_terminal_preferences", + "columns": { + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "default_profile_id": { + "name": "default_profile_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + } + }, + "indexes": {}, + "foreignKeys": { + "workspace_terminal_preferences_workspace_id_workspaces_id_fk": { + "name": "workspace_terminal_preferences_workspace_id_workspaces_id_fk", + "tableFrom": "workspace_terminal_preferences", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "workspace_worktrees": { + "name": "workspace_worktrees", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "workspace_id": { + "name": "workspace_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "canonical_path": { + "name": "canonical_path", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "label": { + "name": "label", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "branch": { + "name": "branch", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "base_ref": { + "name": "base_ref", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "managed": { + "name": "managed", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "last_seen_at": { + "name": "last_seen_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + }, + "stale": { + "name": "stale", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + } + }, + "indexes": { + "idx_workspace_worktrees_path": { + "name": "idx_workspace_worktrees_path", + "columns": [ + "workspace_id", + "canonical_path" + ], + "isUnique": true + }, + "idx_workspace_worktrees_workspace": { + "name": "idx_workspace_worktrees_workspace", + "columns": [ + "workspace_id", + "stale" + ], + "isUnique": false + } + }, + "foreignKeys": { + "workspace_worktrees_workspace_id_workspaces_id_fk": { + "name": "workspace_worktrees_workspace_id_workspaces_id_fk", + "tableFrom": "workspace_worktrees", + "tableTo": "workspaces", + "columnsFrom": [ + "workspace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "workspaces": { + "name": "workspaces", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "path": { + "name": "path", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "provider_config": { + "name": "provider_config", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'{}'" + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" + }, + "pinned": { + "name": "pinned", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "last_opened_at": { + "name": "last_opened_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "sort_order": { + "name": "sort_order", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "is_git_repo": { + "name": "is_git_repo", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 1 + }, + "deleted_at": { + "name": "deleted_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "workspaces_path_unique": { + "name": "workspaces_path_unique", + "columns": [ + "path" + ], + "isUnique": true + }, + "idx_workspaces_sort_order": { + "name": "idx_workspaces_sort_order", + "columns": [ + "\"sort_order\" asc" + ], + "isUnique": false + }, + "idx_workspaces_pinned_last_opened": { + "name": "idx_workspaces_pinned_last_opened", + "columns": [ + "\"pinned\" desc", + "\"last_opened_at\" desc" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + } + }, + "views": {}, + "enums": {}, + "_meta": { + "schemas": {}, + "tables": {}, + "columns": {} + }, + "internal": { + "indexes": { + "idx_messages_notice_session_sequence": { + "columns": { + "json_extract(\"system_notice\", '$.sessionId')": { + "isExpression": true + }, + "\"sequence\" desc": { + "isExpression": true + } + } + }, + "idx_thread_startups_workspace_updated": { + "columns": { + "\"updated_at\" desc": { + "isExpression": true + } + } + }, + "idx_threads_workspace_recency": { + "columns": { + "\"updated_at\" desc": { + "isExpression": true + } + } + }, + "idx_workspace_environment_automatic_setup_attempts_thread": { + "columns": { + "\"created_at\" desc": { + "isExpression": true + } + } + }, + "idx_workspaces_sort_order": { + "columns": { + "\"sort_order\" asc": { + "isExpression": true + } + } + }, + "idx_workspaces_pinned_last_opened": { + "columns": { + "\"pinned\" desc": { + "isExpression": true + }, + "\"last_opened_at\" desc": { + "isExpression": true + } + } + } + } + } +} \ No newline at end of file diff --git a/apps/server/drizzle/meta/_journal.json b/apps/server/drizzle/meta/_journal.json index 9aef29df4..ee9e5a450 100644 --- a/apps/server/drizzle/meta/_journal.json +++ b/apps/server/drizzle/meta/_journal.json @@ -470,6 +470,13 @@ "when": 1790275595252, "tag": "0065_ambitious_blindfold", "breakpoints": true + }, + { + "idx": 66, + "version": "6", + "when": 1790286163932, + "tag": "0066_fat_amazoness", + "breakpoints": true } ] } \ No newline at end of file diff --git a/apps/server/src/application/transport/__tests__/push.test.ts b/apps/server/src/application/transport/__tests__/push.test.ts index 871ad7851..01e229e63 100644 --- a/apps/server/src/application/transport/__tests__/push.test.ts +++ b/apps/server/src/application/transport/__tests__/push.test.ts @@ -127,6 +127,22 @@ describe("broadcast", () => { resetTransportPayloadValidatorForTest(); }); + it("keeps the durable publication identity while assigning a process sequence", () => { + const received: Array<{ buf: Buffer; binary: boolean }> = []; + const socket = fakeOpenSocket(received); + addClient(socket); + subscribeClientToThread(socket, "thread-a"); + const publicationId = "3"; + const first = broadcast("agent.event", { type: "turnStarted", threadId: "thread-a", + turnExecutionId: "00000000-0000-4000-8000-000000000001", publicationId }); + const replay = broadcast("agent.event", { type: "turnStarted", threadId: "thread-a", + turnExecutionId: "00000000-0000-4000-8000-000000000001", publicationId }); + expect(first).toMatchObject({ publicationId, sequence: 1 }); + expect(replay).toMatchObject({ publicationId, sequence: 2 }); + expect(received.map(({ buf }) => JSON.parse(buf.toString("utf-8")).data.publicationId)) + .toEqual([publicationId, publicationId]); + }); + it("routes thread-scoped events only to clients subscribed to that thread", () => { const a: Array<{ buf: Buffer; binary: boolean }> = []; const b: Array<{ buf: Buffer; binary: boolean }> = []; diff --git a/apps/server/src/features/agents/canonical/__tests__/canonical-agent-writer-client.test.ts b/apps/server/src/features/agents/canonical/__tests__/canonical-agent-writer-client.test.ts index 94ad3678a..7956e6a07 100644 --- a/apps/server/src/features/agents/canonical/__tests__/canonical-agent-writer-client.test.ts +++ b/apps/server/src/features/agents/canonical/__tests__/canonical-agent-writer-client.test.ts @@ -194,7 +194,7 @@ describe("canonical SQLite writer", () => { expect(created).toBe(2); expect(published).toEqual([AgentEventType.TurnStarted]); expect(receipt).toMatchObject({ kind: "committed", livePublication: [{ - publicationId: "publication-lease:1:0", after: "writer", + publicationId: "1", after: "writer", event: { type: AgentEventType.TurnStarted, turnExecutionId: EXECUTION_ID }, }] }); expect(db.prepare("SELECT COUNT(*) AS count FROM messages WHERE id = ?").get("publication-user")) @@ -342,7 +342,7 @@ describe("canonical SQLite writer", () => { : new Worker(new URL("../canonical-agent-writer.worker.ts", import.meta.url), { type: "module" })); const receipt = await writer.transactSemantic(reclassified, () => {}); expect(receipt).toMatchObject({ kind: "committed", operationId: "live-text-lease:3", - livePublication: [{ publicationId: "live-text-lease:3:0", after: "writer" }] }); + livePublication: [{ publicationId: "2", after: "writer" }] }); expect(created).toBe(2); expect(new ParentAssistantTextCheckpointService(db).restore(EXECUTION_ID)).toBe(""); expect(db.prepare("SELECT COUNT(*) AS count FROM canonical_agent_items WHERE id = ?") diff --git a/apps/server/src/features/agents/canonical/__tests__/canonical-execution-semantic-writer.test.ts b/apps/server/src/features/agents/canonical/__tests__/canonical-execution-semantic-writer.test.ts index fcb61d3d8..3e6696f12 100644 --- a/apps/server/src/features/agents/canonical/__tests__/canonical-execution-semantic-writer.test.ts +++ b/apps/server/src/features/agents/canonical/__tests__/canonical-execution-semantic-writer.test.ts @@ -18,6 +18,7 @@ import { NarrativeRecoveryDelta } from "../../turns/narrative-recovery-delta.js" import { CanonicalAgentBoundary } from "../canonical-agent-boundary.js"; import type { CodexSystemWriterIntent } from "../canonical-codex-system-error-projection.js"; import { CanonicalExecutionSemanticWriter } from "../canonical-execution-semantic-writer.js"; +import { ExecutionLivePublicationRelease } from "../execution-live-publication-release.js"; import type { DataOnlyParentTurnStartInput } from "../canonical-parent-turn-write.js"; const THREAD_ID = "thread-1"; @@ -379,7 +380,7 @@ describe("CanonicalExecutionSemanticWriter through ExecutionWorkerHandler", () = publication: { after: "writer", event }, }); expect(committedMessage).toMatchObject({ kind: "committed", livePublication: [{ - publicationId: "lease-1:2:0", event, + publicationId: "1", event, }] }); expect(new MessageRepo(db).findByIdInThreadIncludingInternal(THREAD_ID, message.messageId)) .toMatchObject({ id: message.messageId, content: message.content, model: null, is_internal: true }); @@ -562,7 +563,7 @@ describe("CanonicalExecutionSemanticWriter through ExecutionWorkerHandler", () = turnExecutionId: EXECUTION_ID, delta: "unknown draft" }; const appended = await writer.transact({ ...unknown, livePublication: [{ after: "writer", event: unknownEvent }] }); expect(appended).toMatchObject({ kind: "committed", assistantTextCheckpoint: { durableThrough: 1 }, - livePublication: [{ publicationId: "lease-1:2:0", event: unknownEvent }] }); + livePublication: [{ publicationId: "1", event: unknownEvent }] }); expect(new ParentAssistantTextCheckpointService(db).restore(EXECUTION_ID)).toBe("unknown draft"); const thought = { kind: "narrationSegment" as const, record: { @@ -578,12 +579,14 @@ describe("CanonicalExecutionSemanticWriter through ExecutionWorkerHandler", () = db.run("CREATE TRIGGER fail_compound_receipt BEFORE INSERT ON canonical_writer_operation_receipts WHEN NEW.kind = 'semantic:live-event' AND NEW.operation_id = 'lease-1:3' BEGIN SELECT RAISE(ABORT, 'compound receipt unavailable'); END"); await expect(writer.transact(reclassified)).rejects.toThrow("compound receipt unavailable"); expect(new ParentAssistantTextCheckpointService(db).restore(EXECUTION_ID)).toBe("unknown draft"); + expect(db.prepare("SELECT last_sequence FROM canonical_writer_live_publication_heads WHERE thread_id = ?") + .get(THREAD_ID)).toEqual({ last_sequence: 1 }); expect(new CanonicalAgentBoundary(db, () => {}).loadParentNarrativeRecovery(TURN_ID)).toEqual([]); expect(published).toHaveLength(publicationCount); db.run("DROP TRIGGER fail_compound_receipt"); const reclassifiedReceipt = await writer.transact(reclassified); expect(reclassifiedReceipt).toMatchObject({ kind: "committed", livePublication: [ - { publicationId: "lease-1:3:0", event: boundary }, + { publicationId: "2", event: boundary }, ] }); expect(new ParentAssistantTextCheckpointService(db).restore(EXECUTION_ID)).toBe(""); expect(new CanonicalAgentBoundary(db, () => {}).loadParentNarrativeRecovery(TURN_ID)).toEqual([thought]); @@ -601,7 +604,7 @@ describe("CanonicalExecutionSemanticWriter through ExecutionWorkerHandler", () = livePublication: [{ after: "writer" as const, event: finalBoundary }] }; expect(await writer.transact(promoted)).toMatchObject({ kind: "committed", assistantTextCheckpoint: { durableThrough: 1 }, - livePublication: [{ publicationId: "lease-1:4:0", event: finalBoundary }], + livePublication: [{ publicationId: "3", event: finalBoundary }], }); expect(new ParentAssistantTextCheckpointService(db).restore(EXECUTION_ID)).toBe("final thought"); expect(new CanonicalAgentBoundary(db, () => {}).loadParentNarrativeRecovery(TURN_ID)).toEqual([]); @@ -611,6 +614,32 @@ describe("CanonicalExecutionSemanticWriter through ExecutionWorkerHandler", () = })); }); + it("advances one thread publication identity across terminal and a new execution", async () => { + const firstStart = operation(1, { kind: "begin", providerId: "codex", input: startInput() }); + const firstEvent = { type: AgentEventType.TurnStarted, threadId: THREAD_ID, + turnExecutionId: EXECUTION_ID }; + expect(await writer.transact({ ...firstStart, livePublication: [{ after: "writer", event: firstEvent }] })) + .toMatchObject({ kind: "committed", livePublication: [{ publicationId: "1" }] }); + expect(writer.interruptWorkerLoss(loss).kind).toBe("committed"); + + const nextExecution = { ...execution, turnId: "turn-next", + executionId: "00000000-0000-4000-8000-000000000002" }; + const nextLease = { ...lease, leaseId: "lease-next" }; + const nextStart: ExecutionSemanticOperation = { + operationId: "lease-next:1", execution: nextExecution, lease: nextLease, ordinal: 1, + mutation: { kind: "begin", providerId: "codex", input: { + ...startInput(), turnId: nextExecution.turnId, executionId: nextExecution.executionId, + userMessage: { kind: "create", messageId: "next-user", content: "Next question", sequence: 3 }, + } }, + livePublication: [{ after: "writer", event: { ...firstEvent, + turnExecutionId: nextExecution.executionId } }], + }; + expect(await writer.transact(nextStart)) + .toMatchObject({ kind: "committed", livePublication: [{ publicationId: "2" }] }); + expect(db.prepare("SELECT last_sequence FROM canonical_writer_live_publication_heads WHERE thread_id = ?") + .get(THREAD_ID)).toEqual({ last_sequence: 2 }); + }); + it("durably acknowledges a boundary with no new text or narrative rows", async () => { expect((await send(1, { kind: "start", providerId: "codex", input: startInput() })).kind).toBe("committed"); const boundary = { type: AgentEventType.AssistantMessageBoundary, threadId: THREAD_ID, @@ -619,7 +648,7 @@ describe("CanonicalExecutionSemanticWriter through ExecutionWorkerHandler", () = livePublication: [{ after: "writer" as const, event: boundary }] }; const receipt = await writer.transact(op); expect(receipt).toMatchObject({ kind: "committed", livePublication: [ - { publicationId: "lease-1:2:0", event: boundary }, + { publicationId: "1", event: boundary }, ] }); expect(await writer.transact(op)).toEqual(receipt); expect(db.prepare("SELECT COUNT(*) AS count FROM canonical_writer_operation_receipts WHERE execution_id = ? AND operation_id = ?") @@ -645,13 +674,25 @@ describe("CanonicalExecutionSemanticWriter through ExecutionWorkerHandler", () = db.run("DROP TRIGGER fail_system_receipt"); const receipt = await writer.transact(op); - expect(receipt).toMatchObject({ kind: "committed", livePublication: [{ publicationId: "lease-1:2:0", + expect(receipt).toMatchObject({ kind: "committed", livePublication: [{ publicationId: "1", after: "writer", event: { ...notice, messageId: expect.any(String) } }] }); if (receipt.kind !== "committed") return; const messageId = receipt.livePublication?.[0]?.event.type === "system" ? receipt.livePublication[0].event.messageId : undefined; expect(messageId).toBeDefined(); if (!messageId) return; + const released: AgentEvent[] = []; + const release = new ExecutionLivePublicationRelease({ + isBound: () => true, + publish: (event) => { released.push(event); }, + }); + release.release(op, receipt); + expect(released).toEqual([{ ...notice, messageId, publicationId: "1" }]); + const actualPublication = receipt.livePublication?.[0]; + if (!actualPublication) throw new Error("Missing system live publication"); + expect(() => release.release(op, { ...receipt, livePublication: [{ ...actualPublication, + event: { ...notice, messageId, message: "Changed notice" }, + }] })).toThrow("Live AgentEvent publication receipt is invalid"); expect(new MessageRepo(db).findByIdInThread(THREAD_ID, messageId)) .toMatchObject({ role: "system", content: notice.message }); expect(await writer.transact(op)).toEqual(receipt); @@ -717,7 +758,7 @@ describe("CanonicalExecutionSemanticWriter through ExecutionWorkerHandler", () = db.run("DROP TRIGGER fail_task_receipt"); const receipt = await writer.transact(op); expect(receipt).toMatchObject({ kind: "committed", livePublication: [ - { publicationId: "lease-1:2:0", event: toolUse }, + { publicationId: "1", event: toolUse }, ] }); expect(new TaskRepo(db).get(THREAD_ID)).toEqual([ { content: "Ship task", status: "pending", group: "Tasks" }, @@ -884,14 +925,16 @@ describe("CanonicalExecutionSemanticWriter through ExecutionWorkerHandler", () = await expect(writer.transact(finish)).rejects.toThrow("finish unavailable"); expect(db.prepare("SELECT terminal_outcome FROM canonical_agent_ingest_checkpoints WHERE execution_id = ?") .get(EXECUTION_ID)).toEqual({ terminal_outcome: null }); + expect(db.prepare("SELECT status FROM threads WHERE id = ?").get(THREAD_ID)).toEqual({ status: "active" }); db.run("DROP TRIGGER fail_live_finish"); const receipt = await writer.transact(finish); expect(receipt).toMatchObject({ kind: "committed", livePublication: [{ - publicationId: "lease-1:2:0", after: "terminal", event: ended, + publicationId: "1", after: "terminal", event: ended, }] }); expect(db.prepare("SELECT terminal_outcome FROM canonical_agent_ingest_checkpoints WHERE execution_id = ?") .get(EXECUTION_ID)).toEqual({ terminal_outcome: "completed" }); + expect(db.prepare("SELECT status FROM threads WHERE id = ?").get(THREAD_ID)).toEqual({ status: "completed" }); db.close(true); db = openDatabase({ dbPath: path }); writer = new CanonicalExecutionSemanticWriter(db, () => {}); @@ -937,7 +980,7 @@ describe("CanonicalExecutionSemanticWriter through ExecutionWorkerHandler", () = db.prepare("UPDATE canonical_writer_operation_receipts SET receipt_json = ? WHERE execution_id = ? AND operation_id = ?") .run(JSON.stringify({ ...beginReceipt, publicationVersion: 1, livePublication: [{ - publicationId: "lease-1:1:0", after: "writer", + publicationId: "1", after: "writer", event: { ...started, threadId: "another-thread" }, }] }), EXECUTION_ID, begin.operationId); await expect(writer.transact(begin)).rejects.toThrow("Live publication receipt does not match its operation"); diff --git a/apps/server/src/features/agents/canonical/__tests__/canonical-execution-writer-port.test.ts b/apps/server/src/features/agents/canonical/__tests__/canonical-execution-writer-port.test.ts index e0f780626..01437279e 100644 --- a/apps/server/src/features/agents/canonical/__tests__/canonical-execution-writer-port.test.ts +++ b/apps/server/src/features/agents/canonical/__tests__/canonical-execution-writer-port.test.ts @@ -93,6 +93,7 @@ describe("execution semantic writer transport", () => { }; expect((await port.transact(begin)).kind).toBe("committed"); expect(published.map((event) => event.type)).toEqual([AgentEventType.TurnStarted]); + expect(published[0]?.publicationId).toBe("1"); const delta: AgentEvent = { type: AgentEventType.TextDelta, threadId: THREAD_ID, turnExecutionId: EXECUTION_ID, delta: "answer", isFinalResponse: true }; @@ -107,6 +108,8 @@ describe("execution semantic writer transport", () => { expect((await port.transact(append)).kind).toBe("committed"); expect(published.map((event) => event.type)) .toEqual([AgentEventType.TurnStarted, AgentEventType.TextDelta, AgentEventType.System]); + expect(published.slice(1).map((event) => event.publicationId)) + .toEqual(["2", "3"]); expect((await port.transact(append)).kind).toBe("committed"); expect(published).toHaveLength(3); @@ -167,7 +170,7 @@ describe("execution semantic writer transport", () => { }); if (admission.kind !== "admitted") throw new Error(`Execution admission failed: ${admission.kind}`); await expect(admission.completion).resolves.toMatchObject({ kind: "reply", result: { - kind: "committed", livePublication: [{ publicationId: `${claim.lease.leaseId}:1:0` }], + kind: "committed", livePublication: [{ publicationId: "1" }], } }); expect(published.map((event) => event.type)).toEqual([AgentEventType.TurnStarted]); expect(db.prepare("SELECT COUNT(*) AS count FROM messages WHERE thread_id = ?").get(THREAD_ID)) @@ -192,9 +195,9 @@ describe("execution semantic writer transport", () => { livePublication: [{ after: "writer", event }], }; expect((await port.transact(operation)).kind).toBe("committed"); - expect(published).toEqual([event]); + expect(published).toEqual([{ ...event, publicationId: "1" }]); expect((await port.transact(operation)).kind).toBe("committed"); - expect(published).toEqual([event]); + expect(published).toEqual([{ ...event, publicationId: "1" }]); }); it("replays a committed live receipt when the public publisher becomes available", async () => { @@ -218,6 +221,52 @@ describe("execution semantic writer transport", () => { expect(published).toHaveLength(1); }); + it("replays the same public identity when publication succeeds before the host loses its reply", async () => { + writer = new CanonicalAgentWriterClient(NodePath.join(directory, "app.sqlite")); + const registry = new AgentEventPublicationRegistry(); + const published: AgentEvent[] = []; + let loseReply = true; + registry.bind((event) => { + published.push(event); + if (loseReply) { + loseReply = false; + throw new Error("host lost reply after publish"); + } + }); + const operation: ExecutionSemanticOperation = { ...beginOperation(), livePublication: [{ + after: "writer", event: { + type: AgentEventType.TurnStarted, threadId: THREAD_ID, turnExecutionId: EXECUTION_ID, + }, + }] }; + const firstHost = new CanonicalExecutionWriterPort(writer, () => {}, new ExecutionLivePublicationRelease(registry)); + await expect(firstHost.transact(operation)).rejects.toThrow("host lost reply after publish"); + const restartedHost = new CanonicalExecutionWriterPort(writer, () => {}, new ExecutionLivePublicationRelease(registry)); + expect((await restartedHost.transact(operation)).kind).toBe("committed"); + expect(published.map((event) => event.publicationId)) + .toEqual(["1", "1"]); + expect(db.prepare("SELECT COUNT(*) AS count FROM messages WHERE thread_id = ?").get(THREAD_ID)) + .toEqual({ count: 1 }); + }); + + it("keeps releasing after the recent host cache reaches an all-day event count", () => { + let published = 0; + const release = new ExecutionLivePublicationRelease({ + isBound: () => true, + publish: () => { published++; }, + }); + const started: AgentEvent = { type: AgentEventType.TurnStarted, threadId: THREAD_ID, + turnExecutionId: EXECUTION_ID }; + for (let ordinal = 1; ordinal <= 8_193; ordinal++) { + const operationId = `${lease.leaseId}:${ordinal}`; + const operation: ExecutionSemanticOperation = { ...beginOperation(), operationId, ordinal, + livePublication: [{ after: "writer", event: started }] }; + release.release(operation, { kind: "committed", operationId, durableRevision: ordinal, + livePublication: [{ after: "writer", event: started, + publicationId: String(ordinal) }] }); + } + expect(published).toBe(8_193); + }); + it("commits and replays a semantic start through the dedicated SQLite worker", async () => { writer = new CanonicalAgentWriterClient(NodePath.join(directory, "app.sqlite")); const published: string[] = []; diff --git a/apps/server/src/features/agents/canonical/canonical-execution-semantic-writer.ts b/apps/server/src/features/agents/canonical/canonical-execution-semantic-writer.ts index ad622693f..6b28c4bff 100644 --- a/apps/server/src/features/agents/canonical/canonical-execution-semantic-writer.ts +++ b/apps/server/src/features/agents/canonical/canonical-execution-semantic-writer.ts @@ -15,6 +15,7 @@ import { import { z } from "zod"; import { ACTIVE_TURN_WRITE_BATCH_LIMITS } from "../../../runtime/persistence/sqlite/bounded-write-batches.js"; +import { ThreadRepo } from "../../thread-control/persistence/thread-repo.js"; import type { ExecutionIdentity, ExecutionLease } from "../execution/execution-mailbox-protocol.js"; import type { ExecutionProviderCommitReceipt, @@ -130,7 +131,7 @@ const storedReceiptSchema = z.object({ durableRevision: z.number().int(), publicationVersion: z.literal(1), livePublication: z.array(z.object({ - publicationId: z.string(), + publicationId: z.string().regex(/^[1-9]\d*$/).max(16), after: z.enum(["writer", "terminal"]), event: AgentEventSchema(), })).min(1).max(MAX_LIVE_PUBLICATION_EVENTS).optional(), @@ -164,6 +165,7 @@ const storedPublicationChunkPageSchema = z.array(storedOperationSchema.extend({ .max(PUBLICATION_CHUNK_PAGE_SIZE); const storedEnvelopeRowSchema = z.object({ envelope_json: z.string() }); const durableSequenceRowSchema = z.object({ last_durable_sequence: z.number().int() }); +const livePublicationHeadSchema = z.object({ last_sequence: z.number().int().positive().max(Number.MAX_SAFE_INTEGER) }); interface SemanticHead { readonly execution: ExecutionIdentity; @@ -199,12 +201,14 @@ export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter private readonly systemProjection: CanonicalCodexSystemErrorProjection; private readonly tasks: TaskRepo; private readonly plans: PlanRepo; + private readonly threads: ThreadRepo; private readonly assistantText: ParentAssistantTextCheckpointService; private readonly canonical: CanonicalAgentBoundary; private readonly findOperation: ReturnType; private readonly listPublicationChunks: ReturnType; private readonly findPublishedEvent: ReturnType; private readonly findDurableSequence: ReturnType; + private readonly advanceLivePublication: ReturnType; private readonly insertOperation: ReturnType; private readonly commitPendingFinish: ReturnType; private readonly updateHead: ReturnType; @@ -228,11 +232,15 @@ export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter this.systemProjection = new CanonicalCodexSystemErrorProjection(db); this.tasks = new TaskRepo(db); this.plans = new PlanRepo(db); + this.threads = new ThreadRepo(db); this.assistantText = new ParentAssistantTextCheckpointService(db); this.findOperation = db.prepare("SELECT kind, input_hash, receipt_json FROM canonical_writer_operation_receipts WHERE execution_id = ? AND operation_id = ?"); this.listPublicationChunks = db.prepare("SELECT operation_id, kind, input_hash, receipt_json FROM canonical_writer_operation_receipts WHERE execution_id = ? AND operation_id > ? AND operation_id < ? ORDER BY operation_id LIMIT ?"); this.findPublishedEvent = db.prepare("SELECT envelope_json FROM canonical_agent_events WHERE execution_id = ? AND accepted_sequence = ?"); this.findDurableSequence = db.prepare("SELECT last_durable_sequence FROM canonical_agent_ingest_checkpoints WHERE execution_id = ?"); + this.advanceLivePublication = db.prepare(`INSERT INTO canonical_writer_live_publication_heads (thread_id, last_sequence) + VALUES (?, ?) ON CONFLICT(thread_id) DO UPDATE SET last_sequence = last_sequence + excluded.last_sequence + WHERE last_sequence <= 9007199254740991 - excluded.last_sequence RETURNING last_sequence`); this.insertOperation = db.prepare("INSERT INTO canonical_writer_operation_receipts (execution_id, operation_id, kind, input_hash, receipt_json) VALUES (?, ?, ?, ?, ?)"); this.commitPendingFinish = db.prepare("UPDATE canonical_writer_operation_receipts SET kind = ?, receipt_json = ? WHERE execution_id = ? AND operation_id = ? AND kind = ? AND input_hash = ?"); this.updateHead = db.prepare("UPDATE canonical_writer_operation_receipts SET receipt_json = ? WHERE execution_id = ? AND operation_id = ? AND kind = ?"); @@ -306,8 +314,7 @@ export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter this.storePublicationChunks(input.execution.executionId, hash, sequences); this.storeHead({ ...current, ordinal: current.ordinal + 1, durableRevision: receipt.durableRevision, terminal: true }); - this.storeReceipt(operation, hash, receipt); - return receipt; + return this.storeReceipt(operation, hash, receipt); } private async applySupported(operation: ExecutionSemanticOperation, hash: string): Promise { @@ -360,8 +367,7 @@ export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter }; this.insertOperation.run(operation.execution.executionId, HEAD_ID, HEAD_KIND, fingerprint(head.lease), JSON.stringify(head)); this.storePublicationChunks(operation.execution.executionId, hash, result.events.map((event) => event.acceptedSequence)); - this.storeReceipt(operation, hash, receipt); - return receipt; + return this.storeReceipt(operation, hash, receipt); })()); } @@ -402,8 +408,7 @@ export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter executionId: event.routing.executionId, sequence: event.acceptedSequence, }))); - this.storeReceipt(operation, hash, receipt); - return receipt; + return this.storeReceipt(operation, hash, receipt); })()); } @@ -438,8 +443,7 @@ export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter // Assistant text has its own durable sequence; it does not advance the canonical event revision. const receipt = committed(operation, head.durableRevision, undefined, undefined, result); this.storeHead({ ...head, ordinal: operation.ordinal }); - this.storeReceipt(operation, hash, receipt); - return receipt; + return this.storeReceipt(operation, hash, receipt); })(); } @@ -452,8 +456,7 @@ export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter this.persistNarrativeDelta(mutation.input); const receipt = committed(operation, head.durableRevision); this.storeHead({ ...head, ordinal: operation.ordinal }); - this.storeReceipt(operation, hash, receipt); - return receipt; + return this.storeReceipt(operation, hash, receipt); })(); } @@ -473,11 +476,10 @@ export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter const { textResult, planQuestions, planOutput } = this.applyLiveFeatureEffects(operation, head); const livePublication = this.projectLiveSystem(operation); const committedReceipt = committed(operation, head.durableRevision, undefined, undefined, textResult, - livePublication, planQuestions, planOutput); + undefined, planQuestions, planOutput); this.storeHead({ ...head, ordinal: operation.ordinal, ...(mutation.message ? { assignedMessageId: mutation.message.messageId } : {}) }); - this.storeReceipt(operation, hash, committedReceipt); - return committedReceipt; + return this.storeReceipt(operation, hash, committedReceipt, livePublication); })(); if (mutation.text.kind === "reclassify") { this.assistantText.discardRecoveryJournal(operation.execution.executionId); @@ -485,7 +487,7 @@ export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter return receipt; } - private projectLiveSystem(operation: ExecutionSemanticOperation): readonly ExecutionLivePublicationReceipt[] | undefined { + private projectLiveSystem(operation: ExecutionSemanticOperation): readonly ExecutionLivePublicationIntent[] | undefined { const mutation = operation.mutation; if (mutation.kind !== "live-event" || mutation.systemIntents === undefined) return undefined; const publication = liveEventPublication(operation); @@ -493,7 +495,7 @@ export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter const result = this.systemProjection.projectBoundSystem( operation.execution, publication.event, mutation.systemIntents, publication.after, ); - return [{ publicationId: `${operation.operationId}:0`, after: result.after, event: result.event }]; + return [{ after: result.after, event: result.event }]; } private applyLiveFeatureEffects( @@ -597,6 +599,8 @@ export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter const current = this.requireNextHead(operation); if (current.providerId !== mutation.input.providerId) throw new SemanticConflict(); const receipt = committed(operation, batch.durableSequence); + const status = mutation.outcome === "cancelled" ? "interrupted" : mutation.outcome; + if (!this.threads.updateStatus(operation.execution.threadId, status)) throw new SemanticConflict(); this.storeHead({ ...current, ordinal: operation.ordinal, durableRevision: receipt.durableRevision, terminal: true }); this.storeReceipt(operation, hash, receipt); } @@ -623,8 +627,7 @@ export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter const receipt = committed(operation, head.durableRevision); this.storeHead({ ...head, ordinal: operation.ordinal, assignedMessageId: mutation.input.assistant.messageId ?? head.assignedMessageId ?? null }); - this.storeReceipt(operation, hash, receipt); - return receipt; + return this.storeReceipt(operation, hash, receipt); })(); } @@ -634,8 +637,7 @@ export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter const next = this.applyControlMutation(head, operation); const receipt = committed(operation, head.durableRevision); this.storeHead({ ...next, ordinal: operation.ordinal }); - this.storeReceipt(operation, hash, receipt); - return receipt; + return this.storeReceipt(operation, hash, receipt); })(); } @@ -752,8 +754,15 @@ export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter } } - private storeReceipt(operation: ExecutionSemanticOperation, hash: string, receipt: ExecutionWriteReceipt): void { - const receiptJson = JSON.stringify({ ...receipt, publicationVersion: 1 }); + private storeReceipt( + operation: ExecutionSemanticOperation, + hash: string, + receipt: Extract, + publicationOverride?: readonly ExecutionLivePublicationIntent[], + ): Extract { + const publication = this.numberLivePublication(operation, publicationOverride); + const storedReceipt = publication ? { ...receipt, livePublication: publication } : receipt; + const receiptJson = JSON.stringify({ ...storedReceipt, publicationVersion: 1 }); if (operation.livePublication && Buffer.byteLength(receiptJson, "utf8") > MAX_LIVE_RECEIPT_BYTES) { throw new SemanticConflict(); } @@ -762,7 +771,7 @@ export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter "semantic:finish", receiptJson, operation.execution.executionId, operation.operationId, PENDING_FINISH_KIND, hash, ); if (updated.changes !== 1) throw new SemanticConflict(); - return; + return storedReceipt; } this.insertOperation.run( operation.execution.executionId, @@ -771,6 +780,21 @@ export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter hash, receiptJson, ); + return storedReceipt; + } + + private numberLivePublication( + operation: ExecutionSemanticOperation, + publicationOverride?: readonly ExecutionLivePublicationIntent[], + ): readonly ExecutionLivePublicationReceipt[] | undefined { + const intents = publicationOverride ?? livePublicationFor(operation); + if (!intents) return undefined; + const row = livePublicationHeadSchema.safeParse( + this.advanceLivePublication.get(operation.execution.threadId, intents.length), + ); + if (!row.success) throw new SemanticConflict(); + const first = row.data.last_sequence - intents.length + 1; + return intents.map((intent, index) => ({ ...intent, publicationId: String(first + index) })); } private replay(operation: ExecutionSemanticOperation, hash: string, stored: StoredOperation): ExecutionWriteReceipt { @@ -785,7 +809,8 @@ export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter throw new Error("Live publication receipt exceeds its size limit"); } const receipt = storedReceiptSchema.parse(JSON.parse(stored.receipt_json)); - if (!this.matchesLivePublicationReceipt(operation, receipt.livePublication)) { + if (!validReceiptPublicationIds(receipt.livePublication) + || !this.matchesLivePublicationReceipt(operation, receipt.livePublication)) { throw new Error("Live publication receipt does not match its operation"); } return receipt.operationId === operation.operationId && Number.isSafeInteger(receipt.durableRevision) @@ -801,7 +826,8 @@ export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter if (mutation.kind === "live-event" && mutation.systemIntents?.[0]?.kind === "system-notice") { return this.matchesGeneratedNoticeReceipt(operation, actual); } - return fingerprint(actual ?? null) === fingerprint(livePublicationFor(operation) ?? null); + return fingerprint(actual?.map(({ after, event }) => ({ after, event })) ?? null) + === fingerprint(livePublicationFor(operation) ?? null); } private matchesGeneratedNoticeReceipt( @@ -812,7 +838,8 @@ export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter const published = actual?.[0]; if (actual?.length !== 1 || expected?.event.type !== "system" || expected.event.messageId || !isGeneratedNoticePublication(published)) return false; - return fingerprint({ ...published, event: { ...published.event, messageId: undefined } }) === fingerprint(expected); + return fingerprint({ after: published.after, event: { ...published.event, messageId: undefined } }) + === fingerprint(expected); } private publishStoredEvents(executionId: string, hash: string): void { @@ -895,6 +922,7 @@ function validLivePublicationShape( mutationKind: ExecutionSemanticOperation["mutation"]["kind"], ): boolean { return Array.isArray(publication) && publication.length > 0 && publication.length <= MAX_LIVE_PUBLICATION_EVENTS + && publication.every((intent) => intent.event.publicationId === undefined) && (mutationKind === "begin" || mutationKind === "append-events" || mutationKind === "finish" || mutationKind === "live-event" && publication.length === 1); } @@ -1192,7 +1220,7 @@ function committed( providerCommit?: ExecutionProviderCommitReceipt, providerEvents?: readonly ProjectedCommittedProviderEvent[], assistantTextCheckpoint?: ParentAssistantTextCheckpointResult, - livePublication: readonly ExecutionLivePublicationReceipt[] | undefined = livePublicationFor(operation), + livePublication?: readonly ExecutionLivePublicationReceipt[], planQuestions?: ExecutionPlanQuestionsReceipt, planOutput?: ReturnType, ): Extract { @@ -1207,9 +1235,9 @@ function committed( }; } -function livePublicationFor(operation: ExecutionSemanticOperation): readonly ExecutionLivePublicationReceipt[] | undefined { - return operation.livePublication?.map((intent, index) => ({ - publicationId: `${operation.operationId}:${index}`, after: intent.after, +function livePublicationFor(operation: ExecutionSemanticOperation): readonly ExecutionLivePublicationIntent[] | undefined { + return operation.livePublication?.map((intent) => ({ + after: intent.after, event: AgentEventSchema().parse(intent.event), })); } @@ -1221,6 +1249,13 @@ function isGeneratedNoticePublication( && z.string().uuid().safeParse(receipt.event.messageId).success; } +function validReceiptPublicationIds(publication: readonly ExecutionLivePublicationReceipt[] | undefined): boolean { + if (!publication) return true; + const first = Number(publication[0]?.publicationId); + return Number.isSafeInteger(first) && first > 0 + && publication.every((entry, index) => entry.publicationId === String(first + index)); +} + function conflict(operation: ExecutionSemanticOperation): Extract { return { kind: "conflict", operationId: operation.operationId }; } diff --git a/apps/server/src/features/agents/canonical/execution-live-publication-release.ts b/apps/server/src/features/agents/canonical/execution-live-publication-release.ts index 84831c86e..a845b0ca2 100644 --- a/apps/server/src/features/agents/canonical/execution-live-publication-release.ts +++ b/apps/server/src/features/agents/canonical/execution-live-publication-release.ts @@ -5,7 +5,7 @@ import type { ExecutionLivePublicationReceipt, ExecutionSemanticOperation, ExecutionWriteReceipt, } from "../execution/execution-worker-handler.js"; -const MAX_TRACKED_PUBLICATIONS = 8_192; +const MAX_RECENT_PUBLICATIONS = 8_192; const MAX_RECEIPT_EVENTS = 64; const MAX_RECEIPT_BYTES = 512 * 1024; @@ -15,7 +15,7 @@ export interface LiveAgentEventPublisher { publish(event: AgentEvent): void; } -/** Releases committed live events once per host lifetime, even when a writer reply is replayed. */ +/** Releases committed live events with stable identities for transport replay. */ export class ExecutionLivePublicationRelease { private readonly published = new Set(); @@ -26,14 +26,15 @@ export class ExecutionLivePublicationRelease { if (receipt.kind !== "committed" || !receipt.livePublication) return; const events = this.validate(operation, receipt); if (!this.publisher.isBound()) throw new Error("Live AgentEvent publisher is not bound"); - const newCount = events.filter((entry) => !this.published.has(entry.key)).length; - if (this.published.size + newCount > MAX_TRACKED_PUBLICATIONS) { - throw new Error("Live AgentEvent publication tracking is full"); - } for (const entry of events) { if (this.published.has(entry.key)) continue; this.publisher.publish(entry.event); + // Recent suppression saves repeat host work. The durable wire identity handles older replays. + this.published.delete(entry.key); this.published.add(entry.key); + if (this.published.size > MAX_RECENT_PUBLICATIONS) { + this.published.delete(this.published.values().next().value as string); + } } } @@ -44,7 +45,8 @@ export class ExecutionLivePublicationRelease { const entries = receipt.livePublication ?? []; if (receipt.operationId !== operation.operationId || !operation.livePublication || entries.length !== operation.livePublication.length || entries.length > MAX_RECEIPT_EVENTS - || Buffer.byteLength(JSON.stringify(entries), "utf8") > MAX_RECEIPT_BYTES) { + || Buffer.byteLength(JSON.stringify(entries), "utf8") > MAX_RECEIPT_BYTES + || !contiguousPublicationIds(entries)) { throw new Error("Live AgentEvent publication receipt is invalid"); } return entries.map((entry, index) => this.validateEntry(operation, entry, index)); @@ -52,7 +54,7 @@ export class ExecutionLivePublicationRelease { private validateEntry(operation: ExecutionSemanticOperation, entry: ExecutionLivePublicationReceipt, index: number) { const expected = operation.livePublication?.[index]; - if (!expected || !samePublicationHeader(operation, entry, expected.after, index)) { + if (!expected || !samePublicationHeader(operation, entry.after, expected.after)) { throw new Error("Live AgentEvent publication receipt is invalid"); } const parsed = AgentEventSchema().safeParse(entry.event); @@ -60,21 +62,35 @@ export class ExecutionLivePublicationRelease { if (!parsed.success || !original.success) throw new Error("Live AgentEvent publication receipt is invalid"); if (parsed.data.threadId !== operation.execution.threadId || parsed.data.turnExecutionId !== operation.execution.executionId - || !NodeUtil.isDeepStrictEqual(parsed.data, original.data)) { + || !samePublishedEvent(operation, parsed.data, original.data)) { throw new Error("Live AgentEvent publication receipt is invalid"); } - return { key: JSON.stringify([operation.execution.executionId, entry.publicationId]), event: parsed.data }; + return { key: JSON.stringify([operation.execution.executionId, entry.publicationId]), + event: { ...parsed.data, publicationId: entry.publicationId } }; } } function samePublicationHeader( operation: ExecutionSemanticOperation, - entry: ExecutionLivePublicationReceipt, + after: "writer" | "terminal", expectedBarrier: "writer" | "terminal", - index: number, ): boolean { - return entry.publicationId === `${operation.operationId}:${index}` - && entry.after === expectedBarrier && entry.after === barrierFor(operation.mutation.kind); + return after === expectedBarrier && after === barrierFor(operation.mutation.kind); +} + +function contiguousPublicationIds(entries: readonly ExecutionLivePublicationReceipt[]): boolean { + const first = Number(entries[0]?.publicationId); + return Number.isSafeInteger(first) && first > 0 + && entries.every((entry, index) => entry.publicationId === String(first + index)); +} + +function samePublishedEvent(operation: ExecutionSemanticOperation, actual: AgentEvent, expected: AgentEvent): boolean { + if (NodeUtil.isDeepStrictEqual(actual, expected)) return true; + if (operation.mutation.kind !== "live-event" || operation.mutation.systemIntents?.[0]?.kind !== "system-notice" + || actual.type !== "system" || expected.type !== "system" || expected.messageId + || !actual.messageId) return false; + const { messageId: _generatedId, ...withoutGeneratedId } = actual; + return NodeUtil.isDeepStrictEqual(withoutGeneratedId, expected); } function barrierFor(kind: ExecutionSemanticOperation["mutation"]["kind"]): "writer" | "terminal" | null { diff --git a/apps/server/src/features/agents/events/__tests__/provider-event-publication.test.ts b/apps/server/src/features/agents/events/__tests__/provider-event-publication.test.ts index 6a9107d97..087b3a7c7 100644 --- a/apps/server/src/features/agents/events/__tests__/provider-event-publication.test.ts +++ b/apps/server/src/features/agents/events/__tests__/provider-event-publication.test.ts @@ -49,6 +49,25 @@ describe("provider event publication ownership", () => { }); }); + it("publishes a committed terminal replay without changing a newer thread status", () => { + const deps = buildPublicationDeps(); + const event: AgentEvent = { + type: AgentEventType.TurnComplete, + threadId: "parent-thread", + turnExecutionId: "00000000-0000-4000-8000-000000000001", + publicationId: "1", + reason: "completed", + costUsd: null, + tokensIn: 1, + tokensOut: 1, + }; + + expect(publishParentProviderEvent(event, event, deps)).toBe(true); + expect(deps.publishAgentEvent).toHaveBeenCalledWith(event); + expect(deps.updateThreadStatus).not.toHaveBeenCalled(); + expect(deps.publishThreadStatus).not.toHaveBeenCalled(); + }); + it("publishes a parent error and updates its status", () => { const deps = buildPublicationDeps(); const parentEvent: AgentEvent = { diff --git a/apps/server/src/features/agents/events/provider-event-publication.ts b/apps/server/src/features/agents/events/provider-event-publication.ts index 82eb6d253..036b6e271 100644 --- a/apps/server/src/features/agents/events/provider-event-publication.ts +++ b/apps/server/src/features/agents/events/provider-event-publication.ts @@ -16,6 +16,9 @@ export function publishParentProviderEvent( if (!shouldPublishParentEvent(event)) return false; deps.publishAgentEvent(enrichedEvent); + // A semantic writer committed this status before releasing its terminal event. + // A late replay must not overwrite the status of a newer execution. + if (event.publicationId !== undefined) return true; if (event.type === "turnComplete") { deps.updateThreadStatus(event.threadId, "completed"); deps.publishThreadStatus({ threadId: event.threadId, status: "completed" }); diff --git a/apps/server/src/features/agents/execution/__tests__/execution-worker-loss-coordinator.test.ts b/apps/server/src/features/agents/execution/__tests__/execution-worker-loss-coordinator.test.ts index 72b1b4f86..eb3de49e6 100644 --- a/apps/server/src/features/agents/execution/__tests__/execution-worker-loss-coordinator.test.ts +++ b/apps/server/src/features/agents/execution/__tests__/execution-worker-loss-coordinator.test.ts @@ -146,11 +146,11 @@ describe("ExecutionWorkerLossCoordinator with a file-backed writer", () => { livePublication: [{ after: "writer", event: started }], }); expect(begin.livePublication).toEqual([{ - publicationId: `${claim.lease.leaseId}:1:0`, after: "writer", event: started, + publicationId: "1", after: "writer", event: started, }]); expect(db.prepare("SELECT receipt_json FROM canonical_writer_operation_receipts WHERE execution_id = ? AND operation_id = ?") .get(execution.executionId, begin.operationId)) - .toMatchObject({ receipt_json: expect.stringContaining(`${claim.lease.leaseId}:1:0`) }); + .toMatchObject({ receipt_json: expect.stringContaining('"publicationId":"1"') }); const tool = { kind: "toolCall" as const, sequence: 2, sortOrder: 0, record: { id: "transport-tool", message_id: "", parent_tool_call_id: null, @@ -176,12 +176,12 @@ describe("ExecutionWorkerLossCoordinator with a file-backed writer", () => { } }, }], livePublication: [{ after: "writer", event: toolUse }] }); expect(event.livePublication).toEqual([{ - publicationId: `${claim.lease.leaseId}:3:0`, after: "writer", event: toolUse, + publicationId: "2", after: "writer", event: toolUse, }]); expect(published).toContain(`${execution.executionId}:transport-tool`); expect(db.prepare("SELECT receipt_json FROM canonical_writer_operation_receipts WHERE execution_id = ? AND operation_id = ?") .get(execution.executionId, event.operationId)) - .toMatchObject({ receipt_json: expect.stringContaining(`${claim.lease.leaseId}:3:0`) }); + .toMatchObject({ receipt_json: expect.stringContaining('"publicationId":"2"') }); await send({ kind: "provider-outcome", outcome: "completed" }); await send({ kind: "stage-terminal", input: { @@ -196,7 +196,7 @@ describe("ExecutionWorkerLossCoordinator with a file-backed writer", () => { projection: { kind: "writer-staged" }, }, livePublication: [{ after: "terminal", event: ended }] }); expect(finish.livePublication).toEqual([{ - publicationId: `${claim.lease.leaseId}:6:0`, after: "terminal", event: ended, + publicationId: "3", after: "terminal", event: ended, }]); expect(db.prepare("SELECT terminal_outcome FROM canonical_agent_ingest_checkpoints WHERE execution_id = ?") .get(execution.executionId)).toEqual({ terminal_outcome: "completed" }); diff --git a/apps/server/src/features/agents/orchestration/__tests__/start-agent-orchestration.test.ts b/apps/server/src/features/agents/orchestration/__tests__/start-agent-orchestration.test.ts index dc1cd36f8..90740477b 100644 --- a/apps/server/src/features/agents/orchestration/__tests__/start-agent-orchestration.test.ts +++ b/apps/server/src/features/agents/orchestration/__tests__/start-agent-orchestration.test.ts @@ -115,4 +115,21 @@ describe("agent orchestration", () => { }); expect(orchestration.pullRequestCompletionEffect.schedule).toHaveBeenCalledWith("thread-1"); }); + + it("releases a committed terminal receipt even when the legacy runtime suppression is set", () => { + const orchestration = buildOrchestration(); + orchestration.runtime.shouldSuppressTurnComplete.mockReturnValue(true); + const legacy: AgentEvent = { type: AgentEventType.TurnComplete, threadId: "thread-1", + reason: "completed", costUsd: null, tokensIn: 1, tokensOut: 1 }; + orchestration.publish(legacy); + expect(orchestration.publishedEvents).toEqual([]); + + const committed: AgentEvent = { ...legacy, + turnExecutionId: "00000000-0000-4000-8000-000000000001", publicationId: "1" }; + orchestration.publish(committed); + expect(orchestration.publishedEvents).toEqual([committed]); + expect(orchestration.threadRepo.updateStatus).not.toHaveBeenCalled(); + expect(orchestration.publishThreadStatus).not.toHaveBeenCalled(); + expect(orchestration.pullRequestCompletionEffect.schedule).toHaveBeenCalledWith("thread-1"); + }); }); diff --git a/apps/server/src/features/agents/orchestration/agent-event-publication-service.ts b/apps/server/src/features/agents/orchestration/agent-event-publication-service.ts index ed3444991..e7c74be7b 100644 --- a/apps/server/src/features/agents/orchestration/agent-event-publication-service.ts +++ b/apps/server/src/features/agents/orchestration/agent-event-publication-service.ts @@ -62,6 +62,8 @@ export class AgentEventPublicationService { } private shouldSuppress(event: AgentEvent): boolean { + // A stable publication already passed the writer's semantic and terminal barriers. + if (event.publicationId) return false; if (event.type === AgentEventType.Ended) return this.dependencies.runtime.shouldSuppressTurnEnded(event.threadId); if (event.type === AgentEventType.TurnComplete) return this.dependencies.runtime.shouldSuppressTurnComplete(event.threadId); return event.type === AgentEventType.Error diff --git a/apps/server/src/runtime/persistence/sqlite/schema.ts b/apps/server/src/runtime/persistence/sqlite/schema.ts index 31e59b947..72b6cd142 100644 --- a/apps/server/src/runtime/persistence/sqlite/schema.ts +++ b/apps/server/src/runtime/persistence/sqlite/schema.ts @@ -756,6 +756,16 @@ export const canonicalWriterOperationReceipts = sqliteTable( (table) => [primaryKey({ columns: [table.executionId, table.operationId] })], ); +/** Monotonic live publication identity survives execution changes and receipt pruning. */ +export const canonicalWriterLivePublicationHeads = sqliteTable( + "canonical_writer_live_publication_heads", + { + threadId: text("thread_id").primaryKey().notNull() + .references(() => threads.id, { onDelete: "cascade" }), + lastSequence: integer("last_sequence").notNull(), + }, +); + /** Durable accepted and committed progress for one canonical execution. */ export const canonicalAgentIngestCheckpoints = sqliteTable( "canonical_agent_ingest_checkpoints", diff --git a/apps/web/src/__tests__/agent-event-branches.test.ts b/apps/web/src/__tests__/agent-event-branches.test.ts index dfb45def1..792e1091f 100644 --- a/apps/web/src/__tests__/agent-event-branches.test.ts +++ b/apps/web/src/__tests__/agent-event-branches.test.ts @@ -515,6 +515,76 @@ describe("handleAgentEvent branches", () => { .toBe("00000000-0000-4000-8000-000000000002"); }); + it("does not reapply a stable publication after thread rehydration and a server restart", () => { + const threadId = "thread-stable-publication"; + const executionId = "00000000-0000-4000-8000-000000000001"; + const publicationId = "2"; + const firstEpoch = "00000000-0000-4000-8000-000000000002"; + const nextEpoch = "00000000-0000-4000-8000-000000000003"; + const key = `mcode-agent-publication-v2:${threadId}`; + sessionStorage.removeItem(key); + try { + resetThreadStoreForTests({ currentThreadId: threadId, + runningThreadIds: new Set([threadId]), + records: new Map([[threadId, { ...createEmptyThreadRecord(), runtimePhase: "running", + turnExecutionId: executionId }]]), + }); + const original: Extract = { type: "toolUse", threadId, turnExecutionId: executionId, + publicationId, epoch: firstEpoch, sequence: 8, toolCallId: "stable-call", + toolName: "Read", toolInput: { path: "/original" } }; + useThreadStore.getState().handleAgentEvent(original); + const persistedCalls = readThreadField(threadId, (record) => record.toolCalls); + expect(persistedCalls).toHaveLength(1); + + resetThreadStoreForTests({ currentThreadId: threadId, + runningThreadIds: new Set([threadId]), + records: new Map([[threadId, { ...createEmptyThreadRecord(), runtimePhase: "running", + turnExecutionId: executionId, toolCalls: persistedCalls }]]), + }); + useThreadStore.getState().handleAgentEvent({ ...original, epoch: nextEpoch, + sequence: 1, toolInput: { path: "/replayed" } }); + expect(readThreadField(threadId, (record) => record.toolCalls)).toEqual(persistedCalls); + expect(readThreadField(threadId, (record) => record.lastAgentEventEpoch)).toBe(nextEpoch); + useThreadStore.getState().handleAgentEvent({ ...original, publicationId: undefined, + epoch: nextEpoch, sequence: 2, toolCallId: "legacy-call" }); + expect(readThreadField(threadId, (record) => record.toolCalls)).toHaveLength(2); + } finally { + sessionStorage.removeItem(key); + } + }); + + it("does not repeat terminal projection after a stable completion is replayed", () => { + const threadId = "thread-stable-terminal"; + const executionId = "00000000-0000-4000-8000-000000000011"; + const key = `mcode-agent-publication-v2:${threadId}`; + sessionStorage.removeItem(key); + try { + resetThreadStoreForTests({ currentThreadId: threadId, + runningThreadIds: new Set([threadId]), + records: new Map([[threadId, { ...createEmptyThreadRecord(), runtimePhase: "running", + turnExecutionId: executionId, streaming: "completed answer" }]]), + }); + const completion = { type: "turnComplete", threadId, turnExecutionId: executionId, + publicationId: "3", sequence: 3, + epoch: "00000000-0000-4000-8000-000000000012", reason: "end_turn", + costUsd: null, tokensIn: 0, tokensOut: 0 } as AgentEvent; + useThreadStore.getState().handleAgentEvent(completion); + const completed = readThreadField(threadId, (record) => record); + const messageCount = completed.messages.length; + expect(messageCount).toBeGreaterThan(0); + + resetThreadStoreForTests({ currentThreadId: threadId, + runningThreadIds: new Set(), records: new Map([[threadId, completed]]), + }); + useThreadStore.getState().handleAgentEvent({ ...completion, + epoch: "00000000-0000-4000-8000-000000000013", sequence: 1 }); + expect(readThreadField(threadId, (record) => record.messages)).toHaveLength(messageCount); + expect(readThreadField(threadId, (record) => record.runtimePhase)).toBe(completed.runtimePhase); + } finally { + sessionStorage.removeItem(key); + } + }); + it("retains hook progress for an inactive thread", () => { const backgroundThreadId = "thread-background-hook"; resetThreadStoreForTests({ diff --git a/apps/web/src/stores/thread-store/__tests__/stable-agent-event-publications.test.ts b/apps/web/src/stores/thread-store/__tests__/stable-agent-event-publications.test.ts new file mode 100644 index 000000000..f8282b488 --- /dev/null +++ b/apps/web/src/stores/thread-store/__tests__/stable-agent-event-publications.test.ts @@ -0,0 +1,73 @@ +import type { AgentEvent } from "@mcode/contracts"; +import { describe, expect, it } from "vitest"; +import { StableAgentEventPublications } from "../stable-agent-event-publications"; + +const EXECUTION_ID = "00000000-0000-4000-8000-000000000001"; +const NEXT_EXECUTION_ID = "00000000-0000-4000-8000-000000000002"; + +function event(sequence: number, executionId = EXECUTION_ID): AgentEvent { + return { type: "toolUse", threadId: "thread-publication", turnExecutionId: executionId, + publicationId: String(sequence), toolCallId: `call-${sequence}`, + toolName: "Read", toolInput: {} }; +} + +function storage(): Pick { + const items = new Map(); + return { + getItem: (key) => items.get(key) ?? null, + setItem: (key, value) => { items.set(key, value); }, + }; +} + +describe("stable AgentEvent publications", () => { + it("rejects a publish-then-crash replay after recreating the client and changing server epoch", () => { + const shared = storage(); + const firstClient = new StableAgentEventPublications(() => shared); + const published = event(7); + expect(firstClient.accept({ ...published, epoch: "00000000-0000-4000-8000-000000000004" })) + .toBe(true); + const reloadedClient = new StableAgentEventPublications(() => shared); + expect(reloadedClient.accept({ ...published, epoch: "00000000-0000-4000-8000-000000000005" })) + .toBe(false); + expect(reloadedClient.accept(event(8))).toBe(true); + expect(reloadedClient.accept(event(7))).toBe(false); + }); + + it("rejects an arbitrarily late prior-execution replay after an all-day stream", () => { + const shared = storage(); + const cursor = new StableAgentEventPublications(() => shared); + for (let sequence = 1; sequence <= 8_193; sequence++) { + expect(cursor.accept(event(sequence))).toBe(true); + } + expect(cursor.accept(event(8_194, NEXT_EXECUTION_ID))).toBe(true); + expect(cursor.accept(event(1))).toBe(false); + expect(cursor.accept(event(8_193))).toBe(false); + }); + + it("lets two independent tabs each apply the same publication", () => { + const firstStore = storage(); + const secondStore = storage(); + const firstTab = new StableAgentEventPublications(() => firstStore); + const secondTab = new StableAgentEventPublications(() => secondStore); + expect(firstTab.accept(event(1))).toBe(true); + expect(secondTab.accept(event(1))).toBe(true); + expect(firstTab.accept(event(1))).toBe(false); + expect(secondTab.accept(event(1))).toBe(false); + }); + + it("fails closed if its durable cursor cannot be stored", () => { + const cursor = new StableAgentEventPublications(() => ({ + getItem: () => null, + setItem: () => { throw new Error("quota"); }, + })); + expect(cursor.accept(event(1))).toBe(false); + }); + + it("fails closed on an oversized stored cursor", () => { + const cursor = new StableAgentEventPublications(() => ({ + getItem: () => "1".repeat(17), + setItem: () => { throw new Error("unexpected write"); }, + })); + expect(cursor.accept(event(1))).toBe(false); + }); +}); diff --git a/apps/web/src/stores/thread-store/agent-event-preflight.ts b/apps/web/src/stores/thread-store/agent-event-preflight.ts index 5bd7a6ce3..f5df6ca2c 100644 --- a/apps/web/src/stores/thread-store/agent-event-preflight.ts +++ b/apps/web/src/stores/thread-store/agent-event-preflight.ts @@ -23,6 +23,7 @@ export type AgentEventHandlerTable = { /** Narrow store operations needed to validate and sequence one agent event. */ export interface AgentEventPreflightContext { clearApiRetry: (threadId: string) => void; + acceptPublication?: (event: AgentEvent) => boolean; flushPendingTextDeltas: () => void; getCurrentThreadId: () => string | null; getRecord: (threadId: string) => ThreadRecord; @@ -187,6 +188,7 @@ export function prepareAgentEvent( const incomingExecutionId = eventExecutionId(event); if (!acceptsExecution(event, runtimeRecord, incomingExecutionId)) return null; if (!acceptsSequence(context, event)) return null; + if (context.acceptPublication && !context.acceptPublication(event)) return null; const currentThreadId = context.getCurrentThreadId(); const activeThread = isActiveThread( diff --git a/apps/web/src/stores/thread-store/stable-agent-event-publications.ts b/apps/web/src/stores/thread-store/stable-agent-event-publications.ts new file mode 100644 index 000000000..2a68e8386 --- /dev/null +++ b/apps/web/src/stores/thread-store/stable-agent-event-publications.ts @@ -0,0 +1,44 @@ +import type { AgentEvent } from "@mcode/contracts"; + +const STORAGE_PREFIX = "mcode-agent-publication-v2:"; +const MAX_SEQUENCE_BYTES = 16; + +type PublicationStorage = Pick; + +function publicationSequence(value: string): number | null { + if (value.length > MAX_SEQUENCE_BYTES || !/^[1-9]\d*$/.test(value)) return null; + const sequence = Number(value); + return Number.isSafeInteger(sequence) ? sequence : null; +} + +/** Retains one per-thread cursor across server epochs, executions, and browser reloads. */ +export class StableAgentEventPublications { + constructor(private readonly storage: () => PublicationStorage | undefined) {} + + /** Reserve a publication before applying its UI effects. A storage failure fails closed. */ + accept(event: AgentEvent): boolean { + if (!event.publicationId) return true; + if (!event.turnExecutionId || publicationSequence(event.publicationId) === null) return false; + try { + return this.reserve(event.threadId, event.publicationId); + } catch { + return false; + } + } + + private reserve(threadId: string, publicationId: string): boolean { + const store = this.storage(); + if (!store) return false; + const key = `${STORAGE_PREFIX}${threadId}`; + const stored = store.getItem(key); + const previous = stored === null ? 0 : publicationSequence(stored); + if (previous === null || Number(publicationId) <= previous) return false; + store.setItem(key, publicationId); + return true; + } +} + +/** Browser-tab cursor with constant retained bytes per thread. */ +export const stableAgentEventPublications = new StableAgentEventPublications( + () => typeof sessionStorage === "undefined" ? undefined : sessionStorage, +); diff --git a/apps/web/src/stores/threadStore.ts b/apps/web/src/stores/threadStore.ts index 52d64b666..4267ffc4a 100644 --- a/apps/web/src/stores/threadStore.ts +++ b/apps/web/src/stores/threadStore.ts @@ -78,6 +78,7 @@ import { prepareAgentEvent, type AgentEventHandlerTable, } from "./thread-store/agent-event-preflight"; +import { stableAgentEventPublications } from "./thread-store/stable-agent-event-publications"; import { hydrateRunningThreads as hydrateRunningThreadRecords, transferThreadRuntime as transferOptimisticThreadRuntime, @@ -3774,6 +3775,7 @@ export const useThreadStore = create((zustandSet, get) => { handleAgentEvent: (event) => { if (!hasAgentEventHandler(agentEventHandlers, event)) return; const runtime = prepareAgentEvent({ + acceptPublication: (incoming) => stableAgentEventPublications.accept(incoming), clearApiRetry: (id) => patchRec(id, { apiRetry: undefined }), flushPendingTextDeltas, getCurrentThreadId: () => get().currentThreadId, diff --git a/packages/contracts/src/events/__tests__/agent-event.test.ts b/packages/contracts/src/events/__tests__/agent-event.test.ts index 7a9eaf5f8..7df55d6d4 100644 --- a/packages/contracts/src/events/__tests__/agent-event.test.ts +++ b/packages/contracts/src/events/__tests__/agent-event.test.ts @@ -2,6 +2,16 @@ import { describe, it, expect } from "vitest"; import { AgentEventSchema, AgentEventType } from "../agent-event.js"; describe("AgentEvent provider_unavailable", () => { + it("preserves a bounded durable publication identity", () => { + const event = { type: "turnStarted", threadId: "t-1", + turnExecutionId: "00000000-0000-4000-8000-000000000001", + publicationId: "1" }; + expect(AgentEventSchema().parse(event)).toEqual(event); + expect(AgentEventSchema().safeParse({ ...event, publicationId: "x" }).success).toBe(false); + expect(AgentEventSchema().safeParse({ ...event, publicationId: "1".repeat(17) }).success) + .toBe(false); + }); + it("parses a disabled-provider event", () => { const parsed = AgentEventSchema().parse({ type: "providerUnavailable", diff --git a/packages/contracts/src/events/agent-event.ts b/packages/contracts/src/events/agent-event.ts index 89578645e..bd2a1829c 100644 --- a/packages/contracts/src/events/agent-event.ts +++ b/packages/contracts/src/events/agent-event.ts @@ -384,6 +384,8 @@ const AgentEventSequenceSchema = z.object({ epoch: AgentEventEpochSchema.optional(), /** Mcode-owned execution identity for the turn that produced this event. */ turnExecutionId: z.string().uuid().optional(), + /** Durable semantic receipt identity; survives a server event-epoch change. */ + publicationId: z.string().regex(/^[1-9]\d*$/).max(16).optional(), }); /** Validated agent event payload, including an optional server ordering sequence. */ From 15fa7030ad09b5c2d083efeb2e0670b68277220c Mon Sep 17 00:00:00 2001 From: "(CJ) Chukwudi Nwobodo" <142016413+chuks-qua@users.noreply.github.com> Date: Fri, 25 Sep 2026 11:01:00 +0100 Subject: [PATCH 109/136] feat(server): bind provider batches to execution attempts --- .../__tests__/execution-mailbox-owner.test.ts | 52 +++++++ .../execution/execution-mailbox-owner.ts | 10 +- .../execution-provider-event-ownership.ts | 143 ++++++++++++++++++ 3 files changed, 204 insertions(+), 1 deletion(-) create mode 100644 apps/server/src/features/agents/execution/execution-provider-event-ownership.ts diff --git a/apps/server/src/features/agents/execution/__tests__/execution-mailbox-owner.test.ts b/apps/server/src/features/agents/execution/__tests__/execution-mailbox-owner.test.ts index 215e9cc82..41988315e 100644 --- a/apps/server/src/features/agents/execution/__tests__/execution-mailbox-owner.test.ts +++ b/apps/server/src/features/agents/execution/__tests__/execution-mailbox-owner.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it } from "vitest"; import { ExecutionMailboxOwner } from "../execution-mailbox-owner.js"; +import { ExecutionProviderEventOwnership } from "../execution-provider-event-ownership.js"; import { ExecutionMailboxScheduler, type ExecutionMailboxCommand } from "../execution-mailbox-scheduler.js"; import type { ExecutionIdentity, ExecutionWorkerPort, ExecutionWorkerReply, ExecutionWorkerRequest } from "../execution-mailbox-protocol.js"; import type { ExecutionWorkCommand, ExecutionWorkerResult } from "../execution-worker-handler.js"; @@ -70,3 +71,54 @@ describe("ExecutionMailboxOwner", () => { scheduler.shutdown(); }); }); + +describe("ExecutionProviderEventOwnership", () => { + it("submits an exact attempt to its mailbox and rejects retired or superseded attempts", async () => { + const worker = new WorkerPort(); + const scheduler = new ExecutionMailboxScheduler({ + workerCount: 1, + limits: { + maxPending: 8, maxPendingBytes: 128_000, reservedControl: 2, reservedControlBytes: 16_000, + maxPerExecutionPending: 8, maxPerExecutionBytes: 128_000, + reservedPerExecutionControl: 2, reservedPerExecutionControlBytes: 16_000, + }, + createWorker: () => worker, + onWorkerLost: () => {}, + }); + const owner = new ExecutionMailboxOwner(scheduler); + const ownership = new ExecutionProviderEventOwnership(owner); + expect(ownership.resolve(execution.executionId)).toEqual({ kind: "rejected" }); + const start = owner.start({ execution, ownerEpoch: 1, providerId: "codex", parentTurn }); + await expect(ownership.bind(execution, 1)).rejects.toThrow("no matching execution owner"); + worker.reply(0, { kind: "committed", operationId: `${worker.requests[0]!.lease.leaseId}:1`, durableRevision: 1 }); + await start; + await ownership.bind(execution, 1); + const route = ownership.resolve(execution.executionId); + if (route.kind !== "worker") throw new Error("Expected worker route"); + const batch = { + threadId: execution.threadId, turnId: execution.turnId, executionId: execution.executionId, + batchId: "batch-1", deliveryAttempt: 1, phase: "running", events: [], + }; + const submitted = route.submit(batch); + expect(worker.requests[1]?.command).toMatchObject({ kind: "event", phase: "running", events: [] }); + const nextAttempt = ownership.bind(execution, 2); + expect(ownership.resolve(execution.executionId)).toEqual({ kind: "rejected" }); + await expect(route.submit(batch)).rejects.toThrow("retired execution attempt"); + worker.reply(1, { + kind: "committed", operationId: `${worker.requests[1]!.lease.leaseId}:2`, durableRevision: 1, + providerCommit: { + outcome: "committed", conversationRevision: 1, rosterRevision: 0, + acceptedThrough: 1, durableThrough: 1, eventCount: 0, + }, + }); + await expect(submitted).resolves.toMatchObject({ batchId: "batch-1", + deliveryAttempt: 1, commit: { outcome: "committed" } }); + + await nextAttempt; + expect(ownership.resolve(execution.executionId)).toMatchObject({ kind: "worker", deliveryAttempt: 2 }); + await ownership.retire(execution); + expect(ownership.resolve(execution.executionId)).toEqual({ kind: "rejected" }); + expect(worker.requests).toHaveLength(2); + scheduler.shutdown(); + }); +}); diff --git a/apps/server/src/features/agents/execution/execution-mailbox-owner.ts b/apps/server/src/features/agents/execution/execution-mailbox-owner.ts index 9ad249d86..6c4f456e0 100644 --- a/apps/server/src/features/agents/execution/execution-mailbox-owner.ts +++ b/apps/server/src/features/agents/execution/execution-mailbox-owner.ts @@ -8,6 +8,7 @@ type Scheduler = ExecutionMailboxScheduler>; +} + +interface SwitchingRoute { + readonly kind: "switching"; + readonly execution: ExecutionIdentity; + readonly deliveryAttempt: number; + readonly drained: Promise; +} + +type Route = ActiveRoute | SwitchingRoute; + +/** + * Binds provider batches to the execution mailbox after its durable start. + * Unknown and retired executions fail closed while this worker route is active. + */ +export class ExecutionProviderEventOwnership implements ProviderEventOwnership { + private readonly routes = new Map(); + + constructor(private readonly owner: Pick) {} + + /** Admit one provider attempt only while the exact mailbox still owns the thread. */ + async bind(execution: ExecutionIdentity, deliveryAttempt: number): Promise { + this.requireOwner(execution, deliveryAttempt); + const existing = this.routes.get(execution.executionId); + if (existing?.kind === "switching") { + await existing.drained; + return this.bind(execution, deliveryAttempt); + } + if (existing) this.requireReplacement(existing, execution, deliveryAttempt); + if (existing?.deliveryAttempt === deliveryAttempt) return; + if (existing) await this.drainPrevious(existing, execution, deliveryAttempt); + this.routes.set(execution.executionId, { + kind: "active", execution, deliveryAttempt, pending: new Set(), + }); + } + + private requireOwner(execution: ExecutionIdentity, deliveryAttempt: number): void { + if (!Number.isSafeInteger(deliveryAttempt) || deliveryAttempt < 1 + || !this.owner.isStarted(execution)) { + throw new Error("Provider attempt has no matching execution owner"); + } + } + + private requireReplacement(existing: ActiveRoute, execution: ExecutionIdentity, deliveryAttempt: number): void { + if (!sameExecution(existing.execution, execution) || deliveryAttempt < existing.deliveryAttempt) { + throw new Error("Provider attempt cannot replace its current owner"); + } + } + + private async drainPrevious(existing: ActiveRoute, execution: ExecutionIdentity, deliveryAttempt: number): Promise { + const switching: SwitchingRoute = { + kind: "switching", execution, deliveryAttempt, + drained: Promise.allSettled(existing.pending).then(() => undefined), + }; + this.routes.set(execution.executionId, switching); + await switching.drained; + if (this.routes.get(execution.executionId) !== switching || !this.owner.isStarted(execution)) { + throw new Error("Provider attempt lost its execution owner while draining"); + } + } + + /** Keep the execution rejected after its terminal fence releases ownership. */ + async retire(execution: ExecutionIdentity): Promise { + const current = this.routes.get(execution.executionId); + if (current && sameExecution(current.execution, execution)) { + this.routes.delete(execution.executionId); + if (current.kind === "switching") await current.drained; + else await Promise.allSettled(current.pending); + } + } + + /** Select the exact mailbox, then check again before asynchronous admission. */ + resolve(executionId: string): ProviderEventOwnershipRoute { + const route = this.routes.get(executionId); + if (route?.kind !== "active" || !this.owner.isStarted(route.execution)) { + return { kind: "rejected" }; + } + return { + kind: "worker", + ...route.execution, + deliveryAttempt: route.deliveryAttempt, + submit: (batch) => this.submit(route, batch), + }; + } + + private async submit(route: ActiveRoute, batch: WorkerOwnedProviderEventBatch): Promise { + if (!this.isCurrent(route, batch)) { + throw new Error("Provider batch belongs to a retired execution attempt"); + } + const pending = this.owner.submit(route.execution, { + kind: "event", + phase: batch.phase, + nativeCursor: batch.nativeCursor ?? null, + events: batch.events, + }); + route.pending.add(pending); + let result: Awaited; + try { + result = await pending; + } finally { + route.pending.delete(pending); + } + if (result.kind !== "committed" || !result.providerCommit) { + throw new Error("Execution worker did not commit provider batch"); + } + const commit: ProviderEventCommitReceipt = { + ...result.providerCommit, + outcome: result.providerCommit.outcome === "terminal-outcome-confirmed" + ? "duplicate" : result.providerCommit.outcome, + }; + if (commit.outcome !== "committed" && result.providerEvents?.length) { + throw new Error("Noncommitted provider batch returned projected events"); + } + return { + batchId: batch.batchId, + deliveryAttempt: batch.deliveryAttempt, + commit, + providerEvents: result.providerEvents ?? [], + }; + } + + private isCurrent(route: ActiveRoute, batch: WorkerOwnedProviderEventBatch): boolean { + return this.routes.get(route.execution.executionId) === route + && this.owner.isStarted(route.execution) + && batch.deliveryAttempt === route.deliveryAttempt + && sameExecution(batch, route.execution); + } +} + +function sameExecution(left: ExecutionIdentity | undefined, right: ExecutionIdentity): boolean { + return left?.threadId === right.threadId && left.turnId === right.turnId + && left.executionId === right.executionId; +} From ad4c30d38b09e54cecbaf8296452de607fbc7aa5 Mon Sep 17 00:00:00 2001 From: "(CJ) Chukwudi Nwobodo" <142016413+chuks-qua@users.noreply.github.com> Date: Fri, 25 Sep 2026 15:14:06 +0100 Subject: [PATCH 110/136] feat(server): give Codex turns worker-owned execution --- .../features/multi-surface-journeys.md | 8 + .../application/bootstrap/server-bootstrap.ts | 8 + .../__tests__/ws-server-health.test.ts | 32 +- .../src/application/transport/ws-server.ts | 4 + ...ical-codex-system-error-projection.test.ts | 26 +- ...anonical-execution-semantic-writer.test.ts | 369 ++++++++++- .../canonical-execution-writer-port.test.ts | 29 + .../canonical-parent-turn-write.test.ts | 26 + .../canonical-plan-live-projection.test.ts | 6 + .../canonical-agent-writer.worker.ts | 25 +- ...canonical-codex-system-error-projection.ts | 19 +- .../canonical-execution-semantic-writer.ts | 602 ++++++++++++++++-- .../canonical/canonical-parent-turn-write.ts | 18 +- .../execution-live-publication-release.ts | 43 +- .../execution-plan-question-release.ts | 11 +- .../__tests__/agent-service-container.test.ts | 383 ++++++++++- .../agents/composition/register-agents.ts | 54 ++ .../__tests__/message-repo.test.ts | 19 + .../conversation/persistence/message-repo.ts | 12 + .../codex-live-event-effects.test.ts | 121 ++++ ...xecution-file-evidence-coordinator.test.ts | 131 ++++ .../__tests__/execution-mailbox-owner.test.ts | 26 +- .../execution-mailbox-scheduler.test.ts | 18 + .../execution-worker-file-evidence.test.ts | 137 ++++ .../execution-worker-handler.test.ts | 38 ++ .../execution-worker-loss-coordinator.test.ts | 7 +- .../execution-worker-parent-events.test.ts | 297 +++++++++ .../provider-live-event-effects.test.ts | 90 +++ .../worker-owned-turn-runtime.test.ts | 73 +++ .../execution/codex-live-event-effects.ts | 157 +++++ .../execution/codex-live-event-reducer.ts | 25 +- .../execution-file-evidence-coordinator.ts | 152 +++++ .../execution/execution-mailbox-owner.ts | 25 +- .../execution/execution-mailbox-scheduler.ts | 1 + .../execution-provider-event-ownership.ts | 56 +- .../execution-worker-file-evidence.ts | 156 +++++ .../execution/execution-worker-handler.ts | 446 +++++++++++-- .../execution-worker-loss-coordinator.ts | 7 +- .../agents/execution/execution.worker.ts | 14 +- .../provider-execution-event-state.ts | 149 +++++ .../execution/provider-live-event-effects.ts | 100 +++ .../execution/worker-owned-turn-runtime.ts | 116 ++++ .../orchestration/turn-runtime-controller.ts | 560 +++++++++++++++- .../__tests__/turn-event-pipeline.test.ts | 41 +- .../turn-admission-dispatch-coordinator.ts | 42 +- .../agents/turns/turn-event-pipeline.ts | 11 + .../claude-canonical-event-publisher.test.ts | 18 + .../claude-provider-result-error.test.ts | 63 ++ .../adapters/claude/claude-provider.ts | 55 +- .../provider-catalog-service.test.ts | 47 +- .../provider-catalog-snapshot-repo.ts | 3 +- .../catalog/provider-catalog-service.ts | 22 +- .../canonical-live-event-publisher.ts | 14 +- .../composition/provider-host-ports.ts | 6 +- .../composition/register-providers.ts | 2 + .../__tests__/terminal-cleanup-ledger.test.ts | 23 +- .../cleanup/terminal-cleanup-ledger.ts | 4 +- .../threadStore-turn-persist.test.ts | 32 + apps/web/src/stores/threadStore.ts | 12 +- apps/web/src/transport/ws-events.ts | 13 +- docs/internals/narrative-pipeline.md | 21 +- docs/internals/provider-architecture.md | 41 +- packages/contracts/src/ws/channels.ts | 2 +- .../codex-canonical-event-publisher.test.ts | 188 ++++++ .../codex/codex-provider-first-turn.test.ts | 181 ++++++ .../codex/codex-provider-test-fixture.ts | 4 +- packages/providers/src/factory-types.ts | 9 +- packages/providers/src/index.ts | 1 + .../codex/codex-canonical-event-publisher.ts | 166 +++++ .../src/private/codex/codex-provider.ts | 85 +++ .../cursor-session-continuity.test.ts | 50 +- scripts/perf/seven-thread-live-harness.mjs | 123 +++- .../perf/seven-thread-live-harness.test.mjs | 39 ++ 73 files changed, 5627 insertions(+), 287 deletions(-) create mode 100644 apps/server/src/features/agents/execution/__tests__/codex-live-event-effects.test.ts create mode 100644 apps/server/src/features/agents/execution/__tests__/execution-file-evidence-coordinator.test.ts create mode 100644 apps/server/src/features/agents/execution/__tests__/execution-worker-file-evidence.test.ts create mode 100644 apps/server/src/features/agents/execution/__tests__/execution-worker-parent-events.test.ts create mode 100644 apps/server/src/features/agents/execution/__tests__/provider-live-event-effects.test.ts create mode 100644 apps/server/src/features/agents/execution/__tests__/worker-owned-turn-runtime.test.ts create mode 100644 apps/server/src/features/agents/execution/codex-live-event-effects.ts create mode 100644 apps/server/src/features/agents/execution/execution-file-evidence-coordinator.ts create mode 100644 apps/server/src/features/agents/execution/execution-worker-file-evidence.ts create mode 100644 apps/server/src/features/agents/execution/provider-execution-event-state.ts create mode 100644 apps/server/src/features/agents/execution/provider-live-event-effects.ts create mode 100644 apps/server/src/features/agents/execution/worker-owned-turn-runtime.ts create mode 100644 packages/providers/src/__tests__/codex/codex-canonical-event-publisher.test.ts create mode 100644 packages/providers/src/private/codex/codex-canonical-event-publisher.ts diff --git a/.agents/skills/verify-mcode/references/features/multi-surface-journeys.md b/.agents/skills/verify-mcode/references/features/multi-surface-journeys.md index fdcb8c8d6..1555b2a7b 100644 --- a/.agents/skills/verify-mcode/references/features/multi-surface-journeys.md +++ b/.agents/skills/verify-mcode/references/features/multi-surface-journeys.md @@ -19,6 +19,14 @@ If cleanup is incomplete, use the receipt's exact path with `--cleanup-receipt --confirm-run` before another run. Do not delete tasks by title or age. +For the worker-owned turn change in issue #1760, collect at least ten matched +receipts. Compare server-loop delay, public event throughput, memory, and +`metrics.workerQueue` peaks with the recorded baseline. Require model-picker +and Stop acknowledgement within 2 seconds, terminal readiness within 5 seconds, +and ordered durable events in every run. Run `--stop-one` as a separate case; +require the six peers to finish and the stopped task to remain cancelled after +reload. Trace and fix a missed budget before calling the run complete. + In the owned Electron app, create separate direct tasks in `.dev/fixture-repo`. Open and close the model picker, open a Terminal, switch tasks, and reload. Capture the settled controls and terminal output in a screenshot, record the diff --git a/apps/server/src/application/bootstrap/server-bootstrap.ts b/apps/server/src/application/bootstrap/server-bootstrap.ts index 5975addb5..6b4742fdb 100644 --- a/apps/server/src/application/bootstrap/server-bootstrap.ts +++ b/apps/server/src/application/bootstrap/server-bootstrap.ts @@ -70,6 +70,7 @@ import { startAgentOrchestration, } from "../../features/agents"; import { AgentEventPublicationRegistry } from "../../features/agents/orchestration/agent-event-publication-registry.js"; +import { WorkerOwnedTurnRuntime } from "../../features/agents/execution/worker-owned-turn-runtime.js"; import { AgentEventPublicationRuntimePort, AgentReliabilityPort, @@ -342,6 +343,7 @@ const messageRepo = container.resolve(MessageRepo); const threadRepo = container.resolve(ThreadRepo); const providerRegistry = container.resolve(ProviderRegistry); const providerEventIngress = container.resolve(ProviderEventIngress); +const workerOwnedTurnRuntime = container.resolve(WorkerOwnedTurnRuntime); const cursorProvider = container.resolve("CursorProvider"); const providerAvailability = container.resolve(ProviderAvailabilityService); const toolCallRecordRepo = container.resolve(ToolCallRecordRepo); @@ -664,6 +666,7 @@ codexCatalogService.onSkillsChanged((cwd) => { // Create and start HTTP + WS server const { httpServer, wss } = createWsServer({ + workerQueueDepth: () => workerOwnedTurnRuntime.scheduler.depth(), runtime: hostRuntime, workspaceService, workspaceEnvironmentService, @@ -852,6 +855,9 @@ async function bootstrapServer(): Promise { cleanupWorker.start(); recordStartupCheckpoint("stale startup work recovery completed"); + await workerOwnedTurnRuntime.whenReady(); + recordStartupCheckpoint("execution workers ready"); + // Provider work must start only after every client-visible history route has // one durable display representation. startAgentOrchestration({ @@ -951,6 +957,8 @@ async function shutdown(): Promise { await captureCleanupFailure(() => providerRegistry.shutdown()); shutdownCoordinator.setPhase("shutdown provider event workers"); providerEventIngress.shutdown(); + shutdownCoordinator.setPhase("shutdown execution workers and writer"); + await captureCleanupFailure(() => workerOwnedTurnRuntime.close()); browserAutomationBroker.shutdown(); browserAutomationSessionLease.shutdown(); diff --git a/apps/server/src/application/transport/__tests__/ws-server-health.test.ts b/apps/server/src/application/transport/__tests__/ws-server-health.test.ts index 2d4dad2ea..627c8a98f 100644 --- a/apps/server/src/application/transport/__tests__/ws-server-health.test.ts +++ b/apps/server/src/application/transport/__tests__/ws-server-health.test.ts @@ -8,18 +8,12 @@ import "reflect-metadata"; import { describe, it, expect, afterEach, vi } from "vitest"; import * as NodeHTTP from "node:http"; import { WebSocket } from "ws"; -import { createWsServer } from "../ws-server.js"; +import { createWsServer, type WsServerDeps } from "../ws-server.js"; import type { RouterDeps } from "../ws-router.js"; /** Minimal RouterDeps stub — only agentService is called by the health handler. */ function makeMinimalDeps( - overrides: Partial void; - }> = {}, + overrides: Partial = {}, ): RouterDeps & { authToken: string; singleInstance: boolean; @@ -219,6 +213,28 @@ describe("/health endpoint", () => { expect((body as Record).activeAgents).toBe(2); }); + it("exposes bounded execution queue counts without command content", async () => { + const deps = makeMinimalDeps({ + workerQueueDepth: () => ({ + pending: 3, + pendingBytes: 512, + activeExecutions: 7, + workers: [{ index: 0, queued: 2, inFlight: true }], + }), + }); + ({ httpServer: server } = createWsServer(deps)); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + + const { body } = await getHealth(server); + expect((body as Record).workerQueue).toEqual({ + pending: 3, + pendingBytes: 512, + activeExecutions: 7, + workers: [{ index: 0, queued: 2, inFlight: true }], + }); + expect(JSON.stringify((body as Record).workerQueue)).not.toMatch(/thread|command|prompt|token/i); + }); + it("exposes content-free browser automation reliability diagnostics", async () => { const deps = makeMinimalDeps({ browserAutomationBroker: { diff --git a/apps/server/src/application/transport/ws-server.ts b/apps/server/src/application/transport/ws-server.ts index dfb60f805..1931db1f4 100644 --- a/apps/server/src/application/transport/ws-server.ts +++ b/apps/server/src/application/transport/ws-server.ts @@ -34,6 +34,7 @@ import type { } from "../../features/browser-automation/index.js"; import { EXTERNAL_THREAD_CONTROL_MCP_PATH } from "../../features/thread-control/index.js"; import type { ReliabilityHarnessAdapter } from "../../runtime/reliability-harness/control.js"; +import type { ExecutionMailboxDepth } from "../../features/agents/execution/execution-mailbox-scheduler.js"; /** Constant-time string comparison to prevent timing attacks on token validation. */ function safeTokenEqual(a: string, b: string): boolean { @@ -74,6 +75,8 @@ export type WsServerDeps = RouterDeps & { browserAutomationMcpHandler?: BrowserAutomationMcpHandler; /** Optional opt-in packaged reliability controls. */ reliabilityHarness?: ReliabilityHarnessAdapter; + /** Content-free counts for active execution worker queues. */ + workerQueueDepth?: () => ExecutionMailboxDepth; }; /** Refreshes mutable workspace authorization while preserving one connection's desktop identity. */ @@ -358,6 +361,7 @@ function createHealthBody(deps: WsServerDeps): Record { nightlyEvidence: deps.browserAutomationBroker.nightlyEvidenceStatus(), }; } + if (deps.workerQueueDepth) body.workerQueue = deps.workerQueueDepth(); if (!deps.singleInstance) body.authToken = deps.authToken; return body; } diff --git a/apps/server/src/features/agents/canonical/__tests__/canonical-codex-system-error-projection.test.ts b/apps/server/src/features/agents/canonical/__tests__/canonical-codex-system-error-projection.test.ts index c010b6894..81e6ba19c 100644 --- a/apps/server/src/features/agents/canonical/__tests__/canonical-codex-system-error-projection.test.ts +++ b/apps/server/src/features/agents/canonical/__tests__/canonical-codex-system-error-projection.test.ts @@ -3,7 +3,7 @@ import * as NodeFS from "node:fs"; import * as NodeOS from "node:os"; import * as NodePath from "node:path"; import type { Database } from "bun:sqlite"; -import type { AgentEvent } from "@mcode/contracts"; +import type { AgentEvent, ProviderId } from "@mcode/contracts"; import { afterEach, beforeEach, describe, expect, it } from "vitest"; import { openDatabase } from "../../../../runtime/persistence/sqlite/database.js"; @@ -19,14 +19,14 @@ const EXECUTION_ID = "11111111-1111-4111-8111-111111111111"; const NOW = "2026-09-24T10:00:00.000Z"; const execution = { threadId: THREAD_ID, turnId: TURN_ID, executionId: EXECUTION_ID }; -function seedThread(db: Database): void { +function seedThread(db: Database, providerId: ProviderId = "codex"): void { db.prepare("INSERT INTO workspaces (id, name, path, created_at, updated_at) VALUES (?, ?, ?, ?, ?)") .run("workspace-system-projection", "Workspace", "C:/fixture", NOW, NOW); db.prepare("INSERT INTO threads (id, workspace_id, title, branch, provider, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?)") - .run(THREAD_ID, "workspace-system-projection", "Thread", "main", "codex", NOW, NOW); + .run(THREAD_ID, "workspace-system-projection", "Thread", "main", providerId, NOW, NOW); const turns = new CanonicalParentTurnWrite(db, () => {}); expect(turns.start({ - thread: { id: THREAD_ID, workspaceId: "workspace-system-projection", providerId: "codex", createdAt: NOW }, + thread: { id: THREAD_ID, workspaceId: "workspace-system-projection", providerId, createdAt: NOW }, turnId: TURN_ID, executionId: EXECUTION_ID, permissionMode: "supervised", @@ -127,6 +127,24 @@ describe("CanonicalCodexSystemErrorProjection", () => { .toEqual({ sdk_session_id: null }); }); + it.each(["claude", "cursor"] as const)("persists a %s session cursor with its provider identity", (providerId) => { + const providerDb = openDatabase({ dbPath: NodePath.join(directory, `${providerId}.sqlite`) }); + try { + seedThread(providerDb, providerId); + const providerProjection = new CanonicalCodexSystemErrorProjection(providerDb); + const cursor = boundSystem("sdk_session_id:native-session"); + expect(providerProjection.projectBoundSystem(execution, providerId, cursor, + [{ kind: "session-cursor", event: cursor }], "writer")) + .toMatchObject({ kind: "system", event: { subtype: "sdk_session_id:native-session" } }); + expect(providerDb.prepare("SELECT sdk_session_id FROM threads WHERE id = ?").get(THREAD_ID)) + .toEqual({ sdk_session_id: "native-session" }); + expect(new CanonicalAgentBoundary(providerDb, () => {}).loadCheckpoint(EXECUTION_ID)) + .toMatchObject({ nativeCursor: { providerId, scope: "session", value: "native-session", provenance: "native" } }); + } finally { + providerDb.close(true); + } + }); + it("returns cloneable error terminal input without publishing or terminalizing early", () => { expect(reducer.reduce({ type: "turnStarted", threadId: THREAD_ID, turnExecutionId: EXECUTION_ID }).kind).toBe("reduced"); expect(reducer.reduce({ type: "textDelta", threadId: THREAD_ID, turnExecutionId: EXECUTION_ID, diff --git a/apps/server/src/features/agents/canonical/__tests__/canonical-execution-semantic-writer.test.ts b/apps/server/src/features/agents/canonical/__tests__/canonical-execution-semantic-writer.test.ts index 3e6696f12..7d007df48 100644 --- a/apps/server/src/features/agents/canonical/__tests__/canonical-execution-semantic-writer.test.ts +++ b/apps/server/src/features/agents/canonical/__tests__/canonical-execution-semantic-writer.test.ts @@ -10,7 +10,7 @@ import { openDatabase } from "../../../../runtime/persistence/sqlite/database.js import { MessageRepo } from "../../conversation/persistence/message-repo.js"; import { TaskRepo } from "../../orchestration/persistence/task-repo.js"; import { CodexLiveEventReducer } from "../../execution/codex-live-event-reducer.js"; -import type { ExecutionSemanticOperation } from "../../execution/execution-worker-handler.js"; +import type { ExecutionSemanticOperation, ExecutionWorkCommand } from "../../execution/execution-worker-handler.js"; import { ExecutionWorkerHandler } from "../../execution/execution-worker-handler.js"; import { ParentAssistantTextCheckpointService } from "../../turns/parent-assistant-text-checkpoint-service.js"; import { CodexParentMessageProjection } from "../../turns/codex-parent-message-projection.js"; @@ -125,6 +125,40 @@ function systemLiveOperation(ordinal: number, systemEvent: Extract { + const identity = { threadId: THREAD_ID, turnExecutionId: EXECUTION_ID }; + const terminal: AgentEvent = outcome === "completed" + ? { ...identity, type: "turnComplete", reason: "end_turn", costUsd: null, tokensIn: 20, tokensOut: 5 } + : outcome === "errored" ? { ...identity, type: "error", error: "Provider failed" } + : { ...identity, type: "ended", outcome: "interrupted" }; + return { + kind: "finish-live-event", outcome, + projection: { threadId: THREAD_ID, executionId: EXECUTION_ID, outcome, endedAt: NOW, + assistant: { content: "Final answer", model: null, attachments: [] }, narrative: [] }, + input: { ...execution, providerId: "codex", providerIdentities: [], outcome, + ...(outcome === "errored" ? { error: "Provider failed" } : {}), projection: { kind: "writer-staged" } }, + ...(outcome === "cancelled" ? {} : { providerEvent: { + phase: "running", nativeCursor: null, events: [runtimeEvent(2, terminal)], + } }), + livePublication: [{ after: "terminal", event: terminal }], + }; +} + describe("CanonicalExecutionSemanticWriter through ExecutionWorkerHandler", () => { let directory: string; let path: string; @@ -160,6 +194,339 @@ describe("CanonicalExecutionSemanticWriter through ExecutionWorkerHandler", () = recoveryIncidentId: "incident-1", }; + it("persists raw late hooks after terminal and releases their committed message identity", async () => { + const operations: ExecutionSemanticOperation[] = []; + handler = new ExecutionWorkerHandler({ transact: async (input) => { + operations.push(input); + return writer.transact(input); + } }); + expect(await send(1, { kind: "start", providerId: "codex", input: startInput(), + parentLive: { planFeature: "none", precedingMessageId: "user-1" } })).toMatchObject({ kind: "committed" }); + const raw = (ordinal: number, event: AgentEvent) => send(ordinal, { kind: "event", phase: "running", + nativeCursor: null, events: [runtimeEvent(ordinal, event)] }); + const bound = { threadId: THREAD_ID, turnExecutionId: EXECUTION_ID }; + const started = await raw(2, { ...bound, type: "turnStarted" }); + expect(started.kind, JSON.stringify(started)).toBe("committed"); + expect((await raw(3, { ...bound, type: "textDelta", delta: "Final answer", isFinalResponse: true })).kind).toBe("committed"); + const finish = finishLiveCommand(); + const terminal = await send(4, { kind: "event", phase: "running", nativeCursor: null, + events: [runtimeEvent(4, { ...bound, type: "turnComplete", reason: "end_turn", costUsd: null, tokensIn: 20, tokensOut: 5 })], + terminalInput: finish.input }); + expect(terminal.kind, JSON.stringify(terminal)).toBe("committed"); + expect((await raw(5, { ...bound, type: "hookStarted", hookName: "Save", hookType: "stop" })).kind).toBe("committed"); + const completed = await raw(6, { ...bound, type: "hookCompleted", hookName: "Save", exitCode: 0, durationMs: 12, didBlock: false }); + expect(completed).toMatchObject({ kind: "committed", providerCommit: { eventCount: 1 }, providerEvents: [], + livePublication: [{ after: "terminal", event: { type: "hookCompleted", persistedMessageId: expect.any(String), persistedHookId: expect.any(String) } }] }); + const completedOperation = operations[5]; + if (!completedOperation) throw new Error("Expected hook operation"); + const receipt = await writer.transact(completedOperation); + const released: AgentEvent[] = []; + new ExecutionLivePublicationRelease({ isBound: () => true, publish: (event) => released.push(event) }).release(completedOperation, receipt); + expect(released).toEqual([expect.objectContaining({ type: "hookCompleted", persistedHookId: expect.any(String), persistedMessageId: expect.any(String) })]); + const canonical = new CanonicalAgentBoundary(db); + const message = canonical.loadTerminalProjection(TURN_ID).message; + expect(db.prepare("SELECT message_id, hook_name, duration_ms FROM hook_executions").all()) + .toEqual([{ message_id: message?.id, hook_name: "Save", duration_ms: 12 }]); + expect(JSON.stringify(canonical.loadConversationProjection(THREAD_ID, 10))).toContain('"duration_ms":12'); + expect(canonical.loadCheckpoint(EXECUTION_ID)?.terminalOutcome).toBe("completed"); + expect((await raw(7, { ...bound, type: "ended", outcome: "completed" })).kind).toBe("committed"); + expect(await raw(8, { ...bound, type: "hookProgress", hookName: "Save", output: "too late" })) + .toEqual({ kind: "rejected", reason: "invalid-event-routing" }); + }); + + it("atomically records and releases a late system notice without changing terminal outcome", async () => { + await send(1, { kind: "start", providerId: "codex", input: startInput() }); + await send(2, finishLiveCommand()); + const notice: Extract = { type: "system", threadId: THREAD_ID, + turnExecutionId: EXECUTION_ID, subtype: "provider.notice.unknown-event", message: "Late provider notice", + systemNotice: { kind: "diagnostic", presentation: "timeline", scope: "turn", noticeKey: "late-notice" } }; + const input: ExecutionSemanticOperation = { ...operation(3, { kind: "post-terminal-event", + systemIntents: [{ kind: "system-notice", event: notice }], + providerEvent: { phase: "running", nativeCursor: null, events: [runtimeEvent(3, notice)] } }), + livePublication: [{ after: "terminal", event: notice }] }; + db.run("CREATE TRIGGER fail_late_receipt BEFORE INSERT ON canonical_writer_operation_receipts WHEN NEW.kind = 'semantic:post-terminal-event' BEGIN SELECT RAISE(ABORT, 'late receipt unavailable'); END"); + await expect(writer.transact(input)).rejects.toThrow("late receipt unavailable"); + expect(new MessageRepo(db).listIncludingInternal(THREAD_ID).some((message) => message.content === notice.message)).toBe(false); + expect(db.prepare("SELECT event_id FROM canonical_agent_events WHERE event_id = ?").get(`${EXECUTION_ID}:runtime-3`)).toBeNull(); + db.run("DROP TRIGGER fail_late_receipt"); + const receipt = await writer.transact(input); + if (receipt.kind !== "committed") throw new Error("Expected late receipt"); + const released: AgentEvent[] = []; + new ExecutionLivePublicationRelease({ isBound: () => true, publish: (event) => released.push(event) }).release(input, receipt); + expect(released).toEqual([expect.objectContaining({ ...notice, messageId: expect.any(String) })]); + expect(await writer.transact(input)).toEqual(receipt); + expect(new CanonicalAgentBoundary(db).loadCheckpoint(EXECUTION_ID)?.terminalOutcome).toBe("completed"); + expect(db.prepare("SELECT status FROM threads WHERE id = ?").get(THREAD_ID)).toEqual({ status: "completed" }); + }); + + it("rejects late content, cursor changes, wrong outcome and stale leases without publishing", async () => { + await send(1, { kind: "start", providerId: "codex", input: startInput() }); + await send(2, finishLiveCommand()); + published.length = 0; + const bound = { threadId: THREAD_ID, turnExecutionId: EXECUTION_ID }; + const events: AgentEvent[] = [ + { ...bound, type: "textDelta", delta: "Must not append" }, + { ...bound, type: "toolUse", toolCallId: "late-tool", toolName: "Read", toolInput: {} }, + { ...bound, type: "system", subtype: "sdk_session_id:late-session" }, + { ...bound, type: "system", subtype: "sdk_session_invalidated" }, + { ...bound, type: "ended", outcome: "errored" }, + ]; + for (const event of events) { + expect(await writer.transact({ ...operation(3, { kind: "post-terminal-event" }), + livePublication: [{ after: "terminal", event }] })).toEqual({ kind: "conflict", operationId: "lease-1:3" }); + } + const input: ExecutionSemanticOperation = { ...operation(3, { kind: "post-terminal-event" }), + livePublication: [{ after: "terminal", event: { ...bound, type: "ended", outcome: "completed" } }] }; + expect((await writer.transact({ ...input, lease: { ...lease, workerGeneration: 2 } })).kind).toBe("conflict"); + expect((await writer.transact({ ...input, ordinal: 4 })).kind).toBe("conflict"); + expect(published).toEqual([]); + expect(new CanonicalAgentBoundary(db).loadCheckpoint(EXECUTION_ID)?.terminalOutcome).toBe("completed"); + expect((await writer.transact(input)).kind).toBe("committed"); + }); + + it.each(["completed", "errored"] as const)("commits a %s provider terminal event with its final projection in one command", async (outcome) => { + const sequences: number[] = []; + writer = new CanonicalExecutionSemanticWriter(db, (events) => { + for (const item of events) { + published.push(item.eventId); + if (item.eventId === `${EXECUTION_ID}:runtime-2`) { + expect(db.prepare("SELECT terminal_outcome FROM canonical_agent_ingest_checkpoints WHERE execution_id = ?") + .get(EXECUTION_ID)).toEqual({ terminal_outcome: outcome }); + } + sequences.push(item.acceptedSequence); + } + }); + handler = new ExecutionWorkerHandler(writer); + await send(1, { kind: "start", providerId: "codex", input: startInput() }); + sequences.length = 0; + const command = finishLiveCommand(outcome); + const receipt = await send(2, command); + expect(receipt).toMatchObject({ kind: "committed", providerCommit: { outcome: "committed", eventCount: 1 }, + providerEvents: [], livePublication: [{ publicationId: "1", after: "terminal" }] }); + expect(new MessageRepo(db).listIncludingInternal(THREAD_ID).filter((message) => !message.is_internal)).toEqual(expect.arrayContaining([ + expect.objectContaining({ role: "assistant", content: "Final answer", outcome }), + ])); + expect(sequences).toEqual([...sequences].sort((left, right) => left - right)); + expect(published).toContain(`${EXECUTION_ID}:runtime-2`); + expect(await send(3, { kind: "release" })).toEqual({ kind: "released" }); + const { livePublication, ...mutation } = command; + if (receipt.kind !== "committed") throw new Error("Expected a terminal receipt"); + const released: AgentEvent[] = []; + const release = new ExecutionLivePublicationRelease({ isBound: () => true, publish: (event) => released.push(event) }); + release.release({ ...operation(2, mutation), livePublication }, receipt); + expect(released).toEqual([expect.objectContaining({ type: outcome === "completed" ? "turnComplete" : "error", + publicationId: "1" })]); + published.length = 0; + db.close(true); + db = openDatabase({ dbPath: path }); + writer = new CanonicalExecutionSemanticWriter(db, (events) => published.push(...events.map((event) => event.eventId))); + expect(await writer.transact({ ...operation(2, mutation), livePublication })).toEqual(receipt); + expect(published).toEqual([]); + }); + + it("settles Stop through one synthetic terminal command without a provider draft", async () => { + await send(1, { kind: "start", providerId: "codex", input: startInput() }); + const stop = await handler.handle({ requestId: 2, execution, lease, ordinal: 2, stopWatermark: 1, + command: { kind: "stop", requestId: "stop-1" } }); + expect(stop.result.kind).toBe("committed"); + expect(await send(3, finishLiveCommand("cancelled"))).toMatchObject({ kind: "committed", + livePublication: [{ event: { type: "ended", outcome: "interrupted" } }] }); + expect(db.prepare("SELECT terminal_outcome FROM canonical_agent_ingest_checkpoints WHERE execution_id = ?") + .get(EXECUTION_ID)).toEqual({ terminal_outcome: "cancelled" }); + expect(db.prepare("SELECT status FROM threads WHERE id = ?").get(THREAD_ID)).toEqual({ status: "interrupted" }); + }); + + it.each(["codex", "claude"] as const)("keeps the latest %s session cursor when terminal input has stale identities", async (providerId) => { + db.prepare("UPDATE threads SET provider = ? WHERE id = ?").run(providerId, THREAD_ID); + const start = startInput(); + await send(1, { kind: "start", providerId, input: { ...start, thread: { ...start.thread, providerId } } }); + const cursor: Extract = { type: "system", threadId: THREAD_ID, + turnExecutionId: EXECUTION_ID, subtype: "sdk_session_id:latest-session" }; + expect(await send(2, { kind: "live-event", text: { kind: "unchanged" }, + systemIntents: [{ kind: "session-cursor", event: cursor }], publication: { after: "writer", event: cursor }, + })).toMatchObject({ kind: "committed" }); + const command = finishLiveCommand(); + if (!command.providerEvent) throw new Error("Expected terminal provider evidence"); + const scope = providerId === "codex" ? "thread" : "session"; + expect(await send(3, { ...command, + input: { ...command.input, providerId, + providerIdentities: [{ providerId, scope, value: "stale-session", provenance: "native" }] }, + providerEvent: { ...command.providerEvent, + events: command.providerEvent.events.map((event) => ({ ...event, sourceProviderId: providerId })) }, + })).toMatchObject({ kind: "committed" }); + const expected = { providerId, scope, value: "latest-session", provenance: "native" }; + const canonical = new CanonicalAgentBoundary(db); + expect(canonical.loadThread(THREAD_ID)?.providerIdentities).toEqual([expected]); + expect(canonical.loadCheckpoint(EXECUTION_ID)?.nativeCursor).toEqual(expected); + }); + + it("rolls back terminal provider evidence and staging when preparation cannot reserve its operation", async () => { + await send(1, { kind: "start", providerId: "codex", input: startInput() }); + db.run("CREATE TRIGGER fail_terminal_reservation BEFORE INSERT ON canonical_writer_operation_receipts WHEN NEW.kind = 'semantic:finish-pending' BEGIN SELECT RAISE(ABORT, 'reservation unavailable'); END"); + await expect(send(2, finishLiveCommand())).rejects.toThrow("reservation unavailable"); + expect(db.prepare("SELECT event_id FROM canonical_agent_events WHERE event_id = ?") + .get(`${EXECUTION_ID}:runtime-2`)).toBeNull(); + expect(new MessageRepo(db).listIncludingInternal(THREAD_ID).some((message) => message.role === "assistant")).toBe(false); + expect(published).not.toContain(`${EXECUTION_ID}:runtime-2`); + db.run("DROP TRIGGER fail_terminal_reservation"); + expect((await send(2, finishLiveCommand())).kind).toBe("committed"); + }); + + it("rejects compound terminal identity, outcome, and provider publication mismatches before finalization", async () => { + await send(1, { kind: "start", providerId: "codex", input: startInput() }); + const { livePublication, ...mutation } = finishLiveCommand(); + const input = { ...operation(2, mutation), livePublication }; + const invalid = [ + { ...input, mutation: { ...mutation, projection: { ...mutation.projection, executionId: "other-execution" } } }, + { ...input, mutation: { ...mutation, input: { ...mutation.input, outcome: "errored" as const } } }, + { ...input, mutation: { ...mutation, providerEvent: { phase: "running", nativeCursor: null, events: [event()] } } }, + ]; + for (const candidate of invalid) { + expect(await writer.transact(candidate)).toEqual({ kind: "conflict", operationId: "lease-1:2" }); + } + expect(db.prepare("SELECT kind FROM canonical_writer_operation_receipts WHERE execution_id = ? AND operation_id = ?") + .get(EXECUTION_ID, "lease-1:2")).toBeNull(); + expect(new MessageRepo(db).listIncludingInternal(THREAD_ID).some((message) => message.role === "assistant")).toBe(false); + expect(await writer.transact(input)).toMatchObject({ kind: "committed" }); + }); + + it("resumes a compound terminal operation after final receipt failure and database reopen", async () => { + await send(1, { kind: "start", providerId: "codex", input: startInput() }); + db.run("CREATE TRIGGER fail_compound_terminal_receipt BEFORE UPDATE OF kind ON canonical_writer_operation_receipts WHEN NEW.kind = 'semantic:finish-live-event' BEGIN SELECT RAISE(ABORT, 'terminal receipt unavailable'); END"); + const command = finishLiveCommand(); + await expect(send(2, command)).rejects.toThrow("terminal receipt unavailable"); + expect(db.prepare("SELECT terminal_outcome FROM canonical_agent_ingest_checkpoints WHERE execution_id = ?") + .get(EXECUTION_ID)).toEqual({ terminal_outcome: null }); + expect(db.prepare("SELECT kind FROM canonical_writer_operation_receipts WHERE execution_id = ? AND operation_id = ?") + .get(EXECUTION_ID, "lease-1:2")).toEqual({ kind: "semantic:finish-pending" }); + expect(new MessageRepo(db).listIncludingInternal(THREAD_ID).some((message) => message.role === "assistant" && !message.is_internal)).toBe(false); + expect(published).not.toContain(`${EXECUTION_ID}:runtime-2`); + db.run("DROP TRIGGER fail_compound_terminal_receipt"); + db.close(true); + db = openDatabase({ dbPath: path }); + writer = new CanonicalExecutionSemanticWriter(db, (events) => published.push(...events.map((event) => event.eventId))); + const { livePublication, ...mutation } = command; + const input = { ...operation(2, mutation), livePublication }; + expect(await writer.transact({ ...input, mutation: { ...mutation, + projection: { ...mutation.projection, assistant: { ...mutation.projection.assistant, content: "Changed retry" } }, + } })).toEqual({ kind: "conflict", operationId: "lease-1:2" }); + expect(await writer.transact(input)).toMatchObject({ kind: "committed", livePublication: [{ publicationId: "1" }] }); + expect(published.filter((id) => id === `${EXECUTION_ID}:runtime-2`)).toHaveLength(1); + }); + + it("retries only unacknowledged compound terminal publication chunks", async () => { + let failTerminalPublication = false; + writer = new CanonicalExecutionSemanticWriter(db, (events) => { + if (failTerminalPublication && events.some((event) => event.eventId !== `${EXECUTION_ID}:runtime-2`)) { + throw new Error("terminal publisher unavailable"); + } + published.push(...events.map((event) => event.eventId)); + }); + handler = new ExecutionWorkerHandler(writer); + await send(1, { kind: "start", providerId: "codex", input: startInput() }); + failTerminalPublication = true; + const command = finishLiveCommand(); + await expect(send(2, command)).rejects.toThrow("terminal publisher unavailable"); + expect(published.filter((id) => id === `${EXECUTION_ID}:runtime-2`)).toHaveLength(1); + db.close(true); + db = openDatabase({ dbPath: path }); + writer = new CanonicalExecutionSemanticWriter(db, (events) => published.push(...events.map((event) => event.eventId))); + const { livePublication, ...mutation } = command; + expect(await writer.transact({ ...operation(2, mutation), livePublication })).toMatchObject({ kind: "committed" }); + expect(published.filter((id) => id === `${EXECUTION_ID}:runtime-2`)).toHaveLength(1); + expect(published).toContain(`${EXECUTION_ID}:turn.completed`); + }); + + it("recovers a prepared compound terminal projection when its worker is lost", async () => { + await send(1, { kind: "start", providerId: "codex", input: startInput() }); + db.run("CREATE TRIGGER fail_terminal_before_loss BEFORE UPDATE OF kind ON canonical_writer_operation_receipts WHEN NEW.kind = 'semantic:finish-live-event' BEGIN SELECT RAISE(ABORT, 'terminal receipt unavailable'); END"); + await expect(send(2, finishLiveCommand())).rejects.toThrow("terminal receipt unavailable"); + expect(writer.interruptWorkerLoss(loss)).toMatchObject({ kind: "committed" }); + expect(db.prepare("SELECT terminal_outcome FROM canonical_agent_ingest_checkpoints WHERE execution_id = ?") + .get(EXECUTION_ID)).toEqual({ terminal_outcome: "interrupted" }); + expect(new MessageRepo(db).listIncludingInternal(THREAD_ID)).toEqual(expect.arrayContaining([ + expect.objectContaining({ role: "assistant", content: "Final answer", outcome: "interrupted", is_internal: false }), + ])); + db.run("DROP TRIGGER fail_terminal_before_loss"); + const { livePublication, ...mutation } = finishLiveCommand(); + expect(await writer.transact({ ...operation(2, mutation), livePublication })) + .toEqual({ kind: "conflict", operationId: "lease-1:2" }); + }); + + it("commits a provider event and its parent effects with one replayable publication", async () => { + await send(1, { kind: "start", providerId: "codex", input: startInput() }); + const input = parentTextOperation(); + const receipt = await writer.transact(input); + expect(receipt).toMatchObject({ kind: "committed", providerCommit: { eventCount: 1 }, providerEvents: [], + assistantTextCheckpoint: { outcome: "committed" }, + livePublication: [{ publicationId: "1", event: { delta: "Durable parent text" } }] }); + expect(new ParentAssistantTextCheckpointService(db).restore(EXECUTION_ID)).toBe("Durable parent text"); + expect(published).toContain(`${EXECUTION_ID}:runtime-2`); + expect(await writer.transact(input)).toEqual(receipt); + expect(new ParentAssistantTextCheckpointService(db).restore(EXECUTION_ID)).toBe("Durable parent text"); + }); + + it("rolls back canonical append, parent effects, and publication when their receipt fails", async () => { + await send(1, { kind: "start", providerId: "codex", input: startInput() }); + const input = parentTextOperation(); + db.run("CREATE TRIGGER fail_parent_receipt BEFORE INSERT ON canonical_writer_operation_receipts WHEN NEW.kind = 'semantic:append-events' BEGIN SELECT RAISE(ABORT, 'parent receipt unavailable'); END"); + await expect(writer.transact(input)).rejects.toThrow("parent receipt unavailable"); + expect(new ParentAssistantTextCheckpointService(db).restore(EXECUTION_ID)).toBe(""); + expect(db.prepare("SELECT event_id FROM canonical_agent_events WHERE event_id = ?") + .get(`${EXECUTION_ID}:runtime-2`)).toBeNull(); + expect(published).not.toContain(`${EXECUTION_ID}:runtime-2`); + db.run("DROP TRIGGER fail_parent_receipt"); + expect(await writer.transact(input)).toMatchObject({ kind: "committed", providerEvents: [], + livePublication: [{ publicationId: "1" }] }); + }); + + it("rejects parent effects without the matching provider event or exact execution", async () => { + await send(1, { kind: "start", providerId: "codex", input: startInput() }); + const input = parentTextOperation(); + if (input.mutation.kind !== "append-events") throw new Error("Expected provider append"); + const unrelated: AgentEvent = { type: AgentEventType.TextDelta, threadId: THREAD_ID, + turnExecutionId: EXECUTION_ID, delta: "A different event" }; + expect(await writer.transact({ ...input, mutation: { ...input.mutation, events: [runtimeEvent(2, unrelated)] } })) + .toEqual({ kind: "conflict", operationId: "lease-1:2" }); + expect(await writer.transact({ ...input, mutation: { ...input.mutation, parentLive: { + text: { kind: "append", inputs: [{ ...execution, executionId: "wrong-execution", sequence: 1, text: "Durable parent text" }] }, + } } })).toEqual({ kind: "conflict", operationId: "lease-1:2" }); + expect(new ParentAssistantTextCheckpointService(db).restore(EXECUTION_ID)).toBe(""); + expect(db.prepare("SELECT event_id FROM canonical_agent_events WHERE event_id = ?") + .get(`${EXECUTION_ID}:runtime-2`)).toBeNull(); + expect(published).not.toContain(`${EXECUTION_ID}:runtime-2`); + }); + + it("replays a compound system notice with its writer-assigned message identity", async () => { + await send(1, { kind: "start", providerId: "codex", input: startInput() }); + const notice: Extract = { + type: "system", threadId: THREAD_ID, turnExecutionId: EXECUTION_ID, + subtype: "provider.notice.unknown-event", message: "Codex reported an update.", + systemNotice: { kind: "diagnostic", presentation: "timeline", scope: "turn", noticeKey: "compound-notice" }, + }; + const live = systemLiveOperation(2, notice); + if (live.mutation.kind !== "live-event") throw new Error("Expected live system effects"); + const input = { ...live, mutation: { + kind: "append-events" as const, phase: "running", nativeCursor: null, + events: [runtimeEvent(2, notice)], parentLive: { text: live.mutation.text, systemIntents: live.mutation.systemIntents }, + } }; + const receipt = await writer.transact(input); + expect(receipt).toMatchObject({ kind: "committed", providerEvents: [], livePublication: [ + { event: { ...notice, messageId: expect.any(String) } }, + ] }); + const released: AgentEvent[] = []; + new ExecutionLivePublicationRelease({ + isBound: () => true, + publish: (event) => { released.push(event); }, + }).release(input, receipt); + expect(released).toEqual([expect.objectContaining({ + ...notice, messageId: expect.any(String), publicationId: "1", + })]); + expect(await writer.transact(input)).toEqual(receipt); + expect(db.prepare("SELECT COUNT(*) AS count FROM messages WHERE thread_id = ? AND role = 'system'").get(THREAD_ID)) + .toEqual({ count: 1 }); + }); + it("commits context and compaction projections before publishing their canonical events", async () => { const seenAtPublication: { eventId: string; state: unknown }[] = []; writer = new CanonicalExecutionSemanticWriter(db, (events) => { diff --git a/apps/server/src/features/agents/canonical/__tests__/canonical-execution-writer-port.test.ts b/apps/server/src/features/agents/canonical/__tests__/canonical-execution-writer-port.test.ts index 01437279e..2a691de47 100644 --- a/apps/server/src/features/agents/canonical/__tests__/canonical-execution-writer-port.test.ts +++ b/apps/server/src/features/agents/canonical/__tests__/canonical-execution-writer-port.test.ts @@ -267,6 +267,35 @@ describe("execution semantic writer transport", () => { expect(published).toBe(8_193); }); + it("releases a real Codex context estimate with optional undefined fields", () => { + const published: AgentEvent[] = []; + const release = new ExecutionLivePublicationRelease({ + isBound: () => true, + publish: (event) => { published.push(event); }, + }); + const event: AgentEvent = { + type: AgentEventType.ContextEstimate, + threadId: THREAD_ID, + turnExecutionId: EXECUTION_ID, + tokensIn: 30_787, + tokensOut: 25, + totalProcessedTokens: 30_812, + contextWindow: 258_400, + cacheReadTokens: undefined, + }; + const operation: ExecutionSemanticOperation = { + ...beginOperation(), + livePublication: [{ after: "writer", event }], + }; + release.release(operation, { + kind: "committed", + operationId: operation.operationId, + durableRevision: 1, + livePublication: [{ after: "writer", event: { ...event }, publicationId: "1" }], + }); + expect(published).toEqual([{ ...event, publicationId: "1" }]); + }); + it("commits and replays a semantic start through the dedicated SQLite worker", async () => { writer = new CanonicalAgentWriterClient(NodePath.join(directory, "app.sqlite")); const published: string[] = []; diff --git a/apps/server/src/features/agents/canonical/__tests__/canonical-parent-turn-write.test.ts b/apps/server/src/features/agents/canonical/__tests__/canonical-parent-turn-write.test.ts index 774a61312..06c7f825d 100644 --- a/apps/server/src/features/agents/canonical/__tests__/canonical-parent-turn-write.test.ts +++ b/apps/server/src/features/agents/canonical/__tests__/canonical-parent-turn-write.test.ts @@ -118,6 +118,32 @@ describe("CanonicalParentTurnWrite", () => { NodeFS.rmSync(directory, { recursive: true, force: true }); }); + it("stages the assistant after provider system notices that follow its user prompt", () => { + writer.start(startInput()); + const messages = new MessageRepo(db); + messages.createSystemNotice(THREAD_ID, "Provider notice", 2, undefined); + const messageId = deriveTurnAssistantMessageId(THREAD_ID, "user-1"); + + expect(writer.stageLiveAssistant({ threadId: THREAD_ID, turnId: TURN_ID, executionId: EXECUTION_ID }, { + precedingMessageId: "user-1", messageId, content: "Answer", model: null, attachments: [], + })).toBe(true); + expect(messages.findByIdInThreadIncludingInternal(THREAD_ID, messageId)) + .toMatchObject({ sequence: 3, is_internal: true, content: "Answer" }); + }); + + it("does not stage the assistant after a later user prompt", () => { + writer.start(startInput()); + const messages = new MessageRepo(db); + messages.createSystemNotice(THREAD_ID, "Provider notice", 2, undefined); + messages.create(THREAD_ID, "user", "Later prompt", 3); + const messageId = deriveTurnAssistantMessageId(THREAD_ID, "user-1"); + + expect(writer.stageLiveAssistant({ threadId: THREAD_ID, turnId: TURN_ID, executionId: EXECUTION_ID }, { + precedingMessageId: "user-1", messageId, content: "Answer", model: null, attachments: [], + })).toBe(false); + expect(messages.findByIdInThreadIncludingInternal(THREAD_ID, messageId)).toBeNull(); + }); + it("commits cloneable start, event checkpoint and terminal projection across a reload", async () => { const start = startInput(); expect(() => structuredClone(start)).not.toThrow(); diff --git a/apps/server/src/features/agents/canonical/__tests__/canonical-plan-live-projection.test.ts b/apps/server/src/features/agents/canonical/__tests__/canonical-plan-live-projection.test.ts index a2ac63547..322c72a80 100644 --- a/apps/server/src/features/agents/canonical/__tests__/canonical-plan-live-projection.test.ts +++ b/apps/server/src/features/agents/canonical/__tests__/canonical-plan-live-projection.test.ts @@ -88,6 +88,12 @@ describe("Codex live plan projections on the sole writer", () => { release.release(write, receipt); release.release(write, receipt); expect(published).toEqual([{ threadId: THREAD_ID, questions }]); + const compound = { ...write, mutation: { + kind: "append-events" as const, phase: "running", nativeCursor: null, events: [], + parentLive: { text: write.mutation.text, planQuestions: questions }, + } }; + expect(new ExecutionPlanQuestionRelease(() => {}).validate(compound, receipt)).toEqual(receipt.kind === "committed" + ? receipt.planQuestions : null); db.close(true); db = openDatabase({ dbPath: path }); writer = new CanonicalExecutionSemanticWriter(db, () => {}); diff --git a/apps/server/src/features/agents/canonical/canonical-agent-writer.worker.ts b/apps/server/src/features/agents/canonical/canonical-agent-writer.worker.ts index 072716413..eb1393da0 100644 --- a/apps/server/src/features/agents/canonical/canonical-agent-writer.worker.ts +++ b/apps/server/src/features/agents/canonical/canonical-agent-writer.worker.ts @@ -134,7 +134,8 @@ async function handleSemanticWrite( correlation: Pick, ): Promise { try { - if (!semanticWriter || semanticPublication) throw new Error("Semantic writer is not ready"); + const writer = semanticWriter; + if (!writer || semanticPublication) throw new Error("Semantic writer is not ready"); if (request.kind === "semantic-transact" ? request.operation.operationId !== request.operationId || request.operation.execution.executionId !== request.executionId @@ -144,8 +145,8 @@ async function handleSemanticWrite( } semanticPublication = correlation; const receipt = request.kind === "semantic-transact" - ? await semanticWriter.transact(request.operation) - : semanticWriter.interruptWorkerLoss(request.input); + ? await withSQLiteBusyRetry(() => writer.transact(request.operation)) + : await withSQLiteBusyRetry(() => writer.interruptWorkerLoss(request.input)); return { ...correlation, kind: "semantic-transacted", receipt }; } catch { return { ...correlation, kind: "failed", reason: "write-failed" }; @@ -154,6 +155,24 @@ async function handleSemanticWrite( } } +async function withSQLiteBusyRetry(work: () => Promise | T): Promise { + const delaysMs = [10, 30, 90, 270, 500]; + for (const delayMs of delaysMs) { + try { + return await work(); + } catch (error) { + if (!isSQLiteBusy(error)) throw error; + await new Promise((resolve) => setTimeout(resolve, delayMs)); + } + } + return work(); +} + +function isSQLiteBusy(error: unknown): boolean { + return error instanceof Error && "code" in error + && typeof error.code === "string" && error.code.startsWith("SQLITE_BUSY"); +} + async function handleWrite( request: Extract, correlation: Pick, diff --git a/apps/server/src/features/agents/canonical/canonical-codex-system-error-projection.ts b/apps/server/src/features/agents/canonical/canonical-codex-system-error-projection.ts index 2b48eb507..4231be344 100644 --- a/apps/server/src/features/agents/canonical/canonical-codex-system-error-projection.ts +++ b/apps/server/src/features/agents/canonical/canonical-codex-system-error-projection.ts @@ -1,6 +1,6 @@ import type { Database } from "bun:sqlite"; import * as NodeUtil from "node:util"; -import type { AgentEvent } from "@mcode/contracts"; +import { ProviderIdSchema, type AgentEvent, type ProviderId } from "@mcode/contracts"; import { ThreadRepo } from "../../thread-control/persistence/thread-repo.js"; import { MessageRepo } from "../conversation/persistence/message-repo.js"; @@ -73,7 +73,7 @@ export class CanonicalCodexSystemErrorProjection { } if (event.type === "system") { if (!reduction.writer.every(isSystemWriterIntent)) throw new Error("Codex system has a non-system intent"); - return this.projectBoundSystem(reduction.execution, event, reduction.writer, reduction.publication.after); + return this.projectBoundSystem(reduction.execution, "codex", event, reduction.writer, reduction.publication.after); } if (event.type === "error") return this.prepareError(reduction, event, endedAt); throw new Error(`Codex ${event.type} needs another feature owner`); @@ -82,6 +82,7 @@ export class CanonicalCodexSystemErrorProjection { /** Apply one bound system event and return the event that the committed receipt must publish. */ projectBoundSystem( execution: ExecutionIdentity, + providerId: string, event: SystemEvent, intents: readonly CodexSystemWriterIntent[], after: "writer" | "terminal", @@ -92,16 +93,20 @@ export class CanonicalCodexSystemErrorProjection { if (!matchesCodexSystemIntents(event, intents)) { throw new Error("Codex system intents do not match the event"); } + const provider = ProviderIdSchema.parse(providerId); + if (provider !== "codex" && provider !== "claude" && provider !== "cursor") { + throw new Error(`Provider ${provider} has no execution-bound system projection`); + } return this.db.transaction(() => { let published: SystemEvent = { ...event }; for (const intent of intents) { - published = this.applySystemIntent(execution.executionId, intent, published); + published = this.applySystemIntent(execution.executionId, provider, intent, published); } return structuredClone({ kind: "system", event: published, after } satisfies SystemProjectionResult); })(); } - private applySystemIntent(executionId: string, intent: CodexSystemWriterIntent, event: SystemEvent): SystemEvent { + private applySystemIntent(executionId: string, providerId: ProviderId, intent: CodexSystemWriterIntent, event: SystemEvent): SystemEvent { switch (intent.kind) { case "notice-session": if (event.subtype !== "provider.session.started") throw new Error("Codex notice session subtype is invalid"); @@ -114,12 +119,12 @@ export class CanonicalCodexSystemErrorProjection { return { ...event, messageId: message.id }; } case "session-cursor": - this.applyCursor(executionId, event); + this.applyCursor(executionId, providerId, event); return event; } } - private applyCursor(executionId: string, event: SystemEvent): void { + private applyCursor(executionId: string, providerId: ProviderId, event: SystemEvent): void { if (event.subtype === "sdk_session_invalidated") { if (!this.threads.clearSdkSessionId(event.threadId)) throw new Error("Codex cursor thread is missing"); return; @@ -129,7 +134,7 @@ export class CanonicalCodexSystemErrorProjection { if (!cursor) return; if (!this.threads.updateSdkSessionId(event.threadId, cursor) || !this.canonical.recordNativeCursor(executionId, { - providerId: "codex", scope: "thread", value: cursor, provenance: "native", + providerId, scope: providerId === "codex" ? "thread" : "session", value: cursor, provenance: "native", })) throw new Error("Codex cursor could not be persisted for this execution"); } diff --git a/apps/server/src/features/agents/canonical/canonical-execution-semantic-writer.ts b/apps/server/src/features/agents/canonical/canonical-execution-semantic-writer.ts index 6b28c4bff..f098e4314 100644 --- a/apps/server/src/features/agents/canonical/canonical-execution-semantic-writer.ts +++ b/apps/server/src/features/agents/canonical/canonical-execution-semantic-writer.ts @@ -1,5 +1,6 @@ import type { Database } from "bun:sqlite"; import * as NodeCrypto from "node:crypto"; +import * as NodeUtil from "node:util"; import { AgentEventSchema, AgentEventType, @@ -9,7 +10,10 @@ import { PlanRecordSchema, PlanSectionNavSchema, ProviderIdSchema, + ProviderIdentitySchema, TurnOutcomeSchema, + type ProviderIdentity, + type ParentNarrativeRecoveryItem, type TurnOutcome, } from "@mcode/contracts"; import { z } from "zod"; @@ -26,6 +30,7 @@ import type { ExecutionWriteReceipt, ExecutionLivePublicationReceipt, ExecutionPlanQuestionsReceipt, + ExecutionTerminalPersistenceReceipt, } from "../execution/execution-worker-handler.js"; import type { CanonicalAgentEventPublisher } from "./canonical-agent-boundary.js"; import { CanonicalAgentBoundary } from "./canonical-agent-boundary.js"; @@ -33,9 +38,10 @@ import { CanonicalCommittedProviderProjector } from "./canonical-committed-provi import type { ProviderEventProjection } from "../../providers/composition/provider-event-adapter.js"; import { CanonicalCodexSystemErrorProjection, matchesCodexSystemIntents } from "./canonical-codex-system-error-projection.js"; import { CanonicalContextCompactionProjection } from "./canonical-context-compaction-projection.js"; -import { CanonicalParentTurnWrite, type DataOnlyParentLiveMessageInput } from "./canonical-parent-turn-write.js"; +import { CanonicalParentTurnWrite, type DataOnlyParentLiveMessageInput, type DataOnlyParentTurnFinishInput } from "./canonical-parent-turn-write.js"; import { TaskRepo } from "../orchestration/persistence/task-repo.js"; import { PlanRepo } from "../planning/persistence/plan-repo.js"; +import { HookExecutionRepo } from "../events/persistence/hook-execution-repo.js"; import type { TaskToolWriteIntent } from "../tasks/task-tool-intent-reducer.js"; import { ParentAssistantTextCheckpointService, @@ -47,6 +53,7 @@ import { const HEAD_ID = "semantic:head"; const HEAD_KIND = "semantic-head"; const PUBLICATION_KIND = "semantic-publication"; +const ACKNOWLEDGED_PUBLICATION_KIND = "semantic-publication-acknowledged"; const PENDING_FINISH_KIND = "semantic:finish-pending"; const PUBLICATION_CHUNK_SIZE = 64; const PUBLICATION_CHUNK_PAGE_SIZE = 16; @@ -54,6 +61,9 @@ const MAX_BUFFERED_PUBLICATION_EVENTS = 2_048; const MAX_LIVE_PUBLICATION_EVENTS = 64; const MAX_LIVE_PUBLICATION_BYTES = 256 * 1024; const MAX_LIVE_RECEIPT_BYTES = 512 * 1024; +const LIVE_PUBLICATION_KINDS: ReadonlySet = new Set([ + "begin", "append-events", "finish", "finish-live-event", "live-event", "post-terminal-event", +]); class RejectedCodexProjection extends Error { constructor(readonly diagnostic: Extract["diagnostic"]) { @@ -124,6 +134,7 @@ const storedHeadSchema = z.object({ providerOutcome: TurnOutcomeSchema.nullable(), assignedMessageId: z.string().regex(/^[0-9a-f]{64}$/).nullable().optional(), compacting: z.boolean().default(false), + ended: z.boolean().optional(), }); const storedReceiptSchema = z.object({ kind: z.literal("committed"), @@ -161,6 +172,9 @@ const storedOperationSchema = z.object({ input_hash: z.string(), receipt_json: z.string(), }); +const pendingFinishSchema = storedReceiptSchema.pick({ providerCommit: true }).extend({ + providerIdentities: z.array(ProviderIdentitySchema), +}); const storedPublicationChunkPageSchema = z.array(storedOperationSchema.extend({ operation_id: z.string() })) .max(PUBLICATION_CHUNK_PAGE_SIZE); const storedEnvelopeRowSchema = z.object({ envelope_json: z.string() }); @@ -179,6 +193,7 @@ interface SemanticHead { readonly providerOutcome: TurnOutcome | null; readonly assignedMessageId?: string | null; readonly compacting: boolean; + readonly ended?: boolean; } type StoredOperation = z.infer; @@ -193,19 +208,25 @@ export interface LostExecutionInterruption { readonly recoveryIncidentId: string; } -/** Adapts the supported execution mutations to one writer-local canonical SQLite connection. */ +/** + * Adapts execution mutations to one writer-local SQLite connection. + * Outer writes reserve the WAL writer before reading because the main connection may write concurrently. + */ export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter { private readonly turns: CanonicalParentTurnWrite; + private readonly compoundTurns: CanonicalParentTurnWrite; private readonly providerProjector: CanonicalCommittedProviderProjector; private readonly contextCompaction: CanonicalContextCompactionProjection; private readonly systemProjection: CanonicalCodexSystemErrorProjection; private readonly tasks: TaskRepo; private readonly plans: PlanRepo; private readonly threads: ThreadRepo; + private readonly hooks: HookExecutionRepo; private readonly assistantText: ParentAssistantTextCheckpointService; private readonly canonical: CanonicalAgentBoundary; private readonly findOperation: ReturnType; private readonly listPublicationChunks: ReturnType; + private readonly acknowledgePublication: ReturnType; private readonly findPublishedEvent: ReturnType; private readonly findDurableSequence: ReturnType; private readonly advanceLivePublication: ReturnType; @@ -222,6 +243,8 @@ export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter if (this.bufferedPublication) this.bufferPublication(events); else this.publish(events); }); + // Compound terminal batches publish from their durable sequence log only after the final receipt commits. + this.compoundTurns = new CanonicalParentTurnWrite(db, () => {}); const codexBoundary = new CanonicalAgentBoundary(db, (events) => { if (!this.bufferedPublication) throw new Error("Codex projection requires a semantic transaction"); this.bufferPublication(events); @@ -233,9 +256,11 @@ export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter this.tasks = new TaskRepo(db); this.plans = new PlanRepo(db); this.threads = new ThreadRepo(db); + this.hooks = new HookExecutionRepo(db); this.assistantText = new ParentAssistantTextCheckpointService(db); this.findOperation = db.prepare("SELECT kind, input_hash, receipt_json FROM canonical_writer_operation_receipts WHERE execution_id = ? AND operation_id = ?"); this.listPublicationChunks = db.prepare("SELECT operation_id, kind, input_hash, receipt_json FROM canonical_writer_operation_receipts WHERE execution_id = ? AND operation_id > ? AND operation_id < ? ORDER BY operation_id LIMIT ?"); + this.acknowledgePublication = db.prepare("UPDATE canonical_writer_operation_receipts SET kind = ? WHERE execution_id = ? AND operation_id = ? AND kind = ? AND input_hash = ?"); this.findPublishedEvent = db.prepare("SELECT envelope_json FROM canonical_agent_events WHERE execution_id = ? AND accepted_sequence = ?"); this.findDurableSequence = db.prepare("SELECT last_durable_sequence FROM canonical_agent_ingest_checkpoints WHERE execution_id = ?"); this.advanceLivePublication = db.prepare(`INSERT INTO canonical_writer_live_publication_heads (thread_id, last_sequence) @@ -254,11 +279,10 @@ export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter const hash = fingerprint(operation); const existing = this.existingReceipt(operation, hash); if (existing) { - if (existing.kind === "committed" && operation.mutation.kind === "finish") { + if (existing.kind === "committed" && isFinishMutation(operation.mutation)) { this.assistantText.retire(operation.execution.executionId); } - if (existing.kind === "committed" && operation.mutation.kind === "live-event" - && operation.mutation.text.kind === "reclassify") { + if (existing.kind === "committed" && parentLiveOperation(operation)?.mutation.text.kind === "reclassify") { this.assistantText.discardRecoveryJournal(operation.execution.executionId); } return existing; @@ -290,7 +314,7 @@ export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter try { const receipt = this.withBufferedPublication(() => this.db.transaction( () => this.writeLostInterruption(input, operation, hash), - )()); + ).immediate()); this.turns.retireInterruptedText(input.execution.executionId); return receipt; } catch (error) { @@ -319,6 +343,7 @@ export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter private async applySupported(operation: ExecutionSemanticOperation, hash: string): Promise { if (LIVE_RECOVERY_KINDS.has(operation.mutation.kind)) return this.applyLiveRecovery(operation, hash); + if (isFinishMutation(operation.mutation)) return await this.finish(operation, hash); switch (operation.mutation.kind) { case "begin": return this.begin(operation, hash); case "append-events": return this.append(operation, hash); @@ -326,7 +351,7 @@ export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter case "stop-requested": case "provider-outcome": return this.control(operation, hash); case "stage-terminal": return this.stageTerminal(operation, hash); - case "finish": return await this.finish(operation, hash); + case "post-terminal-event": return this.postTerminalEvent(operation, hash); default: return conflict(operation); } } @@ -335,7 +360,7 @@ export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter const row = this.loadOperation(operation.execution.executionId, operation.operationId); if (!row) return null; if (row.kind !== PENDING_FINISH_KIND) return this.replay(operation, hash, row); - return operation.mutation.kind === "finish" && row.input_hash === hash ? null : conflict(operation); + return isFinishMutation(operation.mutation) && row.input_hash === hash ? null : conflict(operation); } private begin(operation: ExecutionSemanticOperation, hash: string): ExecutionWriteReceipt { @@ -368,7 +393,7 @@ export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter this.insertOperation.run(operation.execution.executionId, HEAD_ID, HEAD_KIND, fingerprint(head.lease), JSON.stringify(head)); this.storePublicationChunks(operation.execution.executionId, hash, result.events.map((event) => event.acceptedSequence)); return this.storeReceipt(operation, hash, receipt); - })()); + }).immediate()); } private append(operation: ExecutionSemanticOperation, hash: string): ExecutionWriteReceipt { @@ -378,7 +403,7 @@ export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter || mutation.events.some((event) => event.routing.threadId !== operation.execution.threadId || event.routing.turnId !== operation.execution.turnId || event.routing.executionId !== operation.execution.executionId)) return conflict(operation); - return this.withBufferedPublication(() => this.db.transaction(() => { + const receipt = this.withBufferedPublication(() => this.db.transaction(() => { const head = this.requireNextHead(operation); if (!validPublicationProvider(operation, head.providerId)) throw new SemanticConflict(); const result = this.turns.append({ @@ -391,6 +416,7 @@ export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter const providerEvents = this.providerProjector.project(result.events); const compacting = this.contextCompaction.apply(operation.execution.threadId, head.compacting, providerEvents); if (compacting === null) throw new SemanticConflict(); + const live = this.appendParentLive(operation, head, providerEvents); const checkpoint = durableSequenceRowSchema.parse(this.findDurableSequence.get(operation.execution.executionId)); const providerCommit: ExecutionProviderCommitReceipt = { outcome: result.outcome, @@ -400,16 +426,42 @@ export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter durableThrough: result.durableThrough, eventCount: result.events.length, }; - const receipt = committed(operation, checkpoint.last_durable_sequence, providerCommit, providerEvents); - this.storeHead({ ...head, ordinal: operation.ordinal, durableRevision: receipt.durableRevision, compacting }); + const receipt = committed(operation, checkpoint.last_durable_sequence, providerCommit, live.providerEvents, + live.textResult, undefined, live.planQuestions, live.planOutput); + this.storeHead({ ...head, ordinal: operation.ordinal, durableRevision: receipt.durableRevision, compacting, + ...(mutation.parentLive?.message ? { assignedMessageId: mutation.parentLive.message.messageId } : {}) }); const publication = this.bufferedPublication?.flat() ?? []; // A Codex child write has its own execution sequence, even when the parent event caused it. this.storePublicationChunks(operation.execution.executionId, hash, publication.map((event) => ({ executionId: event.routing.executionId, sequence: event.acceptedSequence, }))); - return this.storeReceipt(operation, hash, receipt); - })()); + return this.storeReceipt(operation, hash, receipt, live.publication); + }).immediate()); + if (mutation.parentLive?.text.kind === "reclassify") { + this.assistantText.discardRecoveryJournal(operation.execution.executionId); + } + return receipt; + } + + private appendParentLive( + operation: ExecutionSemanticOperation, + head: SemanticHead, + providerEvents: readonly ProjectedCommittedProviderEvent[], + ) { + const live = parentLiveOperation(operation); + if (!live) return { providerEvents, textResult: undefined, planQuestions: undefined, + planOutput: undefined, publication: undefined }; + const publication = liveEventPublication(live); + const matches = providerEvents.filter((projected) => publication + && sameParentLiveEvent(projected.event, publication.event)); + if (matches.length !== 1) throw new SemanticConflict(); + this.requireUnfinishedCheckpoint(operation.execution); + return { + ...this.applyLiveFeatureEffects(live, head), + publication: this.projectLiveSystem(live, head.providerId), + providerEvents: providerEvents.filter((projected) => projected !== matches[0]), + }; } private commitCodexProjection(project: () => ProviderEventProjection): ProviderEventProjection { @@ -444,7 +496,7 @@ export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter const receipt = committed(operation, head.durableRevision, undefined, undefined, result); this.storeHead({ ...head, ordinal: operation.ordinal }); return this.storeReceipt(operation, hash, receipt); - })(); + }).immediate(); } private recordNarrativeDelta(operation: ExecutionSemanticOperation, hash: string): ExecutionWriteReceipt { @@ -457,7 +509,7 @@ export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter const receipt = committed(operation, head.durableRevision); this.storeHead({ ...head, ordinal: operation.ordinal }); return this.storeReceipt(operation, hash, receipt); - })(); + }).immediate(); } private applyLiveRecovery(operation: ExecutionSemanticOperation, hash: string): ExecutionWriteReceipt { @@ -471,29 +523,29 @@ export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter if (mutation.kind !== "live-event") return conflict(operation); const receipt = this.db.transaction(() => { const head = this.requireNextHead(operation); - if (head.providerId !== "codex") throw new SemanticConflict(); + if (!validPublicationProvider(operation, head.providerId)) throw new SemanticConflict(); this.requireUnfinishedCheckpoint(operation.execution); const { textResult, planQuestions, planOutput } = this.applyLiveFeatureEffects(operation, head); - const livePublication = this.projectLiveSystem(operation); + const livePublication = this.projectLiveSystem(operation, head.providerId); const committedReceipt = committed(operation, head.durableRevision, undefined, undefined, textResult, undefined, planQuestions, planOutput); this.storeHead({ ...head, ordinal: operation.ordinal, ...(mutation.message ? { assignedMessageId: mutation.message.messageId } : {}) }); return this.storeReceipt(operation, hash, committedReceipt, livePublication); - })(); + }).immediate(); if (mutation.text.kind === "reclassify") { this.assistantText.discardRecoveryJournal(operation.execution.executionId); } return receipt; } - private projectLiveSystem(operation: ExecutionSemanticOperation): readonly ExecutionLivePublicationIntent[] | undefined { + private projectLiveSystem(operation: ExecutionSemanticOperation, providerId: string): readonly ExecutionLivePublicationIntent[] | undefined { const mutation = operation.mutation; if (mutation.kind !== "live-event" || mutation.systemIntents === undefined) return undefined; - const publication = liveEventPublication(operation); + const publication = operation.livePublication?.length === 1 ? operation.livePublication[0] : undefined; if (publication?.event.type !== "system") throw new SemanticConflict(); const result = this.systemProjection.projectBoundSystem( - operation.execution, publication.event, mutation.systemIntents, publication.after, + operation.execution, providerId, publication.event, mutation.systemIntents, publication.after, ); return [{ after: result.after, event: result.event }]; } @@ -589,16 +641,22 @@ export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter private async finish(operation: ExecutionSemanticOperation, hash: string): Promise { const mutation = operation.mutation; - if (mutation.kind !== "finish" || !validFinish(operation, mutation)) return conflict(operation); + if (!isFinishMutation(mutation) || !validFinish(operation, mutation)) return conflict(operation); if (!this.validFinishHead(operation, mutation.input.providerId)) return conflict(operation); - this.reserveFinish(operation, hash); - this.publishStoredEvents(operation.execution.executionId, hash); - const result = await this.turns.finish(mutation.input, (batch) => { + const providerCommit = mutation.kind === "finish-live-event" + ? this.prepareLiveFinish(operation, hash, mutation) : undefined; + if (mutation.kind === "finish") { + this.reserveFinish(operation, hash); + this.publishStoredEvents(operation.execution.executionId, hash); + } + const turns = mutation.kind === "finish-live-event" ? this.compoundTurns : this.turns; + const input = this.terminalFinishInput(operation, mutation); + const result = await turns.finish(input, (batch) => { this.storePublicationChunks(operation.execution.executionId, hash, batch.publishedSequences); if (batch.terminal) { const current = this.requireNextHead(operation); if (current.providerId !== mutation.input.providerId) throw new SemanticConflict(); - const receipt = committed(operation, batch.durableSequence); + const receipt = committed(operation, batch.durableSequence, providerCommit, providerCommit ? [] : undefined); const status = mutation.outcome === "cancelled" ? "interrupted" : mutation.outcome; if (!this.threads.updateStatus(operation.execution.threadId, status)) throw new SemanticConflict(); this.storeHead({ ...current, ordinal: operation.ordinal, durableRevision: receipt.durableRevision, terminal: true }); @@ -606,13 +664,185 @@ export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter } }); if (result.outcome !== "committed") return conflict(operation); + return this.finishedReceipt(operation, hash); + } + + private finishedReceipt(operation: ExecutionSemanticOperation, hash: string): ExecutionWriteReceipt { const stored = this.loadOperation(operation.execution.executionId, operation.operationId); if (!stored) return conflict(operation); const receipt = this.readReceipt(operation, hash, stored); + if (operation.mutation.kind === "finish-live-event" && receipt.kind === "committed") { + this.publishStoredEvents(operation.execution.executionId, hash, true); + } if (receipt.kind === "committed") this.assistantText.retire(operation.execution.executionId); return receipt; } + private terminalFinishInput( + operation: ExecutionSemanticOperation, + mutation: Extract, + ): DataOnlyParentTurnFinishInput { + if (mutation.kind === "finish") return mutation.input; + const pending = this.loadOperation(operation.execution.executionId, operation.operationId); + if (!pending || pending.kind !== PENDING_FINISH_KIND) throw new SemanticConflict(); + const preparation = pendingFinishSchema.parse(JSON.parse(pending.receipt_json)); + return { ...mutation.input, providerIdentities: preparation.providerIdentities }; + } + + private terminalProviderIdentities(threadId: string, providerId: string): ProviderIdentity[] { + const canonicalThread = this.canonical.loadThread(threadId); + if (!canonicalThread || canonicalThread.providerId !== providerId) throw new SemanticConflict(); + const thread = this.threads.findById(threadId); + if (!thread?.sdk_session_id || thread.provider !== providerId) return [...canonicalThread.providerIdentities]; + return [{ providerId: canonicalThread.providerId, scope: providerId === "codex" ? "thread" : "session", + value: thread.sdk_session_id, provenance: "native" }]; + } + + private prepareLiveFinish( + operation: ExecutionSemanticOperation, + hash: string, + mutation: Extract, + ): ExecutionProviderCommitReceipt | undefined { + if (!validLiveFinish(operation.execution, mutation)) throw new SemanticConflict(); + const prepared = this.withBufferedPublication(() => this.db.transaction(() => { + const head = this.requireNextHead(operation); + const pending = this.loadOperation(operation.execution.executionId, operation.operationId); + if (pending) { + this.reserveFinish(operation, hash); + return committed(operation, head.durableRevision, pendingFinishSchema.parse(JSON.parse(pending.receipt_json)).providerCommit); + } + if (head.assignedMessageId && head.assignedMessageId !== mutation.projection.assistant.messageId) { + throw new SemanticConflict(); + } + const next = this.recordProviderOutcome(head, mutation.outcome); + const evidence = this.appendTerminalEvidence(operation, hash, mutation, next); + const staged = this.turns.stageTerminalProjection(mutation.projection); + this.storeHead({ ...evidence.head, assignedMessageId: staged.messageId }); + this.reserveFinish(operation, hash, evidence.providerCommit, + this.terminalProviderIdentities(operation.execution.threadId, head.providerId)); + return committed(operation, head.durableRevision, evidence.providerCommit); + }).immediate(), false); + if (prepared.kind !== "committed") throw new SemanticConflict(); + return prepared.providerCommit; + } + + private appendTerminalEvidence( + operation: ExecutionSemanticOperation, + hash: string, + mutation: Extract, + head: SemanticHead, + ): { head: SemanticHead; providerCommit: ExecutionProviderCommitReceipt | undefined } { + if (!mutation.providerEvent) return { head, providerCommit: undefined }; + const checkpoint = this.canonical.loadCheckpoint(operation.execution.executionId); + const result = this.turns.append({ ...operation.execution, ...mutation.providerEvent, + nativeCursor: mutation.providerEvent.nativeCursor ?? checkpoint?.nativeCursor ?? null }); + if (result.outcome !== "committed") throw new SemanticConflict(); + const projected = this.providerProjector.project(result.events); + if (!matchesTerminalProviderPublication(operation, projected)) throw new SemanticConflict(); + const compacting = this.contextCompaction.apply(operation.execution.threadId, head.compacting, projected); + if (compacting === null) throw new SemanticConflict(); + const publicationEvents = this.bufferedPublication?.flat() ?? []; + this.storePublicationChunks(operation.execution.executionId, hash, + publicationEvents.map((event) => event.acceptedSequence)); + return { head: { ...head, compacting, durableRevision: result.durableThrough }, providerCommit: { + outcome: result.outcome, conversationRevision: result.conversationRevision, rosterRevision: result.rosterRevision, + acceptedThrough: result.acceptedThrough, durableThrough: result.durableThrough, eventCount: result.events.length, + } }; + } + + private postTerminalEvent(operation: ExecutionSemanticOperation, hash: string): ExecutionWriteReceipt { + const mutation = operation.mutation; + if (mutation.kind !== "post-terminal-event") return conflict(operation); + return this.withBufferedPublication(() => this.db.transaction(() => { + const head = this.requireTerminalHead(operation); + this.requirePostTerminalPublication(operation, head); + const hookDrafts = this.persistTerminalHooks(operation, head, mutation.hooks ?? []); + const providerCommit = this.appendPostTerminalEvidence(operation, hookDrafts); + const live = parentLiveOperation(operation); + const publication = this.terminalHookPublication(operation) + ?? (live ? this.projectLiveSystem(live, head.providerId) : undefined); + return this.storePostTerminalReceipt(operation, hash, head, providerCommit, publication); + }).immediate()); + } + + private requirePostTerminalPublication(operation: ExecutionSemanticOperation, head: SemanticHead): void { + const event = operation.livePublication?.[0]?.event; + if (!event || !validPublicationProvider(operation, head.providerId)) throw new SemanticConflict(); + const checkpoint = this.canonical.loadCheckpoint(operation.execution.executionId); + if (!checkpoint?.terminalOutcome || !matchesLateOutcome(event, checkpoint.terminalOutcome)) throw new SemanticConflict(); + } + + private terminalHookPublication(operation: ExecutionSemanticOperation): readonly ExecutionLivePublicationIntent[] | undefined { + const mutation = operation.mutation; + const publication = operation.livePublication?.[0]; + if (mutation.kind !== "post-terminal-event" || publication?.event.type !== "hookCompleted") return undefined; + const hook = mutation.hooks?.[0]; + if (!hook) return undefined; + const message = this.canonical.loadTerminalProjection(operation.execution.turnId).message; + if (!message) throw new SemanticConflict(); + return [{ after: "terminal", event: { ...publication.event, + persistedMessageId: message.id, persistedHookId: hook.record.id } }]; + } + + private storePostTerminalReceipt(operation: ExecutionSemanticOperation, hash: string, head: SemanticHead, + providerCommit: ExecutionProviderCommitReceipt | undefined, publication: readonly ExecutionLivePublicationIntent[] | undefined): ExecutionWriteReceipt { + const receipt = committed(operation, providerCommit?.durableThrough ?? head.durableRevision, + providerCommit, providerCommit ? [] : undefined); + this.storeHead({ ...head, ordinal: operation.ordinal, durableRevision: receipt.durableRevision, + ...(operation.livePublication?.[0]?.event.type === "ended" ? { ended: true } : {}) }); + const events = this.bufferedPublication?.flat() ?? []; + this.storePublicationChunks(operation.execution.executionId, hash, events.map((entry) => entry.acceptedSequence)); + return this.storeReceipt(operation, hash, receipt, publication); + } + + private requireTerminalHead(operation: ExecutionSemanticOperation): SemanticHead { + const head = this.loadHead(operation.execution.executionId); + if (!head?.terminal || head.ended || head.ordinal + 1 !== operation.ordinal + || !sameExecutionAndLease(head, operation)) throw new SemanticConflict(); + return head; + } + + private persistTerminalHooks( + operation: ExecutionSemanticOperation, + head: SemanticHead, + hooks: readonly Extract[], + ): Extract["events"] { + if (hooks.length === 0) return []; + const message = this.canonical.loadTerminalProjection(operation.execution.turnId).message; + if (!message) throw new SemanticConflict(); + const drafts = []; + for (const hook of hooks) { + const record = hook.record; + const existing = this.db.prepare("SELECT message_id FROM hook_executions WHERE id = ?").get(record.id); + if (existing && !NodeUtil.isDeepStrictEqual(existing, { message_id: message.id })) throw new SemanticConflict(); + if (record.message_id && record.message_id !== message.id) throw new SemanticConflict(); + this.hooks.bulkCreate([{ id: record.id, messageId: message.id, hookName: record.hook_name, + toolName: record.tool_name, phase: record.phase, payload: record.payload, durationMs: record.duration_ms, + didBlock: record.did_block, startedAt: record.started_at, endedAt: record.ended_at, sortOrder: record.sort_order }], true); + drafts.push(terminalHookDraft(operation, head.providerId, { ...record, message_id: message.id })); + } + return drafts; + } + + private appendPostTerminalEvidence( + operation: ExecutionSemanticOperation, + hooks: Extract["events"], + ): ExecutionProviderCommitReceipt | undefined { + const mutation = operation.mutation; + if (mutation.kind !== "post-terminal-event") throw new SemanticConflict(); + const source = mutation.providerEvent; + const events = [...source ? source.events : [], ...hooks]; + if (events.length === 0) return undefined; + const result = this.turns.append({ ...operation.execution, events, + phase: source ? source.phase : "post-terminal", nativeCursor: null }); + if (result.outcome !== "committed") throw new SemanticConflict(); + const projected = this.providerProjector.project(result.events); + if (mutation.providerEvent && !matchesPostTerminalProviderPublication(operation, projected)) throw new SemanticConflict(); + return { outcome: result.outcome, conversationRevision: result.conversationRevision, rosterRevision: result.rosterRevision, + acceptedThrough: result.acceptedThrough, durableThrough: result.durableThrough, + eventCount: source ? source.events.length : 0 }; + } + private stageTerminal(operation: ExecutionSemanticOperation, hash: string): ExecutionWriteReceipt { const mutation = operation.mutation; if (mutation.kind !== "stage-terminal" || mutation.input.threadId !== operation.execution.threadId @@ -628,7 +858,7 @@ export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter this.storeHead({ ...head, ordinal: operation.ordinal, assignedMessageId: mutation.input.assistant.messageId ?? head.assignedMessageId ?? null }); return this.storeReceipt(operation, hash, receipt); - })(); + }).immediate(); } private control(operation: ExecutionSemanticOperation, hash: string): ExecutionWriteReceipt { @@ -638,7 +868,7 @@ export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter const receipt = committed(operation, head.durableRevision); this.storeHead({ ...next, ordinal: operation.ordinal }); return this.storeReceipt(operation, hash, receipt); - })(); + }).immediate(); } private applyControlMutation(head: SemanticHead, operation: ExecutionSemanticOperation): SemanticHead { @@ -723,13 +953,15 @@ export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter this.updateHead.run(JSON.stringify(head), head.execution.executionId, HEAD_ID, HEAD_KIND); } - private reserveFinish(operation: ExecutionSemanticOperation, hash: string): void { + private reserveFinish(operation: ExecutionSemanticOperation, hash: string, providerCommit?: ExecutionProviderCommitReceipt, + providerIdentities?: readonly ProviderIdentity[]): void { const existing = this.loadOperation(operation.execution.executionId, operation.operationId); if (existing) { if (existing.kind !== PENDING_FINISH_KIND || existing.input_hash !== hash) throw new SemanticConflict(); return; } - this.insertOperation.run(operation.execution.executionId, operation.operationId, PENDING_FINISH_KIND, hash, "{}"); + this.insertOperation.run(operation.execution.executionId, operation.operationId, PENDING_FINISH_KIND, hash, + JSON.stringify({ providerCommit, providerIdentities })); } private storePublicationChunks( @@ -766,9 +998,9 @@ export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter if (operation.livePublication && Buffer.byteLength(receiptJson, "utf8") > MAX_LIVE_RECEIPT_BYTES) { throw new SemanticConflict(); } - if (operation.mutation.kind === "finish") { + if (isFinishMutation(operation.mutation)) { const updated = this.commitPendingFinish.run( - "semantic:finish", receiptJson, operation.execution.executionId, operation.operationId, PENDING_FINISH_KIND, hash, + `semantic:${operation.mutation.kind}`, receiptJson, operation.execution.executionId, operation.operationId, PENDING_FINISH_KIND, hash, ); if (updated.changes !== 1) throw new SemanticConflict(); return storedReceipt; @@ -799,7 +1031,9 @@ export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter private replay(operation: ExecutionSemanticOperation, hash: string, stored: StoredOperation): ExecutionWriteReceipt { const receipt = this.readReceipt(operation, hash, stored); - if (receipt.kind === "committed") this.publishStoredEvents(operation.execution.executionId, hash); + if (receipt.kind === "committed") { + this.publishStoredEvents(operation.execution.executionId, hash, operation.mutation.kind === "finish-live-event"); + } return receipt; } @@ -813,21 +1047,46 @@ export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter || !this.matchesLivePublicationReceipt(operation, receipt.livePublication)) { throw new Error("Live publication receipt does not match its operation"); } - return receipt.operationId === operation.operationId && Number.isSafeInteger(receipt.durableRevision) - ? committed(operation, receipt.durableRevision, receipt.providerCommit, receipt.providerEvents, - receipt.assistantTextCheckpoint, receipt.livePublication, receipt.planQuestions, receipt.planOutput) : conflict(operation); + if (receipt.operationId !== operation.operationId || !Number.isSafeInteger(receipt.durableRevision)) { + return conflict(operation); + } + const committedReceipt = committed(operation, receipt.durableRevision, receipt.providerCommit, receipt.providerEvents, + receipt.assistantTextCheckpoint, receipt.livePublication, receipt.planQuestions, receipt.planOutput); + return operation.mutation.kind === "finish-live-event" + ? { ...committedReceipt, terminalPersistence: this.terminalPersistence(operation) } : committedReceipt; + } + + private terminalPersistence(operation: ExecutionSemanticOperation): ExecutionTerminalPersistenceReceipt { + const mutation = operation.mutation; + if (mutation.kind !== "finish-live-event") throw new SemanticConflict(); + const persisted = this.canonical.loadTerminalProjection(operation.execution.turnId); + const message = persisted.message; + if (message && !matchesCommittedTerminalMessage(message, operation.execution, mutation.outcome)) { + throw new SemanticConflict(); + } + const fileEvidence = mutation.input.fileEvidence; + return { + messageId: message?.id ?? null, + outcome: mutation.outcome, + toolCallCount: persisted.toolCallCount, + filesChanged: [...fileEvidence?.filesChanged ?? []], + ...(fileEvidence ? { fileEffects: fileEvidence.fileEffects } : {}), + }; } private matchesLivePublicationReceipt( operation: ExecutionSemanticOperation, actual: readonly ExecutionLivePublicationReceipt[] | undefined, ): boolean { - const mutation = operation.mutation; - if (mutation.kind === "live-event" && mutation.systemIntents?.[0]?.kind === "system-notice") { + const hookPublication = this.terminalHookPublication(operation); + if (hookPublication) { + return publicationFingerprint(actual) === publicationFingerprint(hookPublication); + } + const mutation = parentLiveOperation(operation)?.mutation; + if (mutation?.systemIntents?.[0]?.kind === "system-notice") { return this.matchesGeneratedNoticeReceipt(operation, actual); } - return fingerprint(actual?.map(({ after, event }) => ({ after, event })) ?? null) - === fingerprint(livePublicationFor(operation) ?? null); + return publicationFingerprint(actual) === publicationFingerprint(livePublicationFor(operation)); } private matchesGeneratedNoticeReceipt( @@ -842,7 +1101,7 @@ export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter === fingerprint(expected); } - private publishStoredEvents(executionId: string, hash: string): void { + private publishStoredEvents(executionId: string, hash: string, acknowledge = false): void { const prefix = publicationChunkPrefix(hash); let cursor = prefix; while (true) { @@ -851,17 +1110,29 @@ export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter )); if (chunks.length === 0) return; for (const chunk of chunks) { - if (chunk.kind !== PUBLICATION_KIND || chunk.input_hash !== hash) throw new Error("Semantic publication chunk mismatch"); - const sequences = storedPublicationSequencesSchema.parse(JSON.parse(chunk.receipt_json)); - const events = sequences.map((sequence) => typeof sequence === "number" - ? this.loadPublishedEvent(executionId, sequence) - : this.loadPublishedEvent(sequence.executionId, sequence.sequence)); - this.publish(events); + this.publishStoredChunk(executionId, hash, chunk, acknowledge); } cursor = chunks[chunks.length - 1]!.operation_id; } } + private publishStoredChunk( + executionId: string, + hash: string, + chunk: z.infer[number], + acknowledge: boolean, + ): void { + if (acknowledge && chunk.kind === ACKNOWLEDGED_PUBLICATION_KIND && chunk.input_hash === hash) return; + if (chunk.kind !== PUBLICATION_KIND || chunk.input_hash !== hash) throw new Error("Semantic publication chunk mismatch"); + const sequences = storedPublicationSequencesSchema.parse(JSON.parse(chunk.receipt_json)); + const events = sequences.map((sequence) => typeof sequence === "number" + ? this.loadPublishedEvent(executionId, sequence) + : this.loadPublishedEvent(sequence.executionId, sequence.sequence)); + this.publish(events); + if (acknowledge) this.acknowledgePublication.run(ACKNOWLEDGED_PUBLICATION_KIND, executionId, + chunk.operation_id, PUBLICATION_KIND, hash); + } + private loadPublishedEvent(executionId: string, sequence: number) { const row = storedEnvelopeRowSchema.nullable().parse(this.findPublishedEvent.get(executionId, sequence)); if (!row) throw new Error(`Semantic publication event missing at ${executionId}:${sequence}`); @@ -877,7 +1148,7 @@ export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter this.bufferedPublicationCount += events.length; } - private withBufferedPublication(write: () => ExecutionWriteReceipt): ExecutionWriteReceipt { + private withBufferedPublication(write: () => ExecutionWriteReceipt, publishAfterCommit = true): ExecutionWriteReceipt { if (this.bufferedPublication) throw new Error("Nested semantic publication buffer"); const pending: Parameters[0][] = []; this.bufferedPublication = pending; @@ -885,7 +1156,7 @@ export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter try { const result = write(); this.bufferedPublication = null; - for (const events of pending) this.publish(events); + if (publishAfterCommit) for (const events of pending) this.publish(events); return result; } finally { this.bufferedPublication = null; @@ -907,7 +1178,7 @@ function validLivePublication(operation: ExecutionSemanticOperation): boolean { const publication = operation.livePublication; if (publication === undefined) return true; if (!validLivePublicationShape(publication, operation.mutation.kind)) return false; - const barrier = operation.mutation.kind === "finish" ? "terminal" : "writer"; + const barrier = isFinishMutation(operation.mutation) || operation.mutation.kind === "post-terminal-event" ? "terminal" : "writer"; let bytes = 0; for (const intent of publication) { if (!validLivePublicationIntent(intent, operation, barrier)) return false; @@ -923,8 +1194,8 @@ function validLivePublicationShape( ): boolean { return Array.isArray(publication) && publication.length > 0 && publication.length <= MAX_LIVE_PUBLICATION_EVENTS && publication.every((intent) => intent.event.publicationId === undefined) - && (mutationKind === "begin" || mutationKind === "append-events" || mutationKind === "finish" - || mutationKind === "live-event" && publication.length === 1); + && LIVE_PUBLICATION_KINDS.has(mutationKind) + && (mutationKind === "live-event" || mutationKind === "post-terminal-event" ? publication.length === 1 : true); } function validLivePublicationIntent( @@ -936,7 +1207,7 @@ function validLivePublicationIntent( && intent.event.threadId === operation.execution.threadId && intent.event.turnExecutionId === operation.execution.executionId && (operation.mutation.kind !== "begin" || intent.event.type === "turnStarted") - && (intent.event.type !== "turnStarted" || operation.mutation.kind === "begin") + && validTurnStartedPublication(intent.event, operation.mutation) && (barrier === "terminal" || !isTerminalLiveEvent(intent.event.type)); } @@ -945,20 +1216,140 @@ function isTerminalLiveEvent(type: ExecutionLivePublicationIntent["event"]["type } function validPublicationProvider(operation: ExecutionSemanticOperation, providerId: string): boolean { - return operation.livePublication === undefined || providerId === "codex"; + return operation.livePublication === undefined || ["codex", "claude", "cursor"].includes(providerId); +} + +function validTurnStartedPublication(event: ExecutionLivePublicationIntent["event"], mutation: ExecutionSemanticOperation["mutation"]): boolean { + return event.type !== "turnStarted" || mutation.kind === "begin" + || mutation.kind === "append-events" && mutation.parentLive !== undefined; +} + +function validPostTerminalInput(operation: ExecutionSemanticOperation): boolean { + const mutation = operation.mutation; + const publication = operation.livePublication; + if (mutation.kind !== "post-terminal-event" || publication?.length !== 1) return false; + const event = publication[0]?.event; + if (!event || !AgentEventSchema().safeParse(event).success) return false; + return validTerminalProviderEvent(operation.execution, mutation.providerEvent) + && validPostTerminalEffects(mutation, event) + && Buffer.byteLength(JSON.stringify(operation), "utf8") <= ACTIVE_TURN_WRITE_BATCH_LIMITS.maxBytes; +} + +function validPostTerminalEffects( + mutation: Extract, + event: ExecutionLivePublicationIntent["event"], +): boolean { + const hooks = mutation.hooks ?? []; + if (!Array.isArray(hooks) || hooks.length > 1) return false; + if (event.type === "system") { + return hooks.length === 0 && validPostTerminalSystem(event, mutation.systemIntents); + } + if (mutation.systemIntents !== undefined) return false; + if (event.type === "ended" || event.type === "hookProgress") return hooks.length === 0; + return validLateHookRecords(event, hooks); +} + +function validPostTerminalSystem( + event: Extract, + intents: Extract["systemIntents"], +): boolean { + return Array.isArray(intents) && intents.every((intent) => intent.kind !== "session-cursor") + && matchesCodexSystemIntents(event, intents); +} + +function validLateHookRecords( + event: ExecutionLivePublicationIntent["event"], + hooks: readonly Extract[], +): boolean { + if (event.type !== "hookStarted" && event.type !== "hookCompleted") return false; + if (event.type === "hookCompleted" && (event.persistedMessageId || event.persistedHookId)) return false; + if (event.type === "hookStarted" && hooks.length !== 1) return false; + return hooks.every((hook) => ParentNarrativeRecoveryItemSchema().safeParse(hook).success + && hook.kind === "hook" && hook.record.hook_name === event.hookName + && validLateHookRecord(event, hook.record)); +} + +function validLateHookRecord( + event: Extract, + record: Extract["record"], +): boolean { + if (!Number.isSafeInteger(record.sort_order) || record.sort_order < 0 || !Number.isFinite(Date.parse(record.started_at))) return false; + if (event.type === "hookStarted") return record.ended_at === null && record.phase === "stop"; + return record.ended_at !== null && Number.isFinite(Date.parse(record.ended_at)) + && record.duration_ms === event.durationMs && record.did_block === event.didBlock; +} + +function matchesLateOutcome(event: ExecutionLivePublicationIntent["event"], outcome: TurnOutcome): boolean { + return event.type !== "ended" || event.outcome === undefined + || normalizeTerminalOutcome(event.outcome) === normalizeTerminalOutcome(outcome); +} + +function matchesPostTerminalProviderPublication( + operation: ExecutionSemanticOperation, + projected: readonly ProjectedCommittedProviderEvent[], +): boolean { + const event = operation.livePublication?.[0]?.event; + return projected.length === 1 && event !== undefined + && (event.type === "ended" ? sameTerminalEvent(projected[0]?.event, event) + : NodeUtil.isDeepStrictEqual(projected[0]?.event, event)); +} + +function terminalHookDraft( + operation: ExecutionSemanticOperation, + providerId: string, + record: Extract["record"], +): Extract["events"][number] { + const itemId = `hook:${record.id}`; + return { eventId: `${operation.execution.executionId}:post-terminal:${operation.ordinal}:${itemId}`, + routing: { ...operation.execution, itemId }, sourceProviderId: providerId, sourceIdentities: [], + payload: { type: "item.recorded", item: { id: itemId, threadId: operation.execution.threadId, + turnId: operation.execution.turnId, kind: "system", providerIdentities: [], + payload: { projection: "hook", record }, createdAt: record.started_at, updatedAt: record.ended_at ?? record.started_at } } }; } function validSemanticMutationInput(operation: ExecutionSemanticOperation): boolean { - if (operation.mutation.kind === "append-assistant-text") { - return validAssistantTextInput(operation.mutation.inputs, operation.execution); + switch (operation.mutation.kind) { + case "post-terminal-event": return validPostTerminalInput(operation); + case "finish-live-event": return validLiveFinish(operation.execution, operation.mutation) + && validTerminalPublicationOutcome(operation.mutation, operation.livePublication); + case "append-assistant-text": return validAssistantTextInput(operation.mutation.inputs, operation.execution); + case "narrative-delta": return validNarrativeDeltaInput(operation.mutation.input, operation.execution); + case "live-event": return validLiveEventInput(operation); + case "append-events": return validAppendParentInput(operation); + default: return true; } - if (operation.mutation.kind === "narrative-delta") { - return validNarrativeDeltaInput(operation.mutation.input, operation.execution); +} + +function validAppendParentInput(operation: ExecutionSemanticOperation): boolean { + if (operation.mutation.kind === "append-events" && operation.mutation.parentLive !== undefined) { + const live = parentLiveOperation(operation); + return live !== null && validLiveEventInput(live) + && Buffer.byteLength(JSON.stringify(operation), "utf8") <= ACTIVE_TURN_WRITE_BATCH_LIMITS.maxBytes; } - if (operation.mutation.kind === "live-event") return validLiveEventInput(operation); return true; } +function parentLiveOperation(operation: ExecutionSemanticOperation): + (ExecutionSemanticOperation & { mutation: Extract }) | null { + const mutation = operation.mutation; + if (mutation.kind === "live-event") return { ...operation, mutation }; + if (mutation.kind === "post-terminal-event") { + return { ...operation, mutation: { kind: "live-event", text: { kind: "unchanged" }, systemIntents: mutation.systemIntents } }; + } + return mutation.kind === "append-events" && mutation.parentLive + ? { ...operation, mutation: { kind: "live-event", ...mutation.parentLive } } : null; +} + +function sameParentLiveEvent( + source: ExecutionLivePublicationIntent["event"], + publication: ExecutionLivePublicationIntent["event"], +): boolean { + if (source.type === "message" && publication.type === "message") { + return NodeUtil.isDeepStrictEqual({ ...source, messageId: undefined }, { ...publication, messageId: undefined }); + } + return NodeUtil.isDeepStrictEqual(source, publication); +} + function validLiveEventInput(operation: ExecutionSemanticOperation): boolean { const mutation = operation.mutation; if (mutation.kind !== "live-event") return false; @@ -1065,7 +1456,7 @@ function validLiveTextAssociation( event: ExecutionLivePublicationIntent["event"], ): boolean { switch (text.kind) { - case "unchanged": return event.type === "system" || validNarrativeEvent(event); + case "unchanged": return unchangedTextEvent(event); case "append": return validAppendEvent(event, text.inputs); case "reclassify": return event.type === "assistantMessageBoundary" && event.isFinalResponse === false && text.expectedText.length > 0; @@ -1175,7 +1566,7 @@ function validLease(lease: ExecutionLease): boolean { function validFinish( operation: ExecutionSemanticOperation, - mutation: Extract, + mutation: Extract, ): boolean { return mutation.outcome === mutation.input.outcome && mutation.input.threadId === operation.execution.threadId @@ -1183,6 +1574,77 @@ function validFinish( && mutation.input.executionId === operation.execution.executionId; } +function unchangedTextEvent(event: ExecutionLivePublicationIntent["event"]): boolean { + return ["system", "turnStarted", "generatedAttachment", "contextEstimate", "compacting", "compactSummary"].includes(event.type) + || validNarrativeEvent(event); +} + +function isFinishMutation(mutation: ExecutionSemanticOperation["mutation"]): mutation is Extract< + ExecutionSemanticOperation["mutation"], { kind: "finish" | "finish-live-event" } +> { + return mutation.kind === "finish" || mutation.kind === "finish-live-event"; +} + +function validLiveFinish( + execution: ExecutionIdentity, + mutation: Extract, +): boolean { + return mutation.projection.threadId === execution.threadId + && mutation.projection.executionId === execution.executionId + && mutation.projection.outcome === mutation.outcome + && "kind" in mutation.input.projection + && mutation.input.projection.kind === "writer-staged" + && mutation.input.projection.messageId === mutation.projection.assistant.messageId + && validTerminalProviderEvent(execution, mutation.providerEvent); +} + +function validTerminalProviderEvent( + execution: ExecutionIdentity, + input: Extract["providerEvent"], +): boolean { + return input === undefined || Boolean(input.phase && input.phase.length <= 64 && input.events.length > 0 + && input.events.every((event) => event.routing.threadId === execution.threadId + && event.routing.turnId === execution.turnId && event.routing.executionId === execution.executionId)); +} + +function sameTerminalEvent( + source: ExecutionLivePublicationIntent["event"] | undefined, + publication: ExecutionLivePublicationIntent["event"], +): boolean { + if (!source || !isTerminalLiveEvent(source.type)) return false; + const normalized = source.type === "ended" && source.outcome === "cancelled" + ? { ...source, outcome: "interrupted" } : source; + return NodeUtil.isDeepStrictEqual(normalized, publication); +} + +function matchesTerminalProviderPublication( + operation: ExecutionSemanticOperation, + projected: readonly ProjectedCommittedProviderEvent[], +): boolean { + const publication = operation.livePublication?.[0]; + return projected.length === 1 && operation.livePublication?.length === 1 && publication !== undefined + && sameTerminalEvent(projected[0]?.event, publication.event); +} + +function validTerminalPublicationOutcome( + mutation: Extract, + publications: readonly ExecutionLivePublicationIntent[] | undefined, +): boolean { + if (publications?.length !== 1) return false; + const event = publications[0]?.event; + switch (event?.type) { + case "turnComplete": return mutation.outcome === "completed"; + case "error": return mutation.outcome === "errored" && mutation.input.error === event.error; + case "ended": return event.outcome === undefined + || normalizeTerminalOutcome(event.outcome) === normalizeTerminalOutcome(mutation.outcome); + default: return false; + } +} + +function normalizeTerminalOutcome(outcome: TurnOutcome): Exclude { + return outcome === "cancelled" ? "interrupted" : outcome; +} + function nextHead(head: SemanticHead, operation: ExecutionSemanticOperation): boolean { const lease = head.lease; const candidate = operation.lease; @@ -1198,7 +1660,7 @@ function validLostExecutionInput(input: LostExecutionInterruption, operation: Ex && Boolean(input.reason && input.recoveryIncidentId) && operation.operationId.length <= 256; } -function sameExecutionAndLease(head: SemanticHead, input: LostExecutionInterruption): boolean { +function sameExecutionAndLease(head: SemanticHead, input: Pick): boolean { return head.execution.threadId === input.execution.threadId && head.execution.turnId === input.execution.turnId && head.execution.executionId === input.execution.executionId @@ -1235,6 +1697,16 @@ function committed( }; } +function matchesCommittedTerminalMessage( + message: NonNullable["message"]>, + execution: ExecutionIdentity, + outcome: TurnOutcome, +): boolean { + return message.thread_id === execution.threadId && message.role === "assistant" + && !message.is_internal && message.outcomeExecutionId === execution.executionId + && message.outcome === outcome; +} + function livePublicationFor(operation: ExecutionSemanticOperation): readonly ExecutionLivePublicationIntent[] | undefined { return operation.livePublication?.map((intent) => ({ after: intent.after, @@ -1260,6 +1732,10 @@ function conflict(operation: ExecutionSemanticOperation): Extract ({ after, event })) ?? null); +} + function fingerprint(value: unknown): string { return NodeCrypto.createHash("sha256").update(JSON.stringify(sortJson(value))).digest("hex"); } diff --git a/apps/server/src/features/agents/canonical/canonical-parent-turn-write.ts b/apps/server/src/features/agents/canonical/canonical-parent-turn-write.ts index 0dc3b021b..598b27f27 100644 --- a/apps/server/src/features/agents/canonical/canonical-parent-turn-write.ts +++ b/apps/server/src/features/agents/canonical/canonical-parent-turn-write.ts @@ -284,12 +284,13 @@ export class CanonicalParentTurnWrite { private createLiveAssistant(threadId: string, input: DataOnlyParentLiveMessageInput): boolean { const preceding = this.messages.findByIdInThreadIncludingInternal(threadId, input.precedingMessageId); - if (!preceding || preceding.sequence !== this.messages.getLatestSequenceIncludingInternal(threadId)) return false; + if (!preceding || !this.messages.isLatestNonSystemMessage(threadId, preceding.id)) return false; + const sequence = this.messages.getLatestSequenceIncludingInternal(threadId) + 1; this.messages.createAssistantIdempotent({ id: input.messageId, threadId, content: input.content, - sequence: preceding.sequence + 1, + sequence, model: input.model, attachments: [...input.attachments], isInternal: true, @@ -427,7 +428,10 @@ export class CanonicalParentTurnWrite { ...input, projectTurn: () => projection, finalizeCompatibility: () => { - if (!projection.message) return; + if (!projection.message) { + this.markEmptyTerminalFileChanges(input.threadId, fileEvidence); + return; + } this.messages.setAssistantOutcome(projection.message.id, input.outcome, input.executionId); this.messages.publishAssistant(projection.message.id); if (fileEvidence?.snapshot) { @@ -452,6 +456,12 @@ export class CanonicalParentTurnWrite { }, onBatchWrite); } + private markEmptyTerminalFileChanges(threadId: string, evidence?: PreparedExecutionFileEvidence): void { + if (evidence && (evidence.fileEffects.fileCount > 0 || evidence.filesChanged.length > 0)) { + this.markThreadFilesChanged.run(threadId); + } + } + private loadStagedTerminalProjection(threadId: string, executionId: string, messageId?: string): ParentTurnProjection { if (messageId !== undefined && !/^[0-9a-f]{64}$/.test(messageId)) { throw new Error("Invalid staged assistant message identity"); @@ -488,7 +498,7 @@ function selectTerminalDiff(input: DataOnlyParentTurnFinishInput, hasAssistant: function assertExecutionFileEvidence(input: DataOnlyParentTurnFinishInput, hasAssistant: boolean): void { const evidence = input.fileEvidence; - if (!evidence || !hasAssistant || input.selectedTurnDiff + if (!evidence || input.selectedTurnDiff || (!hasAssistant && evidence.selectedTurnDiff) || !validExecutionFileEvidence(evidence, input.threadId, input.turnId, input.executionId, input.deliveryAttempt)) { throw new Error("Invalid execution file evidence for terminal assistant"); diff --git a/apps/server/src/features/agents/canonical/execution-live-publication-release.ts b/apps/server/src/features/agents/canonical/execution-live-publication-release.ts index a845b0ca2..424073dd3 100644 --- a/apps/server/src/features/agents/canonical/execution-live-publication-release.ts +++ b/apps/server/src/features/agents/canonical/execution-live-publication-release.ts @@ -85,15 +85,44 @@ function contiguousPublicationIds(entries: readonly ExecutionLivePublicationRece } function samePublishedEvent(operation: ExecutionSemanticOperation, actual: AgentEvent, expected: AgentEvent): boolean { - if (NodeUtil.isDeepStrictEqual(actual, expected)) return true; - if (operation.mutation.kind !== "live-event" || operation.mutation.systemIntents?.[0]?.kind !== "system-notice" - || actual.type !== "system" || expected.type !== "system" || expected.messageId - || !actual.messageId) return false; - const { messageId: _generatedId, ...withoutGeneratedId } = actual; - return NodeUtil.isDeepStrictEqual(withoutGeneratedId, expected); + // Writer receipts may arrive from a live object or JSON replay. Compare the same wire shape in both cases. + const wireActual = AgentEventSchema().parse(JSON.parse(JSON.stringify(actual))); + const wireExpected = AgentEventSchema().parse(JSON.parse(JSON.stringify(expected))); + if (NodeUtil.isDeepStrictEqual(wireActual, wireExpected)) return true; + if (isPersistedTerminalHook(operation, wireActual)) { + const { persistedMessageId: _messageId, persistedHookId: _hookId, ...original } = wireActual; + return NodeUtil.isDeepStrictEqual(original, wireExpected); + } + if (!isGeneratedSystemNotice(operation, wireActual, wireExpected)) return false; + const { messageId: _generatedId, ...withoutGeneratedId } = wireActual; + return NodeUtil.isDeepStrictEqual(withoutGeneratedId, wireExpected); +} + +function isPersistedTerminalHook(operation: ExecutionSemanticOperation, actual: AgentEvent): + actual is Extract { + return operation.mutation.kind === "post-terminal-event" && actual.type === "hookCompleted" + && Boolean(actual.persistedMessageId) + && typeof actual.persistedHookId === "string" + && actual.persistedHookId === operation.mutation.hooks?.[0]?.record.id; +} + +function isGeneratedSystemNotice( + operation: ExecutionSemanticOperation, + actual: AgentEvent, + expected: AgentEvent, +): actual is Extract { + const systemIntents = systemIntentsFor(operation.mutation); + return systemIntents?.[0]?.kind === "system-notice" + && actual.type === "system" && expected.type === "system" + && !expected.messageId && Boolean(actual.messageId); +} + +function systemIntentsFor(mutation: ExecutionSemanticOperation["mutation"]) { + if (mutation.kind === "live-event" || mutation.kind === "post-terminal-event") return mutation.systemIntents; + return mutation.kind === "append-events" ? mutation.parentLive?.systemIntents : undefined; } function barrierFor(kind: ExecutionSemanticOperation["mutation"]["kind"]): "writer" | "terminal" | null { if (kind === "begin" || kind === "append-events" || kind === "live-event") return "writer"; - return kind === "finish" ? "terminal" : null; + return kind === "finish" || kind === "finish-live-event" || kind === "post-terminal-event" ? "terminal" : null; } diff --git a/apps/server/src/features/agents/canonical/execution-plan-question-release.ts b/apps/server/src/features/agents/canonical/execution-plan-question-release.ts index 55a6f757f..6a46bfaf3 100644 --- a/apps/server/src/features/agents/canonical/execution-plan-question-release.ts +++ b/apps/server/src/features/agents/canonical/execution-plan-question-release.ts @@ -20,8 +20,7 @@ export class ExecutionPlanQuestionRelease { const batch = receipt.planQuestions; if (!matchesOperation(operation, receipt, batch) || !PlanQuestionBatchSchema().safeParse({ threadId: batch.threadId, questions: batch.questions, - }).success || !NodeUtil.isDeepStrictEqual(batch.questions, operation.mutation.kind === "live-event" - ? operation.mutation.planQuestions : undefined)) { + }).success || !NodeUtil.isDeepStrictEqual(batch.questions, planQuestionsFor(operation))) { throw new Error("Plan-question publication receipt is invalid"); } return batch; @@ -44,9 +43,13 @@ function matchesOperation( receipt: Extract, batch: ExecutionPlanQuestionsReceipt, ): boolean { - return receipt.operationId === operation.operationId && operation.mutation.kind === "live-event" - && Boolean(operation.mutation.planQuestions) + return receipt.operationId === operation.operationId && Boolean(planQuestionsFor(operation)) && operation.livePublication?.[0]?.event.type === "textDelta" && batch.publicationId === `${operation.operationId}:plan-questions` && batch.threadId === operation.execution.threadId; } + +function planQuestionsFor(operation: ExecutionSemanticOperation): readonly PlanQuestion[] | undefined { + if (operation.mutation.kind === "live-event") return operation.mutation.planQuestions; + return operation.mutation.kind === "append-events" ? operation.mutation.parentLive?.planQuestions : undefined; +} diff --git a/apps/server/src/features/agents/composition/__tests__/agent-service-container.test.ts b/apps/server/src/features/agents/composition/__tests__/agent-service-container.test.ts index 432417ee3..ed357640a 100644 --- a/apps/server/src/features/agents/composition/__tests__/agent-service-container.test.ts +++ b/apps/server/src/features/agents/composition/__tests__/agent-service-container.test.ts @@ -2,30 +2,57 @@ import "reflect-metadata"; import * as NodeFS from "node:fs"; import * as NodeOS from "node:os"; import * as NodePath from "node:path"; -import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import * as NodeEvents from "node:events"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import type { Database } from "bun:sqlite"; +import type { WebSocket } from "ws"; import { container } from "tsyringe"; +import type { AgentEvent, IAgentProvider, IProviderRegistry, TurnRequest } from "@mcode/contracts"; import { setupContainer } from "../../../../application/composition/container.js"; +import { WorkerOwnedTurnRuntime } from "../../execution/worker-owned-turn-runtime.js"; +import { ExecutionThreadWorkerPort } from "../../execution/execution-worker-port.js"; +import { CanonicalAgentWriterClient } from "../../canonical/canonical-agent-writer-client.js"; +import { CanonicalExecutionWriterPort } from "../../canonical/canonical-execution-writer-port.js"; +import { ExecutionMailboxOwner } from "../../execution/execution-mailbox-owner.js"; +import { ExecutionMailboxScheduler } from "../../execution/execution-mailbox-scheduler.js"; +import { ExecutionProviderEventOwnership } from "../../execution/execution-provider-event-ownership.js"; +import { ExecutionWorkerLossCoordinator } from "../../execution/execution-worker-loss-coordinator.js"; +import { ExecutionFileEvidenceCoordinator } from "../../execution/execution-file-evidence-coordinator.js"; import { AgentService } from "../../orchestration/agent-service.js"; +import { AgentEventPublicationRegistry } from "../../orchestration/agent-event-publication-registry.js"; +import { TurnRuntimeController } from "../../orchestration/turn-runtime-controller.js"; import { ThreadCreationCoordinator } from "../../turns/thread-creation-coordinator.js"; import { TURN_FEATURE_EFFECTS, TurnFeatureEffects } from "../../turns/turn-feature-effects.js"; +import { WorkspaceRepo } from "../../../projects/persistence/workspace-repo.js"; +import { ThreadRepo } from "../../../thread-control/persistence/thread-repo.js"; +import { MessageRepo } from "../../conversation/persistence/message-repo.js"; +import { ProviderAvailabilityService } from "../../../providers/availability/provider-availability-service.js"; +import { addClient, removeClient } from "../../../../application/transport/push.js"; describe("AgentService container composition", () => { let database: Database | undefined; + let workerRuntime: WorkerOwnedTurnRuntime | undefined; let temporaryDirectory: string | undefined; + let pushClient: WebSocket | undefined; const previousDatabasePath = process.env.MCODE_DB_PATH; - beforeEach(() => { + beforeEach(async () => { container.reset(); temporaryDirectory = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "mcode-agent-service-composition-")); process.env.MCODE_DB_PATH = NodePath.join(temporaryDirectory, "mcode.db"); setupContainer(temporaryDirectory); database = container.resolve("Database"); + workerRuntime = container.resolve(WorkerOwnedTurnRuntime); + await workerRuntime.whenReady(); }); - afterEach(() => { - database?.close(); + afterEach(async () => { + if (pushClient) removeClient(pushClient); + pushClient = undefined; + await workerRuntime?.close(); + workerRuntime = undefined; + database?.close(true); database = undefined; container.reset(); if (previousDatabasePath === undefined) delete process.env.MCODE_DB_PATH; @@ -36,9 +63,26 @@ describe("AgentService container composition", () => { it("resolves AgentService through the production container with explicit feature effects", () => { expect(container.resolve(TURN_FEATURE_EFFECTS)).toBeInstanceOf(TurnFeatureEffects); + expect(container.resolve(ExecutionFileEvidenceCoordinator)) + .toBe(container.resolve(ExecutionFileEvidenceCoordinator)); expect(container.resolve(AgentService)).toBeInstanceOf(AgentService); }); + it("starts one writer and a fixed execution pool from the opened database", async () => { + workerRuntime = container.resolve(WorkerOwnedTurnRuntime); + await workerRuntime.whenReady(); + + expect(container.resolve(CanonicalAgentWriterClient)).toBe(workerRuntime.writer); + expect(container.resolve(CanonicalExecutionWriterPort)).toBe(workerRuntime.writerPort); + expect(container.resolve(ExecutionWorkerLossCoordinator)).toBe(workerRuntime.workerLoss); + expect(container.resolve(ExecutionMailboxScheduler)).toBe(workerRuntime.scheduler); + expect(container.resolve(ExecutionMailboxOwner)).toBe(workerRuntime.owner); + expect(container.resolve(ExecutionProviderEventOwnership)).toBe(workerRuntime.providerEvents); + expect(workerRuntime.scheduler.depth()).toMatchObject({ + pending: 0, activeExecutions: 0, workers: [{ index: 0 }, { index: 1 }, { index: 2 }, { index: 3 }], + }); + }); + it("resolves ThreadService when a worktree thread reaches branch validation", async () => { const coordinator = container.resolve(ThreadCreationCoordinator); @@ -52,4 +96,335 @@ describe("AgentService container composition", () => { permissionMode: "default", })).rejects.toThrow("Branch name contains invalid characters: invalid branch"); }); + + it("commits a Codex turn through its execution owner and releases after Ended", async () => { + const published: AgentEvent[] = []; + const pushes: Array<{ channel: string; data: unknown }> = []; + pushClient = capturePushes(pushes); + let providerError: unknown; + const provider = fakeCodexProvider(async (request) => { + try { + await submitCodexEvent(workerRuntime!, request, 1, { type: "textDelta", + threadId: request.threadId, turnExecutionId: request.turnExecutionId, + delta: "owned answer", isFinalResponse: true }); + await submitCodexEvent(workerRuntime!, request, 2, { type: "turnComplete", + threadId: request.threadId, turnExecutionId: request.turnExecutionId, + providerId: "codex", reason: "end_turn", costUsd: null, tokensIn: 1, tokensOut: 2, + totalProcessedTokens: 3, contextWindow: undefined, cacheReadTokens: undefined }); + await submitCodexEvent(workerRuntime!, request, 3, { type: "ended", + threadId: request.threadId, turnExecutionId: request.turnExecutionId }); + } catch (error) { + providerError = error; + throw error; + } + }); + registerFakeCodex(provider); + const registry = container.resolve(AgentEventPublicationRegistry); + registry.bind((event) => published.push(event)); + registry.start(); + const workspace = container.resolve(WorkspaceRepo).create("worker-test", temporaryDirectory!); + const thread = container.resolve(ThreadRepo).create(workspace.id, "Worker turn", "direct", "main", true, "codex"); + + await container.resolve(AgentService).sendMessage({ threadId: thread.id, content: "hello", permissionMode: "default" }); + await waitFor(() => workerRuntime?.owner.current(thread.id) === undefined); + + expect(providerError).toBeUndefined(); + expect(published.map((event) => event.type)).toEqual(expect.arrayContaining([ + "turnStarted", "textDelta", "turnComplete", "ended", + ])); + expect(container.resolve(TurnRuntimeController).snapshot(thread.id)?.phase).toBe("completed"); + expect(workerRuntime?.scheduler.depth().activeExecutions).toBe(0); + expect(pushes.filter((push) => push.channel === "turn.persisted")).toEqual([ + expect.objectContaining({ data: expect.objectContaining({ threadId: thread.id, + outcome: "completed", executionId: expect.any(String), toolCallCount: 0 }) }), + ]); + expect(pushes.filter((push) => push.channel === "thread.status")).toEqual([ + expect.objectContaining({ data: { threadId: thread.id, status: "completed" } }), + ]); + }); + + it("returns from Stop after a durable worker terminal that preserves partial text", async () => { + const published: AgentEvent[] = []; + const pushes: Array<{ channel: string; data: unknown }> = []; + pushClient = capturePushes(pushes); + const stopProvider = vi.fn(async () => undefined); + const provider = fakeCodexProvider(async (request) => { + await submitCodexEvent(workerRuntime!, request, 1, { type: "textDelta", + threadId: request.threadId, turnExecutionId: request.turnExecutionId, + delta: "partial answer", isFinalResponse: true }); + }, stopProvider); + registerFakeCodex(provider); + const registry = container.resolve(AgentEventPublicationRegistry); + registry.bind((event) => published.push(event)); + registry.start(); + const workspace = container.resolve(WorkspaceRepo).create("worker-stop", temporaryDirectory!); + const thread = container.resolve(ThreadRepo).create(workspace.id, "Worker stop", "direct", "main", true, "codex"); + const service = container.resolve(AgentService); + + await service.sendMessage({ threadId: thread.id, content: "hello", permissionMode: "default" }); + const stopped = await service.stopSession(thread.id); + + expect(stopped.status).toBe("cancelled"); + expect(stopProvider).toHaveBeenCalledWith(`mcode-${thread.id}`); + expect(published.some((event) => event.type === "ended" && event.outcome === "interrupted")).toBe(true); + expect(container.resolve(MessageRepo).listByThread(thread.id, 10).messages + .some((message) => message.role === "assistant" && message.content === "partial answer")).toBe(true); + expect(workerRuntime?.owner.current(thread.id)).toBeUndefined(); + expect(pushes.filter((push) => push.channel === "turn.persisted")).toEqual([ + expect.objectContaining({ data: expect.objectContaining({ threadId: thread.id, + outcome: "cancelled", executionId: expect.any(String) }) }), + ]); + expect(pushes.filter((push) => push.channel === "thread.status")).toEqual([ + expect.objectContaining({ data: { threadId: thread.id, status: "interrupted" } }), + ]); + }); + + it("confirms a stopped turn without assistant text through its durable terminal push", async () => { + const pushes: Array<{ channel: string; data: unknown }> = []; + pushClient = capturePushes(pushes); + registerFakeCodex(fakeCodexProvider(async () => undefined)); + const registry = container.resolve(AgentEventPublicationRegistry); + registry.bind(() => undefined); + registry.start(); + const workspace = container.resolve(WorkspaceRepo).create("worker-empty-stop", temporaryDirectory!); + const thread = container.resolve(ThreadRepo).create(workspace.id, "Empty worker stop", "direct", "main", true, "codex"); + const service = container.resolve(AgentService); + + await service.sendMessage({ threadId: thread.id, content: "hello", permissionMode: "default" }); + const stopped = await service.stopSession(thread.id); + + expect(stopped.status).toBe("cancelled"); + expect(container.resolve(MessageRepo).listByThread(thread.id, 10).messages.map((message) => message.role)) + .toEqual(["user"]); + expect(pushes.filter((push) => push.channel === "turn.persisted")).toEqual([ + expect.objectContaining({ data: expect.objectContaining({ threadId: thread.id, + messageId: null, toolCallCount: 0, outcome: "cancelled", executionId: stopped.turnExecutionId }) }), + ]); + expect(pushes.filter((push) => push.channel === "thread.status")).toEqual([ + expect.objectContaining({ data: { threadId: thread.id, status: "interrupted" } }), + ]); + expect(workerRuntime?.owner.current(thread.id)).toBeUndefined(); + }); + + it("publishes a provider cancelled Ended event from the worker terminal receipt", async () => { + const published: AgentEvent[] = []; + const provider = fakeCodexProvider(async (request) => { + await submitCodexEvent(workerRuntime!, request, 1, { type: "ended", + threadId: request.threadId, turnExecutionId: request.turnExecutionId, outcome: "cancelled" }); + }); + registerFakeCodex(provider); + const registry = container.resolve(AgentEventPublicationRegistry); + registry.bind((event) => published.push(event)); + registry.start(); + const workspace = container.resolve(WorkspaceRepo).create("worker-cancelled", temporaryDirectory!); + const thread = container.resolve(ThreadRepo).create(workspace.id, "Provider cancelled", "direct", "main", true, "codex"); + + await container.resolve(AgentService).sendMessage({ threadId: thread.id, + content: "hello", permissionMode: "default" }); + await waitFor(() => workerRuntime?.owner.current(thread.id) === undefined); + + expect(published.some((event) => event.type === "ended" && event.outcome === "interrupted")).toBe(true); + }); + + it("finishes the exact owned turn when canonical delivery fails after provider send", async () => { + let sent: TurnRequest | undefined; + let reportFailure: ((routing: { threadId: string; turnId: string; + executionId: string; deliveryAttempt: number }, error: Error) => void | Promise) | undefined; + const provider = fakeCodexProvider(async (request) => { + sent = request; + }); + Object.assign(provider, { + setCanonicalTurnDeliveryFailureHandler: (handler: NonNullable) => { + reportFailure = handler; + }, + }); + registerFakeCodex(provider); + const registry = container.resolve(AgentEventPublicationRegistry); + const published: AgentEvent[] = []; + registry.bind((event) => published.push(event)); + registry.start(); + const workspace = container.resolve(WorkspaceRepo).create("worker-delivery", temporaryDirectory!); + const thread = container.resolve(ThreadRepo).create(workspace.id, "Delivery failure", "direct", "main", true, "codex"); + + await container.resolve(AgentService).sendMessage({ threadId: thread.id, + content: "hello", permissionMode: "default" }); + if (!sent || !reportFailure) throw new Error("Expected an exact canonical delivery route"); + const routing = { threadId: sent.threadId, turnId: sent.turnId, + executionId: sent.turnExecutionId, deliveryAttempt: 1 }; + await reportFailure({ ...routing, deliveryAttempt: 2 }, new Error("stale attempt")); + expect(workerRuntime?.owner.current(thread.id)).toBeDefined(); + await reportFailure(routing, new Error("canonical sink failed")); + await waitFor(() => workerRuntime?.owner.current(thread.id) === undefined); + + expect(container.resolve(TurnRuntimeController).snapshot(thread.id)?.phase).toBe("errored"); + expect(published.some((event) => event.type === "error" && event.error === "canonical sink failed")).toBe(true); + }); + + it("admits seven concurrent owned turns and releases each terminal", async () => { + const provider = fakeCodexProvider(async (request) => { + await submitCodexEvent(workerRuntime!, request, 1, { type: "turnComplete", + threadId: request.threadId, turnExecutionId: request.turnExecutionId, + providerId: "codex", reason: "end_turn", costUsd: null, tokensIn: 1, tokensOut: 1 }); + await submitCodexEvent(workerRuntime!, request, 2, { type: "ended", + threadId: request.threadId, turnExecutionId: request.turnExecutionId }); + }); + registerFakeCodex(provider); + const registry = container.resolve(AgentEventPublicationRegistry); + const published: AgentEvent[] = []; + registry.bind((event) => published.push(event)); + registry.start(); + const workspace = container.resolve(WorkspaceRepo).create("worker-concurrent", temporaryDirectory!); + const threads = Array.from({ length: 7 }, (_, index) => + container.resolve(ThreadRepo).create(workspace.id, `Concurrent ${index}`, "direct", "main", true, "codex")); + const service = container.resolve(AgentService); + + await Promise.all(threads.map((thread) => service.sendMessage({ threadId: thread.id, + content: "hello", permissionMode: "default" }))); + await waitFor(() => threads.every((thread) => workerRuntime?.owner.current(thread.id) === undefined)); + + expect(published.filter((event) => event.type === "turnStarted")).toHaveLength(7); + expect(published.filter((event) => event.type === "turnComplete")).toHaveLength(7); + expect(workerRuntime?.scheduler.depth().activeExecutions).toBe(0); + }); + + it("interrupts and releases a rejected provider event whose ordinal cannot be replayed", async () => { + let reportFailure: ((routing: { threadId: string; turnId: string; + executionId: string; deliveryAttempt: number }, error: Error) => void | Promise) | undefined; + const provider = fakeCodexProvider(async (request) => { + try { + await submitCodexEvent(workerRuntime!, request, 1, { type: "turnStarted", + threadId: request.threadId, turnExecutionId: request.turnExecutionId }); + } catch (error) { + if (!reportFailure) throw error; + await reportFailure({ threadId: request.threadId, turnId: request.turnId, + executionId: request.turnExecutionId, deliveryAttempt: 1 }, error as Error); + } + }); + Object.assign(provider, { setCanonicalTurnDeliveryFailureHandler: (handler: NonNullable) => { + reportFailure = handler; + } }); + registerFakeCodex(provider); + const registry = container.resolve(AgentEventPublicationRegistry); + registry.bind(() => undefined); + registry.start(); + const workspace = container.resolve(WorkspaceRepo).create("worker-rejected", temporaryDirectory!); + const thread = container.resolve(ThreadRepo).create(workspace.id, "Rejected event", "direct", "main", true, "codex"); + + await container.resolve(AgentService).sendMessage({ threadId: thread.id, + content: "hello", permissionMode: "default" }); + await waitFor(() => workerRuntime?.owner.current(thread.id) === undefined); + + expect(container.resolve(TurnRuntimeController).snapshot(thread.id)?.phase).toBe("interrupted"); + expect(workerRuntime?.scheduler.depth().activeExecutions).toBe(0); + }); + + it("releases a crashed execution while a peer remains runnable", async () => { + const provider = fakeCodexProvider(async (request) => { + void request; + }); + registerFakeCodex(provider); + const registry = container.resolve(AgentEventPublicationRegistry); + registry.bind(() => undefined); + registry.start(); + const workspace = container.resolve(WorkspaceRepo).create("worker-crash", temporaryDirectory!); + const threads = container.resolve(ThreadRepo); + const lost = threads.create(workspace.id, "Lost worker", "direct", "main", true, "codex"); + const peer = threads.create(workspace.id, "Peer worker", "direct", "main", true, "codex"); + const service = container.resolve(AgentService); + await service.sendMessage({ threadId: lost.id, content: "one", permissionMode: "default" }); + await service.sendMessage({ threadId: peer.id, content: "two", permissionMode: "default" }); + const assignment = requireValue(workerRuntime?.owner.current(lost.id), "Expected lost execution owner"); + const peerAssignment = requireValue(workerRuntime?.owner.current(peer.id), "Expected peer execution owner"); + expect(assignment.lease.workerIndex).not.toBe(peerAssignment.lease.workerIndex); + + const ports = (workerRuntime as unknown as { initialWorkers: ExecutionThreadWorkerPort[] }).initialWorkers; + const crashed = requireValue(ports[assignment.lease.workerIndex], "Lost worker port was not found"); + crashed.terminate(); + crashed.onclose?.(); + await waitFor(() => workerRuntime?.owner.current(lost.id) === undefined); + expect(await workerRuntime?.workerLoss.waitForRecovery(assignment.lease.workerIndex)) + .toMatchObject({ kind: "recovered" }); + + expect(container.resolve(TurnRuntimeController).snapshot(lost.id)?.phase).toBe("interrupted"); + expect(workerRuntime?.owner.current(peer.id)?.execution.executionId).toBe(peerAssignment.execution.executionId); + expect((await service.stopSession(peer.id)).status).toBe("cancelled"); + }); + + function registerFakeCodex(provider: IAgentProvider): void { + container.registerInstance("IProviderRegistry", { + resolve: () => provider, resolveAll: () => [provider], shutdown: () => undefined, + }); + container.registerInstance(ProviderAvailabilityService, { + assertUsable: () => undefined, + } as ProviderAvailabilityService); + } }); + +function fakeCodexProvider( + sendTurn: (request: TurnRequest) => Promise, + stopSession: (sessionId: string) => Promise = async () => undefined, +): IAgentProvider { + return Object.assign(new NodeEvents.EventEmitter(), { + id: "codex" as const, + descriptor: { id: "codex" }, + supportsCompletion: false, + sessionForkOnResume: "unsupported" as const, + maxInputCharactersPerTurn: 16000, + sendTurn, + stopSession, + shutdown: () => undefined, + listModels: async () => [], + setCanonicalTurnEventDeliveryEnabled: () => undefined, + setCanonicalTurnDeliveryFailureHandler: () => undefined, + fenceCanonicalTurnEvents: async () => undefined, + retireCanonicalTurnEvents: async () => undefined, + }) as IAgentProvider; +} + +async function submitCodexEvent( + runtime: WorkerOwnedTurnRuntime, + request: TurnRequest, + sequence: number, + event: AgentEvent, +): Promise { + const route = runtime.providerEvents.resolve(request.turnExecutionId, "codex"); + if (route.kind !== "worker") throw new Error("Codex turn did not bind its worker route"); + const itemId = `codex:${request.turnExecutionId}:item:${sequence}`; + const eventId = `codex:${request.turnExecutionId}:event:${sequence}`; + const timestamp = new Date().toISOString(); + await route.submit({ threadId: request.threadId, turnId: request.turnId, + executionId: request.turnExecutionId, phase: "running", deliveryAttempt: 1, batchId: eventId, + events: [{ eventId, sourceProviderId: "codex", sourceIdentities: [], sourceSequence: sequence, + providerTimestamp: timestamp, + routing: { threadId: request.threadId, turnId: request.turnId, + executionId: request.turnExecutionId, itemId }, + payload: { type: "item.recorded", item: { id: itemId, threadId: request.threadId, + turnId: request.turnId, kind: "system", providerIdentities: [], + payload: { projection: "providerRuntimeEvent", runtimeEvent: { event } }, + createdAt: timestamp, updatedAt: timestamp } } }], + }); +} + +async function waitFor(predicate: () => boolean): Promise { + for (let index = 0; index < 300; index++) { + if (predicate()) return; + await new Promise((resolve) => setTimeout(resolve, 10)); + } + throw new Error("Timed out waiting for worker turn release"); +} + +function capturePushes(pushes: Array<{ channel: string; data: unknown }>): WebSocket { + const client = { OPEN: 1, readyState: 1, bufferedAmount: 0, + send: (payload: string, complete: (error?: Error) => void) => { + const message = JSON.parse(payload) as { channel: string; data: unknown }; + pushes.push(message); + complete(); + } } as unknown as WebSocket; + addClient(client); + return client; +} + +function requireValue(value: T | undefined, message: string): T { + if (value === undefined) throw new Error(message); + return value; +} diff --git a/apps/server/src/features/agents/composition/register-agents.ts b/apps/server/src/features/agents/composition/register-agents.ts index 1b7dca5e8..a8b7b8a69 100644 --- a/apps/server/src/features/agents/composition/register-agents.ts +++ b/apps/server/src/features/agents/composition/register-agents.ts @@ -1,3 +1,5 @@ +import * as NodePath from "node:path"; +import type { Database } from "bun:sqlite"; import { Lifecycle, instanceCachingFactory, type DependencyContainer } from "tsyringe"; import type { HostRuntime } from "@mcode/shared/node/host-runtime"; import { broadcast } from "../../../application/transport/push.js"; @@ -38,6 +40,15 @@ import { AgentTurnCommandPort, } from "../orchestration/agent-turn-command-port.js"; import { AgentEventPublicationRegistry } from "../orchestration/agent-event-publication-registry.js"; +import { CanonicalAgentWriterClient } from "../canonical/canonical-agent-writer-client.js"; +import { CanonicalExecutionWriterPort } from "../canonical/canonical-execution-writer-port.js"; +import { ExecutionMailboxOwner } from "../execution/execution-mailbox-owner.js"; +import { ExecutionMailboxScheduler } from "../execution/execution-mailbox-scheduler.js"; +import { ExecutionProviderEventOwnership } from "../execution/execution-provider-event-ownership.js"; +import { ExecutionWorkerLossCoordinator } from "../execution/execution-worker-loss-coordinator.js"; +import { ExecutionFileEvidenceCoordinator } from "../execution/execution-file-evidence-coordinator.js"; +import type { ExecutionWorkCommand, ExecutionWorkerResult } from "../execution/execution-worker-handler.js"; +import { WorkerOwnedTurnRuntime } from "../execution/worker-owned-turn-runtime.js"; import { AgentEventPublicationRuntimePort, AgentReliabilityPort, @@ -83,6 +94,36 @@ import { TurnRuntimeController } from "../orchestration/turn-runtime-controller. /** Register agent orchestration, event, recovery, and planning services. */ export function registerAgentServices(container: DependencyContainer): void { + container.register(WorkerOwnedTurnRuntime, { + useFactory: instanceCachingFactory((c) => { + const dbPath = c.resolve("Database").filename; + if (!dbPath || dbPath === ":memory:") { + throw new Error("Execution workers require a file-backed database"); + } + return new WorkerOwnedTurnRuntime(NodePath.resolve(dbPath), c.resolve(AgentEventPublicationRegistry)); + }), + }); + container.register("WorkerOwnedTurnRuntime", { + useFactory: (c) => c.resolve(WorkerOwnedTurnRuntime), + }); + container.register(CanonicalAgentWriterClient, { + useFactory: (c) => c.resolve(WorkerOwnedTurnRuntime).writer, + }); + container.register(CanonicalExecutionWriterPort, { + useFactory: (c) => c.resolve(WorkerOwnedTurnRuntime).writerPort, + }); + container.register(ExecutionWorkerLossCoordinator, { + useFactory: (c) => c.resolve(WorkerOwnedTurnRuntime).workerLoss, + }); + container.register>(ExecutionMailboxScheduler, { + useFactory: (c) => c.resolve(WorkerOwnedTurnRuntime).scheduler, + }); + container.register(ExecutionMailboxOwner, { + useFactory: (c) => c.resolve(WorkerOwnedTurnRuntime).owner, + }); + container.register(ExecutionProviderEventOwnership, { + useFactory: (c) => c.resolve(WorkerOwnedTurnRuntime).providerEvents, + }); container.register(TurnDiffService, { useFactory: instanceCachingFactory((c) => new TurnDiffService( new TurnDiffRepo(c.resolve("Database")), @@ -182,6 +223,18 @@ export function registerAgentServices(container: DependencyContainer): void { ); }), }); + container.register(ExecutionFileEvidenceCoordinator, { + useFactory: instanceCachingFactory((c) => new ExecutionFileEvidenceCoordinator( + c.resolve(TURN_FILE_TRACKER), + c.resolve(TurnDiffService), + )), + }); + container.register("ExecutionFileEvidenceCoordinator", { + useFactory: (c) => c.resolve(ExecutionFileEvidenceCoordinator), + }); + container.register("WorkerTurnSnapshotService", { + useFactory: (c) => c.resolve(SnapshotService), + }); container.register(TURN_FINALIZER, { useFactory: instanceCachingFactory((c) => new TurnFinalizer( c.resolve(MessageRepo), @@ -223,6 +276,7 @@ export function registerAgentServices(container: DependencyContainer): void { c.resolve(WorkspaceEnvironmentService), c.resolve(FileService), c.resolve("HostRuntime").platform, + c.resolve(WorkerOwnedTurnRuntime).owner, )), }); container.register(TurnConversationProjectionService, { useClass: TurnConversationProjectionService }, { lifecycle: Lifecycle.Singleton }); diff --git a/apps/server/src/features/agents/conversation/persistence/__tests__/message-repo.test.ts b/apps/server/src/features/agents/conversation/persistence/__tests__/message-repo.test.ts index a1ecb3905..02693a3c7 100644 --- a/apps/server/src/features/agents/conversation/persistence/__tests__/message-repo.test.ts +++ b/apps/server/src/features/agents/conversation/persistence/__tests__/message-repo.test.ts @@ -164,6 +164,25 @@ describe("MessageRepo", () => { }); describe("listByThread", () => { + it("does not let hidden provider notices displace the visible conversation tail", () => { + const user = repo.create("thread-1", "user", "Stop this turn", 1); + repo.createSystemNotice("thread-1", "Provider warning", 2, { + kind: "warning", presentation: "timeline", scope: "turn", sessionId: "session-1", noticeKey: "warning-1", + }); + repo.createSystemNotice("thread-1", "Deprecated setting", 3, { + kind: "deprecation", presentation: "timeline", scope: "turn", sessionId: "session-1", noticeKey: "deprecation-1", + }); + + expect(repo.listByThread("thread-1", 2)).toMatchObject({ + messages: [{ id: user.id }], + hasMore: false, + }); + expect(repo.listByThreadAfter("thread-1", 2, 0)).toMatchObject({ + messages: [{ id: user.id }], + hasMore: false, + }); + }); + it("returns stored canonical legacy provenance without replacing it with the ambiguous fallback", () => { db.prepare(` INSERT INTO messages (id, thread_id, role, content, timestamp, sequence, origin_type, legacy_provenance) diff --git a/apps/server/src/features/agents/conversation/persistence/message-repo.ts b/apps/server/src/features/agents/conversation/persistence/message-repo.ts index abe415d40..7031635d6 100644 --- a/apps/server/src/features/agents/conversation/persistence/message-repo.ts +++ b/apps/server/src/features/agents/conversation/persistence/message-repo.ts @@ -181,6 +181,8 @@ function rowToMessage(row: MessageRow): Message { /** Session-scoped system notices are hidden from transcript reads. */ const notSessionNotice = sql`json_extract(${messages.systemNotice}, '$.scope') IS NOT 'session'`; +// The transcript hides these notices, so they cannot consume a bounded display page. +const notRoutineTranscriptNotice = sql`COALESCE(json_extract(${messages.systemNotice}, '$.kind'), '') NOT IN ('warning', 'configuration', 'deprecation', 'authentication-recovered')`; const DEFAULT_HISTORY_PAGE_SIZE = 100; const MAX_HISTORY_PAGE_SIZE = 500; @@ -593,6 +595,14 @@ export class MessageRepo { return row?.sequence ?? 0; } + /** Provider system notices may follow a user prompt without starting another conversation turn. */ + isLatestNonSystemMessage(threadId: string, messageId: string): boolean { + const latest = this.orm.select({ id: messages.id }).from(messages) + .where(and(eq(messages.threadId, threadId), inArray(messages.role, ["user", "assistant"]))) + .orderBy(desc(messages.sequence)).limit(1).get(); + return latest?.id === messageId; + } + /** Append stored attachments to an existing message, deduping by attachment id. */ appendAttachments(messageId: string, attachments: StoredAttachment[]): StoredAttachment[] { if (attachments.length === 0) return []; @@ -643,6 +653,7 @@ export class MessageRepo { ...(before != null ? [lt(messages.sequence, before)] : []), eq(messages.isInternal, 0), notSessionNotice, + notRoutineTranscriptNotice, ), "DESC", fetchLimit, @@ -700,6 +711,7 @@ export class MessageRepo { gt(messages.sequence, after), eq(messages.isInternal, 0), notSessionNotice, + notRoutineTranscriptNotice, ), "ASC", fetchLimit, diff --git a/apps/server/src/features/agents/execution/__tests__/codex-live-event-effects.test.ts b/apps/server/src/features/agents/execution/__tests__/codex-live-event-effects.test.ts new file mode 100644 index 000000000..e06db28ed --- /dev/null +++ b/apps/server/src/features/agents/execution/__tests__/codex-live-event-effects.test.ts @@ -0,0 +1,121 @@ +import { describe, expect, it } from "vitest"; +import { AgentEventSchema, type AgentEvent } from "@mcode/contracts"; +import { CodexLiveEventReducer, type CodexPlanFeature } from "../codex-live-event-reducer.js"; +import { CodexLiveEventEffects } from "../codex-live-event-effects.js"; +import { deriveTurnAssistantMessageId } from "../../turns/turn-assistant-message-id.js"; + +const execution = { threadId: "thread", turnId: "turn", executionId: "11111111-1111-4111-8111-111111111111" }; +const endedAt = "2026-09-25T12:00:00.000Z"; + +function event(type: AgentEvent["type"], fields: Record = {}): AgentEvent { + return AgentEventSchema().parse({ type, threadId: execution.threadId, turnExecutionId: execution.executionId, ...fields }); +} + +function setup(plan: CodexPlanFeature = "none") { + const reducer = new CodexLiveEventReducer(execution, plan); + const mapper = new CodexLiveEventEffects(execution, "user-message"); + const prepare = (type: AgentEvent["type"], fields: Record = {}) => { + const reduction = reducer.reduce(event(type, fields)); + if (reduction.kind !== "reduced") throw new Error(reduction.reason); + return mapper.prepare(reduction, endedAt); + }; + prepare("turnStarted"); + return { prepare, mapper }; +} + +describe("CodexLiveEventEffects", () => { + it("moves unknown text into narrative and restarts its checkpoint sequence", () => { + const { prepare } = setup(); + expect(prepare("textDelta", { delta: "Looking at the files" }).effects.text).toEqual({ + kind: "append", inputs: [{ ...execution, sequence: 1, text: "Looking at the files" }], + }); + const boundary = prepare("assistantMessageBoundary", { isFinalResponse: false }); + expect(boundary.effects.text).toEqual({ kind: "reclassify", expectedText: "Looking at the files" }); + expect(boundary.effects.narrative?.items).toEqual([ + expect.objectContaining({ kind: "narrationSegment", record: expect.objectContaining({ text: "Looking at the files" }) }), + ]); + expect(prepare("textDelta", { delta: "Fixed", isFinalResponse: true }).effects.text).toEqual({ + kind: "append", inputs: [{ ...execution, sequence: 1, text: "Fixed" }], + }); + }); + + it("promotes narration while deleting its old recovery record", () => { + const { prepare } = setup(); + const thought = prepare("textDelta", { delta: "Answer", isFinalResponse: false }); + const id = thought.effects.narrative?.items[0]?.record.id; + expect(id).toBeDefined(); + const promoted = prepare("assistantMessageBoundary", { isFinalResponse: true }); + expect(promoted.effects.text).toEqual({ kind: "promote", input: { ...execution, sequence: 1, text: "Answer" } }); + expect(promoted.effects.narrative?.discardedItemIds).toEqual([`narrationSegment:${id}`]); + }); + + it("retains TaskCreate input until its result and keeps file intents separate", () => { + const { prepare } = setup(); + const use = prepare("toolUse", { toolCallId: "create", toolName: "TaskCreate", toolInput: { subject: "Fix login" } }); + expect(use.runtime).toContainEqual(expect.objectContaining({ kind: "tool-use" })); + expect(use.effects.narrative?.items[0]?.kind).toBe("toolCall"); + const result = prepare("toolResult", { toolCallId: "create", output: "Task #42 created", isError: false }); + expect(result.effects.taskIntents).toEqual([ + { kind: "append-task", task: { id: "42", content: "Fix login", status: "pending", group: "Tasks" } }, + ]); + }); + + it("carries system intents and leaves context projection explicit", () => { + const { prepare } = setup(); + const system = prepare("system", { subtype: "sdk_session_id:session-1" }); + expect(system.effects.systemIntents).toEqual([{ kind: "session-cursor", event: system.publication.event }]); + const context = prepare("contextEstimate", { tokensIn: 120, totalProcessedTokens: 120, contextWindow: 1000 }); + expect(context.runtime).toEqual([{ kind: "context-usage", tokensIn: 120, contextWindow: 1000 }]); + }); + + it("uses the same assigned assistant identity for live and terminal data", () => { + const { prepare } = setup(); + prepare("textDelta", { delta: "Answer", isFinalResponse: true }); + const message = prepare("message", { content: "Answer", tokens: null }); + const messageId = deriveTurnAssistantMessageId(execution.threadId, "user-message"); + expect(message.effects.message).toMatchObject({ precedingMessageId: "user-message", messageId, content: "Answer" }); + expect(message.publication.event).toMatchObject({ type: "message", messageId }); + const terminal = prepare("error", { error: "Provider disconnected" }); + expect(terminal.terminal).toMatchObject({ threadId: execution.threadId, executionId: execution.executionId, + endedAt, outcome: "errored", assistant: { messageId, content: "Answer" } }); + expect(terminal.publication.after).toBe("terminal"); + expect(terminal.runtime).toContainEqual({ kind: "turn-error", error: "Provider disconnected" }); + expect(structuredClone(terminal)).toEqual(terminal); + }); + + it("carries parsed plan questions with their completing text event", () => { + const { prepare } = setup("questions"); + const question = { id: "q1", category: "AUTH", question: "Which login?", options: [ + { id: "o1", title: "Passkey", description: "Use passkeys.", recommended: true }, + { id: "o2", title: "Password", description: "Use passwords." }, + ] }; + const text = `\`\`\`plan-questions\n${JSON.stringify([question])}\n\`\`\``; + const result = prepare("textDelta", { delta: text }); + expect(result.effects.planQuestions).toEqual([question]); + expect(result.effects.text.kind).toBe("append"); + expect(result.publication.after).toBe("writer"); + }); + + it("rejects another execution before advancing checkpoint state", () => { + const { mapper, prepare } = setup(); + const other = new CodexLiveEventReducer({ ...execution, turnId: "other-turn" }); + const reduction = other.reduce(event("turnStarted")); + if (reduction.kind !== "reduced") throw new Error(reduction.reason); + expect(() => mapper.prepare(reduction)).toThrow("another execution"); + expect(prepare("textDelta", { delta: "Answer" }).effects.text).toMatchObject({ inputs: [{ sequence: 1 }] }); + }); + + it("associates parsed plan output with the staged assistant body", () => { + const { prepare } = setup("output"); + const plan = { title: "Login plan", sections: [ + { id: "s1", title: "Implementation", level: 1, content: "Add passkey login." }, + ] }; + prepare("textDelta", { delta: `\`\`\`plan-output\n${JSON.stringify(plan)}\n\`\`\`` }); + const result = prepare("message", { content: "Provider prose", tokens: null }); + expect(result.effects.planOutput).toEqual({ title: "Login plan", + contentMd: "## Implementation\n\nAdd passkey login.", + sectionsJson: '[{"id":"s1","title":"Implementation","level":1}]', changeSummary: null }); + expect(result.effects.message?.content).toBe("Provider prose"); + expect(result.publication.event).toMatchObject({ messageId: result.effects.message?.messageId }); + }); +}); diff --git a/apps/server/src/features/agents/execution/__tests__/execution-file-evidence-coordinator.test.ts b/apps/server/src/features/agents/execution/__tests__/execution-file-evidence-coordinator.test.ts new file mode 100644 index 000000000..b664ced5e --- /dev/null +++ b/apps/server/src/features/agents/execution/__tests__/execution-file-evidence-coordinator.test.ts @@ -0,0 +1,131 @@ +import "reflect-metadata"; +import * as NodeFSPromises from "node:fs/promises"; +import * as NodeOS from "node:os"; +import * as NodePath from "node:path"; +import type { Database } from "bun:sqlite"; +import { afterEach, describe, expect, it } from "vitest"; + +import type { ProviderFileMutationStart } from "@mcode/contracts"; + +import { openMemoryDatabase } from "../../../../runtime/persistence/sqlite/database.js"; +import { RealGitExecutor } from "../../../projects/git/execution/real-git-executor.js"; +import { SnapshotService } from "../../../projects/diffs/snapshots/snapshot-service.js"; +import { TurnDiffRepo } from "../../turns/persistence/turn-diff-repo.js"; +import { TurnDiffService } from "../../turns/turn-diff-service.js"; +import { TurnFileTracker } from "../../turns/turn-file-tracker.js"; +import { + ExecutionFileEvidenceCoordinator, + prepareFrozenExecutionFileEvidence, +} from "../execution-file-evidence-coordinator.js"; + +const directories: string[] = []; +const databases: Database[] = []; +const input = { + threadId: "thread", turnId: "turn", executionId: "execution", + deliveryAttempt: 1, baselineRef: null, +}; + +async function directory(): Promise { + const path = await NodeFSPromises.mkdtemp(NodePath.join(NodeOS.tmpdir(), "mcode-worker-files-")); + directories.push(path); + return path; +} + +function tracker(): TurnFileTracker { + return new TurnFileTracker(async () => ({ kind: "unavailable" }), () => {}, "win32"); +} + +function diffs(): TurnDiffService { + const db = openMemoryDatabase(); + databases.push(db); + return new TurnDiffService(new TurnDiffRepo(db)); +} + +function mutation(executionId = input.executionId, deliveryAttempt = input.deliveryAttempt): ProviderFileMutationStart { + return { threadId: input.threadId, turnExecutionId: executionId, deliveryAttempt, + toolCallId: "edit", toolName: "Edit", toolInput: { file_path: "tracked.txt" } }; +} + +function toolUse(event: ProviderFileMutationStart) { + return { type: "toolUse" as const, threadId: event.threadId, + turnExecutionId: event.turnExecutionId, toolCallId: event.toolCallId, + toolName: event.toolName, toolInput: event.toolInput }; +} + +afterEach(async () => { + for (const db of databases.splice(0)) db.close(); + await Promise.all(directories.splice(0).map((path) => NodeFSPromises.rm(path, { recursive: true, force: true }))); +}); + +describe("ExecutionFileEvidenceCoordinator", () => { + it("transfers pre-edit evidence and settles only the sealed execution", async () => { + const cwd = await directory(); + const file = NodePath.join(cwd, "tracked.txt"); + await NodeFSPromises.writeFile(file, "before\n"); + const host = tracker(); + const worker = tracker(); + const native = diffs(); + const coordinator = new ExecutionFileEvidenceCoordinator(host, native); + const handoff = coordinator.begin({ ...input, cwd }); + expect(worker.beginTurnFromHandoff(structuredClone(handoff), { + threadId: input.threadId, executionId: input.executionId, cwd, + })).toBe(true); + const event = mutation(); + + expect(coordinator.capture(event)).toBe(true); + await NodeFSPromises.writeFile(file, "after\nextra\n"); + const captured = coordinator.take(toolUse(event), input.deliveryAttempt); + if (!captured) throw new Error("Expected pre-edit file evidence"); + expect(await worker.observeCapturedToolUse(toolUse(event), structuredClone(captured))).toBe(true); + await worker.observeToolResult(input.threadId, event.toolCallId); + const patch = "diff --git a/tracked.txt b/tracked.txt\nindex 1..2\n--- a/tracked.txt\n+++ b/tracked.txt\n@@ -1 +1,2 @@\n-before\n+after\n+extra\n"; + expect(native.push({ + turnId: input.turnId, turnExecutionId: input.executionId, + deliveryAttempt: input.deliveryAttempt, revision: 1, + state: "snapshot", nativeFidelity: "agent", patch, + })).toBe("accepted"); + const frozen = coordinator.seal({ ...input, outcome: "completed" }); + if (!frozen) throw new Error("Expected frozen file evidence"); + expect(() => structuredClone(frozen)).not.toThrow(); + expect(coordinator.seal({ ...input, outcome: "completed" })).toBe(frozen); + expect(coordinator.capture(event)).toBe(false); + + const prepared = await prepareFrozenExecutionFileEvidence( + structuredClone(frozen), worker, new SnapshotService(new RealGitExecutor()), + ); + expect(prepared).toMatchObject({ + threadId: input.threadId, turnId: input.turnId, executionId: input.executionId, + deliveryAttempt: input.deliveryAttempt, filesChanged: ["tracked.txt"], + fileEffects: { fileCount: 1, additions: 2, deletions: 1 }, + selectedTurnDiff: { thread_id: input.threadId, source: "native", patch, revision: 1 }, + }); + expect(() => structuredClone(prepared)).not.toThrow(); + expect(coordinator.retire(input.threadId, input.executionId, input.deliveryAttempt)).toBe(true); + expect(await host.finalEvidenceForExecution(handoff)).toBeNull(); + }); + + it("fences Stop, retries, and late terminal callbacks by execution and attempt", async () => { + const cwd = await directory(); + await NodeFSPromises.writeFile(NodePath.join(cwd, "tracked.txt"), "before\n"); + const coordinator = new ExecutionFileEvidenceCoordinator(tracker(), diffs()); + coordinator.begin({ ...input, cwd }); + expect(() => coordinator.begin({ ...input, deliveryAttempt: 0, cwd })).toThrow("positive delivery attempt"); + expect(coordinator.capture(mutation())).toBe(true); + expect(coordinator.fence(input.threadId, input.executionId, 2)).toBe(false); + expect(coordinator.fence(input.threadId, input.executionId, 1)).toBe(true); + expect(coordinator.take(toolUse(mutation()), 1)).toBeNull(); + expect(coordinator.seal({ ...input, deliveryAttempt: 2, outcome: "cancelled" })).toBeNull(); + const stopped = coordinator.seal({ ...input, outcome: "cancelled" }); + expect(stopped?.outcome).toBe("cancelled"); + expect(() => coordinator.seal({ ...input, outcome: "completed" })).toThrow("outcome changed"); + expect(() => coordinator.begin({ ...input, deliveryAttempt: 2, cwd })).toThrow("Retire sealed"); + expect(coordinator.retire(input.threadId, input.executionId, 1)).toBe(true); + + coordinator.begin({ ...input, deliveryAttempt: 2, cwd }); + expect(coordinator.capture(mutation(input.executionId, 1))).toBe(false); + expect(coordinator.capture(mutation(input.executionId, 2))).toBe(true); + expect(coordinator.seal({ ...input, outcome: "cancelled" })).toBeNull(); + expect(coordinator.retire(input.threadId, input.executionId, 1)).toBe(false); + expect(coordinator.retire(input.threadId, input.executionId, 2)).toBe(true); + }); +}); diff --git a/apps/server/src/features/agents/execution/__tests__/execution-mailbox-owner.test.ts b/apps/server/src/features/agents/execution/__tests__/execution-mailbox-owner.test.ts index 41988315e..349af331a 100644 --- a/apps/server/src/features/agents/execution/__tests__/execution-mailbox-owner.test.ts +++ b/apps/server/src/features/agents/execution/__tests__/execution-mailbox-owner.test.ts @@ -87,6 +87,21 @@ describe("ExecutionProviderEventOwnership", () => { }); const owner = new ExecutionMailboxOwner(scheduler); const ownership = new ExecutionProviderEventOwnership(owner); + const acknowledged: string[] = []; + let reportAcknowledgement: (() => void) | undefined; + const acknowledgementStarted = new Promise((resolve) => { reportAcknowledgement = resolve; }); + let finishAcknowledgement: (() => void) | undefined; + const acknowledgement = new Promise((resolve) => { finishAcknowledgement = resolve; }); + ownership.bindCommitted(async (owned, result) => { + acknowledged.push(`${owned.executionId}:${result.operationId}`); + reportAcknowledgement?.(); + await acknowledgement; + }); + const prepared: string[] = []; + ownership.bindCommandPreparation(async (owned, batch) => { + prepared.push(`${owned.executionId}:${batch.deliveryAttempt}`); + return { kind: "event", phase: batch.phase, nativeCursor: batch.nativeCursor ?? null, events: batch.events }; + }); expect(ownership.resolve(execution.executionId)).toEqual({ kind: "rejected" }); const start = owner.start({ execution, ownerEpoch: 1, providerId: "codex", parentTurn }); await expect(ownership.bind(execution, 1)).rejects.toThrow("no matching execution owner"); @@ -100,8 +115,10 @@ describe("ExecutionProviderEventOwnership", () => { batchId: "batch-1", deliveryAttempt: 1, phase: "running", events: [], }; const submitted = route.submit(batch); + await Promise.resolve(); expect(worker.requests[1]?.command).toMatchObject({ kind: "event", phase: "running", events: [] }); - const nextAttempt = ownership.bind(execution, 2); + let nextAttemptBound = false; + const nextAttempt = ownership.bind(execution, 2).then(() => { nextAttemptBound = true; }); expect(ownership.resolve(execution.executionId)).toEqual({ kind: "rejected" }); await expect(route.submit(batch)).rejects.toThrow("retired execution attempt"); worker.reply(1, { @@ -111,13 +128,20 @@ describe("ExecutionProviderEventOwnership", () => { acceptedThrough: 1, durableThrough: 1, eventCount: 0, }, }); + await acknowledgementStarted; + await new Promise((resolve) => setTimeout(resolve, 0)); + expect(nextAttemptBound).toBe(false); + finishAcknowledgement?.(); await expect(submitted).resolves.toMatchObject({ batchId: "batch-1", deliveryAttempt: 1, commit: { outcome: "committed" } }); + expect(acknowledged).toEqual([`${execution.executionId}:${worker.requests[1]!.lease.leaseId}:2`]); + expect(prepared).toEqual([`${execution.executionId}:1`]); await nextAttempt; expect(ownership.resolve(execution.executionId)).toMatchObject({ kind: "worker", deliveryAttempt: 2 }); await ownership.retire(execution); expect(ownership.resolve(execution.executionId)).toEqual({ kind: "rejected" }); + expect(ownership.resolve("legacy-execution", "claude")).toEqual({ kind: "legacy" }); expect(worker.requests).toHaveLength(2); scheduler.shutdown(); }); diff --git a/apps/server/src/features/agents/execution/__tests__/execution-mailbox-scheduler.test.ts b/apps/server/src/features/agents/execution/__tests__/execution-mailbox-scheduler.test.ts index 7269d78ed..99f214cc3 100644 --- a/apps/server/src/features/agents/execution/__tests__/execution-mailbox-scheduler.test.ts +++ b/apps/server/src/features/agents/execution/__tests__/execution-mailbox-scheduler.test.ts @@ -18,6 +18,7 @@ type Work = | { readonly kind: "checkpoint"; readonly revision: number } | { readonly kind: "effect-result"; readonly effectId: string } | { readonly kind: "stage-terminal" } + | { readonly kind: "finish-from-state" } | { readonly kind: "finalize" }; type Command = Work | { readonly kind: "stop"; readonly requestId: string }; type Result = { readonly revision: number }; @@ -108,6 +109,23 @@ function request(worker: FakeWorker, index: number): ExecutionWorkerRequest { + it("admits worker state finalization after the Stop watermark", async () => { + const { scheduler, workers } = fixture(); + const identity = execution("stopped"); + const lease = claimed(scheduler, identity); + const stop = admitted(scheduler, identity, lease, { kind: "stop", requestId: "stop" }); + const finish = admitted(scheduler, identity, lease, { kind: "finish-from-state" }); + const worker = workers[0]; + if (!worker) throw new Error("Expected worker"); + expect(worker.requests).toHaveLength(1); + worker.reply(request(worker, 0), 1); + expect(request(worker, 1).command.kind).toBe("finish-from-state"); + worker.reply(request(worker, 1), 2); + await expect(stop.completion).resolves.toMatchObject({ kind: "reply" }); + await expect(finish.completion).resolves.toMatchObject({ kind: "reply" }); + scheduler.shutdown(); + }); + it("keeps an execution on one worker and processes each mailbox in order", async () => { const { scheduler, workers } = fixture(2); const first = execution("first"); diff --git a/apps/server/src/features/agents/execution/__tests__/execution-worker-file-evidence.test.ts b/apps/server/src/features/agents/execution/__tests__/execution-worker-file-evidence.test.ts new file mode 100644 index 000000000..b034deda9 --- /dev/null +++ b/apps/server/src/features/agents/execution/__tests__/execution-worker-file-evidence.test.ts @@ -0,0 +1,137 @@ +import * as NodeFSPromises from "node:fs/promises"; +import * as NodeOS from "node:os"; +import * as NodePath from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; + +import type { ProviderFileMutationStart } from "@mcode/contracts"; + +import { TurnFileTracker } from "../../turns/turn-file-tracker.js"; +import { ExecutionFileEvidenceCoordinator } from "../execution-file-evidence-coordinator.js"; +import { ExecutionWorkerFileEvidence } from "../execution-worker-file-evidence.js"; + +const directories: string[] = []; +const execution = { threadId: "thread", turnId: "turn", executionId: "execution" }; + +async function directory(): Promise { + const path = await NodeFSPromises.mkdtemp(NodePath.join(NodeOS.tmpdir(), "mcode-worker-file-settle-")); + directories.push(path); + return path; +} + +function coordinator() { + const captureTracker = new TurnFileTracker(async () => ({ kind: "unavailable" }), () => {}, process.platform); + return new ExecutionFileEvidenceCoordinator(captureTracker, { + begin: () => {}, + takeFinalizationEvidence: () => null, + clearExecution: () => {}, + }); +} + +function mutation(deliveryAttempt = 1): ProviderFileMutationStart { + return { + threadId: execution.threadId, turnExecutionId: execution.executionId, + deliveryAttempt, toolCallId: "edit", toolName: "Edit", + toolInput: { file_path: "tracked.txt" }, + }; +} + +function toolUse(event: ProviderFileMutationStart) { + return { + type: "toolUse" as const, threadId: event.threadId, + turnExecutionId: event.turnExecutionId, toolCallId: event.toolCallId, + toolName: event.toolName, toolInput: event.toolInput, + }; +} + +function toolResult() { + return { + type: "toolResult" as const, threadId: execution.threadId, + turnExecutionId: execution.executionId, toolCallId: "edit", + output: "done", isError: false, + }; +} + +afterEach(async () => { + await Promise.all(directories.splice(0).map((path) => NodeFSPromises.rm(path, { recursive: true, force: true }))); +}); + +describe("ExecutionWorkerFileEvidence", () => { + it("settles a pre-edit capture on the worker after the tool result", async () => { + const cwd = await directory(); + const file = NodePath.join(cwd, "tracked.txt"); + await NodeFSPromises.writeFile(file, "before\n"); + const host = coordinator(); + const worker = new ExecutionWorkerFileEvidence(); + const handoff = host.begin({ ...execution, deliveryAttempt: 1, cwd, baselineRef: null }); + expect(worker.begin({ execution, deliveryAttempt: 1, cwd, handoff: structuredClone(handoff) })).toBe(true); + const event = mutation(); + expect(host.capture(event)).toBe(true); + await NodeFSPromises.writeFile(file, "after\nextra\n"); + const captured = host.take(toolUse(event), 1); + if (!captured) throw new Error("Expected captured pre-edit evidence"); + expect(await worker.observeToolUse({ execution, deliveryAttempt: 1, event: toolUse(event), + captured: structuredClone(captured) })).toBe(true); + expect(await worker.observeToolResult({ execution, deliveryAttempt: 1, event: toolResult() })) + .toMatchObject({ fileCount: 1, additions: 2, deletions: 1 }); + const frozen = host.seal({ ...execution, deliveryAttempt: 1, outcome: "completed" }); + if (!frozen) throw new Error("Expected frozen terminal evidence"); + + const prepared = await worker.settle(structuredClone(frozen)); + expect(prepared).toMatchObject({ + threadId: execution.threadId, turnId: execution.turnId, + executionId: execution.executionId, deliveryAttempt: 1, + filesChanged: ["tracked.txt"], fileEffects: { fileCount: 1, additions: 2, deletions: 1 }, + }); + expect(await worker.settle(structuredClone(frozen))).toBe(prepared); + expect(() => structuredClone(prepared)).not.toThrow(); + expect(worker.retire(execution, 1)).toBe(true); + expect(worker.retire(execution, 1)).toBe(false); + }); + + it("rejects a different root or attempt before it can change file evidence", async () => { + const cwd = await directory(); + const otherCwd = await directory(); + await NodeFSPromises.writeFile(NodePath.join(cwd, "tracked.txt"), "before\n"); + const host = coordinator(); + const worker = new ExecutionWorkerFileEvidence(); + const handoff = host.begin({ ...execution, deliveryAttempt: 2, cwd, baselineRef: null }); + expect(worker.begin({ execution, deliveryAttempt: 2, cwd: otherCwd, handoff })).toBe(false); + expect(worker.begin({ execution, deliveryAttempt: 2, cwd, handoff })).toBe(true); + const event = mutation(2); + expect(host.capture(event)).toBe(true); + const captured = host.take(toolUse(event), 2); + if (!captured) throw new Error("Expected captured pre-edit evidence"); + expect(await worker.observeToolUse({ execution, deliveryAttempt: 1, event: toolUse(event), captured })).toBe(false); + expect(await worker.observeToolResult({ execution, deliveryAttempt: 1, event: toolResult() })).toBeNull(); + const frozen = host.seal({ ...execution, deliveryAttempt: 2, outcome: "cancelled" }); + if (!frozen) throw new Error("Expected frozen terminal evidence"); + expect(await worker.settle({ ...frozen, deliveryAttempt: 1 })).toBeNull(); + expect(worker.retire(execution, 1)).toBe(false); + expect(worker.retire(execution, 2)).toBe(true); + }); + + it("settles an open file tool when Stop arrives before its result", async () => { + const cwd = await directory(); + const file = NodePath.join(cwd, "tracked.txt"); + await NodeFSPromises.writeFile(file, "before\n"); + const host = coordinator(); + const worker = new ExecutionWorkerFileEvidence(); + const handoff = host.begin({ ...execution, deliveryAttempt: 1, cwd, baselineRef: null }); + expect(worker.begin({ execution, deliveryAttempt: 1, cwd, handoff })).toBe(true); + const event = mutation(); + expect(host.capture(event)).toBe(true); + const captured = host.take(toolUse(event), 1); + if (!captured) throw new Error("Expected captured pre-edit evidence"); + expect(await worker.observeToolUse({ execution, deliveryAttempt: 1, event: toolUse(event), captured })) + .toBe(true); + await NodeFSPromises.writeFile(file, "after\n"); + + const frozen = host.seal({ ...execution, deliveryAttempt: 1, outcome: "cancelled" }); + if (!frozen) throw new Error("Expected frozen terminal evidence"); + expect(await worker.settle(frozen)).toMatchObject({ + fileEffects: { fileCount: 1, additions: 1, deletions: 1 }, + }); + expect(await worker.observeToolResult({ execution, deliveryAttempt: 1, event: toolResult() })) + .toBeNull(); + }); +}); diff --git a/apps/server/src/features/agents/execution/__tests__/execution-worker-handler.test.ts b/apps/server/src/features/agents/execution/__tests__/execution-worker-handler.test.ts index bca31588e..d5686ac86 100644 --- a/apps/server/src/features/agents/execution/__tests__/execution-worker-handler.test.ts +++ b/apps/server/src/features/agents/execution/__tests__/execution-worker-handler.test.ts @@ -175,6 +175,44 @@ async function committed(admission: ReturnType, revision: number) } describe("ExecutionWorkerHandler through its scheduler", () => { + it("commits a canonical draft and its parent effects in one operation", async () => { + const { scheduler, writer, lease } = fixture(); + await committed(submit(scheduler, lease, { kind: "start", providerId: "codex", input: START_INPUT }), 1); + await committed(submit(scheduler, lease, { + kind: "event", phase: "running", nativeCursor: null, events: [eventDraft()], + parentLive: { text: { kind: "append", inputs: [TEXT_INPUT] }, narrative: NARRATIVE_INPUT }, + livePublication: [PUBLICATION], + }), 2); + expect(writer.operations[1]).toMatchObject({ + mutation: { + kind: "append-events", events: [eventDraft()], + parentLive: { text: { kind: "append", inputs: [TEXT_INPUT] }, narrative: NARRATIVE_INPUT }, + }, + livePublication: [PUBLICATION], + }); + expect(writer.operations).toHaveLength(2); + scheduler.shutdown(); + }); + + it("rejects a compound event with mismatched parent routing or publication count", async () => { + const { scheduler, handler, writer, lease } = fixture(); + await committed(submit(scheduler, lease, { kind: "start", providerId: "codex", input: START_INPUT }), 1); + for (const command of [ + { kind: "event", phase: "running", nativeCursor: null, events: [eventDraft()], + parentLive: { text: { kind: "append", inputs: [{ ...TEXT_INPUT, executionId: "other" }] } }, livePublication: [PUBLICATION] }, + { kind: "event", phase: "running", nativeCursor: null, events: [eventDraft()], + parentLive: { text: { kind: "unchanged" } }, livePublication: [] }, + ] as const) { + await expect(handler.handle({ + requestId: 2, execution: EXECUTION, lease, ordinal: 2, command, + })).resolves.toMatchObject({ + result: { kind: "rejected", reason: "invalid-text-routing" }, + }); + } + expect(writer.operations).toHaveLength(1); + scheduler.shutdown(); + }); + it("commits one compound live event and forwards its publication receipt", async () => { class PublicationWriter extends RecordingWriter { override async transact(operation: ExecutionSemanticOperation): Promise { diff --git a/apps/server/src/features/agents/execution/__tests__/execution-worker-loss-coordinator.test.ts b/apps/server/src/features/agents/execution/__tests__/execution-worker-loss-coordinator.test.ts index eb3de49e6..cb799595c 100644 --- a/apps/server/src/features/agents/execution/__tests__/execution-worker-loss-coordinator.test.ts +++ b/apps/server/src/features/agents/execution/__tests__/execution-worker-loss-coordinator.test.ts @@ -11,7 +11,7 @@ import { CanonicalExecutionWriterPort } from "../../canonical/canonical-executio import { MessageRepo } from "../../conversation/persistence/message-repo.js"; import { ParentAssistantTextCheckpointService } from "../../turns/parent-assistant-text-checkpoint-service.js"; import type { ExecutionWorkerPort, ExecutionWorkerReply, ExecutionWorkerRequest } from "../execution-mailbox-protocol.js"; -import type { ExecutionMailboxCommand } from "../execution-mailbox-scheduler.js"; +import type { ExecutionLostAssignment, ExecutionMailboxCommand } from "../execution-mailbox-scheduler.js"; import type { ExecutionWorkCommand, ExecutionWorkerResult } from "../execution-worker-handler.js"; import { ExecutionThreadWorkerPort } from "../execution-worker-port.js"; import { ExecutionWorkerLossCoordinator } from "../execution-worker-loss-coordinator.js"; @@ -64,6 +64,7 @@ describe("ExecutionWorkerLossCoordinator with a file-backed writer", () => { let workers: CrashingPort[]; let published: string[]; let recoveryIncidentIds: string[]; + let recoveredAssignments: ExecutionLostAssignment[]; beforeEach(() => { directory = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "mcode-lost-worker-host-")); @@ -78,6 +79,7 @@ describe("ExecutionWorkerLossCoordinator with a file-backed writer", () => { published.push(...events.map((event) => event.eventId)); }); recoveryIncidentIds = []; + recoveredAssignments = []; workers = []; coordinator = new ExecutionWorkerLossCoordinator({ interruptWorkerLoss: (input) => { @@ -92,7 +94,7 @@ describe("ExecutionWorkerLossCoordinator with a file-backed writer", () => { workers.push(worker); return worker; }, - }); + }, (assignment) => recoveredAssignments.push(assignment)); }); afterEach(async () => { @@ -221,6 +223,7 @@ describe("ExecutionWorkerLossCoordinator with a file-backed writer", () => { new ParentAssistantTextCheckpointService(db).appendChunk([{ ...execution, sequence: 1, text: "Partial answer" }]); workers[0]?.crash(); expect(await coordinator.waitForRecovery(0)).toEqual({ kind: "recovered", workerIndex: 0 }); + expect(recoveredAssignments).toEqual([{ execution, lease }]); expect(workers).toHaveLength(2); expect(db.prepare("SELECT terminal_outcome, recovery_incident_id FROM canonical_agent_ingest_checkpoints WHERE execution_id = ?") .get(execution.executionId)).toMatchObject({ terminal_outcome: "interrupted", diff --git a/apps/server/src/features/agents/execution/__tests__/execution-worker-parent-events.test.ts b/apps/server/src/features/agents/execution/__tests__/execution-worker-parent-events.test.ts new file mode 100644 index 000000000..e3da0eb25 --- /dev/null +++ b/apps/server/src/features/agents/execution/__tests__/execution-worker-parent-events.test.ts @@ -0,0 +1,297 @@ +import { AgentEventSchema, ProviderRuntimeEventSchema, type AgentEvent } from "@mcode/contracts"; +import type { ProviderEventDraft } from "@mcode/providers"; +import * as NodeFSPromises from "node:fs/promises"; +import * as NodePath from "node:path"; +import * as NodeOS from "node:os"; +import { afterEach, describe, expect, it } from "vitest"; +import { ExecutionWorkerHandler, type ExecutionSemanticOperation, type ExecutionSemanticWriter, + type ExecutionWorkCommand, type ExecutionWriteReceipt } from "../execution-worker-handler.js"; +import { TurnFileTracker } from "../../turns/turn-file-tracker.js"; + +const execution = { threadId: "thread", turnId: "turn", executionId: "11111111-1111-4111-8111-111111111111" }; +const lease = { ownerEpoch: 1, workerIndex: 0, workerGeneration: 1, leaseId: "lease-1" }; +const now = "2026-09-25T12:00:00.000Z"; +const directories: string[] = []; + +afterEach(async () => { + await Promise.all(directories.splice(0).map((path) => NodeFSPromises.rm(path, { recursive: true, force: true }))); +}); + +async function fileFixture() { + const cwd = await NodeFSPromises.mkdtemp(NodePath.join(NodeOS.tmpdir(), "mcode-handler-files-")); + directories.push(cwd); + const tracker = new TurnFileTracker(async () => ({ kind: "unavailable" }), () => {}, process.platform); + const handoff = tracker.beginExecutionTurn({ ...execution, cwd, baselineRef: null }); + return { cwd, tracker, handoff }; +} + +function eventDraft(sequence: number, type: AgentEvent["type"], fields: Record = {}): ProviderEventDraft { + const event = AgentEventSchema().parse({ type, threadId: execution.threadId, turnExecutionId: execution.executionId, ...fields }); + const itemId = `item-${sequence}`; + return { eventId: `event-${sequence}`, routing: { ...execution, itemId }, sourceProviderId: "codex", sourceIdentities: [], + sourceSequence: sequence, payload: { type: "item.recorded", item: { + id: itemId, threadId: execution.threadId, turnId: execution.turnId, kind: "system", providerIdentities: [], + payload: { projection: "providerRuntimeEvent", runtimeEvent: { event } }, createdAt: now, updatedAt: now, + } } }; +} + +function start(): Extract { + return { kind: "start", providerId: "codex", parentLive: { planFeature: "none", precedingMessageId: "user" }, + input: { thread: { id: execution.threadId, workspaceId: "workspace", providerId: "codex", createdAt: now }, + turnId: execution.turnId, executionId: execution.executionId, permissionMode: "full", providerIdentities: [], + userMessage: { kind: "create", messageId: "user", content: "Task", sequence: 1 } } }; +} + +class Writer implements ExecutionSemanticWriter { + readonly operations: ExecutionSemanticOperation[] = []; + fail = false; + throwFailure = false; + async transact(operation: ExecutionSemanticOperation): Promise { + this.operations.push(operation); + if (this.throwFailure) throw new Error("writer disconnected"); + return this.fail ? { kind: "conflict", operationId: operation.operationId } + : { kind: "committed", operationId: operation.operationId, durableRevision: this.operations.length }; + } +} + +function fixture() { + const writer = new Writer(); + const handler = new ExecutionWorkerHandler(writer); + const send = (ordinal: number, command: ExecutionWorkCommand) => handler.handle({ requestId: ordinal, execution, lease, ordinal, command }); + const event = (ordinal: number, draft: ProviderEventDraft) => send(ordinal, { + kind: "event", phase: "running", nativeCursor: null, events: [draft], + }); + const stop = (ordinal: number) => handler.handle({ requestId: ordinal, execution, lease, ordinal, + stopWatermark: ordinal - 1, command: { kind: "stop", requestId: "user-stop" } }); + return { writer, send, event, stop }; +} + +describe("execution worker parent event ownership", () => { + it.each(["codex", "claude", "cursor"] as const)("preserves %s partial text and narrative when Stop finishes from worker state", async (providerId) => { + const { writer, send, event, stop } = fixture(); + const command = start(); + await send(1, { ...command, providerId, input: { ...command.input, thread: { ...command.input.thread, providerId } } }); + const draft = (sequence: number, type: AgentEvent["type"], fields: Record = {}) => ({ + ...eventDraft(sequence, type, fields), sourceProviderId: providerId, + }); + await event(2, draft(1, "turnStarted")); + await event(3, draft(2, "textDelta", { delta: "Checking files", isFinalResponse: false })); + await event(4, draft(3, "textDelta", { delta: "Partial unclassified answer" })); + expect((await stop(5)).result.kind).toBe("committed"); + const reply = await send(6, { kind: "finish-from-state", outcome: "cancelled", + input: { ...execution, providerId, providerIdentities: [], outcome: "cancelled", projection: { kind: "writer-staged" } } }); + expect(reply.result.kind).toBe("committed"); + expect(writer.operations).toHaveLength(6); + expect(writer.operations[5]).toMatchObject({ mutation: { kind: "finish-live-event", outcome: "cancelled", + projection: { assistant: { content: "Partial unclassified answer" }, narrative: [ + expect.objectContaining({ kind: "narrationSegment", record: expect.objectContaining({ text: "Checking files" }) }), + ] } }, livePublication: [{ after: "terminal", event: { type: "ended", outcome: "interrupted" } }] }); + expect(writer.operations[5]?.mutation).not.toHaveProperty("providerEvent"); + expect((await send(7, { kind: "release" })).result).toEqual({ kind: "released" }); + }); + + it("finishes a setup error before turnStarted without losing the admitted execution", async () => { + const { writer, send } = fixture(); + await send(1, start()); + const input = { ...execution, providerId: "codex", providerIdentities: [], outcome: "errored", projection: { kind: "writer-staged" } } as const; + expect((await send(2, { kind: "finish-from-state", outcome: "errored", input })).result.kind).toBe("rejected"); + const reply = await send(2, { kind: "finish-from-state", outcome: "errored", input: { ...input, error: "Provider executable missing" } }); + expect(reply.result.kind).toBe("committed"); + expect(writer.operations).toHaveLength(2); + expect(writer.operations[1]).toMatchObject({ mutation: { kind: "finish-live-event", outcome: "errored", + projection: { assistant: { content: "" }, narrative: [] } }, + livePublication: [{ event: { type: "error", error: "Provider executable missing" } }], + }); + }); + + it("settles captured edits on the worker before its single terminal writer call", async () => { + const { cwd, tracker, handoff } = await fileFixture(); + const { writer, send } = fixture(); + const file = NodePath.join(cwd, "tracked.txt"); + await NodeFSPromises.writeFile(file, "before\n"); + await send(1, start()); + expect((await send(2, { kind: "begin-files", cwd, handoff, deliveryAttempt: 1 })).result.kind).toBe("committed"); + const command = (draft: ProviderEventDraft): Extract => ({ + kind: "event", phase: "running", nativeCursor: null, deliveryAttempt: 1, events: [draft], + }); + await send(3, command(eventDraft(1, "turnStarted"))); + const toolInput = { file_path: "tracked.txt" }; + const captured = tracker.captureToolUseObservation(execution.threadId, "edit", "Edit", toolInput); + if (!captured) throw new Error("Expected captured file evidence"); + await NodeFSPromises.writeFile(file, "after\nextra\n"); + const use = eventDraft(2, "toolUse", { toolCallId: "edit", toolName: "Edit", toolInput }); + expect((await send(4, { ...command(use), capturedFileObservation: captured })).result.kind).toBe("committed"); + expect((await send(5, command(eventDraft(3, "toolResult", { toolCallId: "edit", output: "Done", isError: false })))).result.kind).toBe("committed"); + const terminal = eventDraft(4, "error", { error: "Disconnected" }); + const reply = await send(6, { ...command(terminal), + terminalInput: { ...execution, providerId: "codex", providerIdentities: [], outcome: "errored", projection: { kind: "writer-staged" } }, + frozenFileEvidence: { handoff, turnId: execution.turnId, deliveryAttempt: 1, outcome: "errored", nativeDiff: null } }); + expect(reply.result.kind).toBe("committed"); + expect(writer.operations).toHaveLength(6); + expect(writer.operations[5]?.mutation).toMatchObject({ kind: "finish-live-event", input: { + error: "Disconnected", fileEvidence: { filesChanged: ["tracked.txt"], fileEffects: { fileCount: 1, additions: 2, deletions: 1 } }, + } }); + expect((await send(7, { kind: "release" })).result).toEqual({ kind: "released" }); + }); + + it("fences attempts and poisons a terminal whose file evidence is missing", async () => { + const { cwd, handoff } = await fileFixture(); + const { writer, send } = fixture(); + await send(1, start()); + await send(2, { kind: "begin-files", cwd, handoff, deliveryAttempt: 1 }); + const started = { kind: "event", phase: "running", nativeCursor: null, events: [eventDraft(1, "turnStarted")] } satisfies ExecutionWorkCommand; + expect((await send(3, { ...started, deliveryAttempt: 2 })).result).toMatchObject({ kind: "rejected", reason: "invalid-event-routing" }); + expect((await send(3, { ...started, deliveryAttempt: 1 })).result.kind).toBe("committed"); + const terminal = { ...started, events: [eventDraft(2, "error", { error: "Disconnected" })], deliveryAttempt: 1, + terminalInput: { ...execution, providerId: "codex", providerIdentities: [], outcome: "errored", projection: { kind: "writer-staged" } }, + } satisfies ExecutionWorkCommand; + expect((await send(4, terminal)).result).toMatchObject({ kind: "rejected", reason: "invalid-event-routing" }); + expect((await send(4, terminal)).result).toMatchObject({ kind: "rejected", reason: "invalid-transition" }); + expect(writer.operations).toHaveLength(3); + }); + + it.each(["claude", "cursor"] as const)("prepares %s parent text, message features, and terminal writes", async (providerId) => { + const { writer, send, event } = fixture(); + const command = start(); + await send(1, { ...command, providerId, input: { ...command.input, thread: { ...command.input.thread, providerId } } }); + const draft = (sequence: number, type: AgentEvent["type"], fields: Record = {}) => ({ + ...eventDraft(sequence, type, fields), sourceProviderId: providerId, + }); + await event(2, draft(1, "turnStarted")); + await event(3, draft(2, "textDelta", { delta: "Answer", isFinalResponse: true })); + expect(writer.operations[2]?.mutation).toMatchObject({ parentLive: { + text: { inputs: [{ sequence: 1, text: "Answer" }] }, + } }); + const message = await event(4, draft(3, "message", { content: "Answer", tokens: null })); + expect(message.result).toMatchObject({ kind: "committed", parentEvent: { + runtime: expect.arrayContaining([{ kind: "assistant-message-feature", providerId, + event: expect.objectContaining({ type: "message", content: "Answer" }) }]), + } }); + const terminal = draft(4, "error", { error: "Disconnected" }); + const reply = await send(5, { kind: "event", phase: "running", nativeCursor: null, events: [terminal], + terminalInput: { ...execution, providerId, providerIdentities: [], outcome: "errored", projection: { kind: "writer-staged" } } }); + expect(reply.result.kind).toBe("committed"); + expect(writer.operations[4]?.mutation).toMatchObject({ kind: "finish-live-event", input: { providerId }, providerEvent: { events: [terminal] } }); + }); + + it.each(["claude", "cursor"] as const)("rejects unsupported %s events and another provider's drafts", async (providerId) => { + const { writer, send, event } = fixture(); + const command = start(); + await send(1, { ...command, providerId, input: { ...command.input, thread: { ...command.input.thread, providerId } } }); + expect((await event(2, eventDraft(1, "turnStarted"))).result).toMatchObject({ kind: "rejected", reason: "invalid-event-routing" }); + expect((await event(2, { ...eventDraft(2, "textDelta", { delta: "Before start" }), sourceProviderId: providerId })).result) + .toMatchObject({ kind: "rejected", reason: "invalid-event-routing" }); + expect(writer.operations).toHaveLength(1); + }); + + it("derives text writes inside the worker and commits them with their canonical draft", async () => { + const { writer, send, event } = fixture(); + await send(1, start()); + const started = await event(2, eventDraft(1, "turnStarted")); + expect(started.result).toMatchObject({ kind: "committed", parentEvent: { runtime: [{ kind: "turn-started" }] } }); + const draft = eventDraft(2, "textDelta", { delta: "Answer", isFinalResponse: true }); + const reply = await event(3, draft); + expect(reply.result.kind).toBe("committed"); + expect(writer.operations[2]).toMatchObject({ + mutation: { kind: "append-events", events: [draft], parentLive: { + text: { kind: "append", inputs: [{ ...execution, sequence: 1, text: "Answer" }] }, + } }, livePublication: [{ after: "writer", event: { type: "textDelta", delta: "Answer" } }], + }); + }); + + it("rejects caller parent effects when the worker owns the reducer", async () => { + const { writer, send } = fixture(); + await send(1, start()); + const reply = await send(2, { kind: "event", phase: "running", nativeCursor: null, + events: [eventDraft(1, "turnStarted")], parentLive: { text: { kind: "unchanged" } } }); + expect(reply.result).toMatchObject({ kind: "rejected", reason: "invalid-event-routing" }); + expect(writer.operations).toHaveLength(1); + }); + + it("rejects malformed parent identity before writing or advancing the reducer", async () => { + const { writer, send, event } = fixture(); + await send(1, start()); + const stale = eventDraft(1, "turnStarted"); + if (stale.payload.type !== "item.recorded") throw new Error("Expected runtime item"); + const reply = await event(2, { ...stale, payload: { ...stale.payload, item: { ...stale.payload.item, turnId: "other" } } }); + expect(reply.result).toMatchObject({ kind: "rejected", reason: "invalid-event-routing" }); + expect(writer.operations).toHaveLength(1); + }); + + it("poisons prepared reducer state when its write conflicts", async () => { + const { writer, send, event } = fixture(); + await send(1, start()); + await event(2, eventDraft(1, "turnStarted")); + writer.fail = true; + const draft = eventDraft(2, "textDelta", { delta: "Answer" }); + expect((await event(3, draft)).result).toMatchObject({ kind: "rejected", reason: "writer-conflict" }); + writer.fail = false; + expect((await event(3, draft)).result).toMatchObject({ kind: "rejected", reason: "invalid-transition" }); + expect(writer.operations).toHaveLength(3); + }); + + it("poisons prepared reducer state when the writer throws", async () => { + const { writer, send, event } = fixture(); + await send(1, start()); + writer.throwFailure = true; + await expect(event(2, eventDraft(1, "turnStarted"))).rejects.toThrow("writer disconnected"); + writer.throwFailure = false; + expect((await event(2, eventDraft(1, "turnStarted"))).result).toMatchObject({ kind: "rejected", reason: "invalid-transition" }); + expect(writer.operations).toHaveLength(2); + }); + + it("requires a matching start message identity and supported provider for live ownership", async () => { + for (const command of [ + { ...start(), parentLive: { planFeature: "none", precedingMessageId: "other-user" } }, + { ...start(), providerId: "claude" }, + ] satisfies ExecutionWorkCommand[]) { + const { writer, send } = fixture(); + expect((await send(1, command)).result).toMatchObject({ kind: "rejected", reason: "invalid-transition" }); + expect(writer.operations).toHaveLength(0); + } + }); + + it("keeps child runtime evidence on the writer path without changing parent text", async () => { + const { writer, send, event } = fixture(); + await send(1, start()); + await event(2, eventDraft(1, "turnStarted")); + const child = eventDraft(2, "textDelta", { delta: "Child answer" }); + if (child.payload.type !== "item.recorded") throw new Error("Expected runtime item"); + const runtime = ProviderRuntimeEventSchema().parse(child.payload.item.payload.runtimeEvent); + const reply = await event(3, { ...child, payload: { ...child.payload, item: { ...child.payload.item, + payload: { ...child.payload.item.payload, runtimeEvent: { ...runtime, + extension: { providerId: "codex", kind: "codex-collaboration", + child: { nativeThreadId: "child-thread", parentCollaborationItemId: "spawn-1" } } } }, + } } }); + expect(reply.result).toMatchObject({ kind: "committed" }); + expect(reply.result).not.toHaveProperty("parentEvent"); + expect(writer.operations[2]?.mutation).not.toHaveProperty("parentLive"); + await event(4, eventDraft(3, "textDelta", { delta: "Parent answer" })); + expect(writer.operations[3]?.mutation).toMatchObject({ parentLive: { + text: { inputs: [{ sequence: 1, text: "Parent answer" }] }, + } }); + }); + + it("requires terminal evidence then commits the terminal draft with finish in one operation", async () => { + const { writer, send, event } = fixture(); + await send(1, start()); + await event(2, eventDraft(1, "turnStarted")); + await event(3, eventDraft(2, "textDelta", { delta: "Partial answer", isFinalResponse: true })); + const terminal = eventDraft(3, "error", { error: "Disconnected" }); + expect((await event(4, terminal)).result).toMatchObject({ kind: "rejected", reason: "invalid-event-routing" }); + expect(writer.operations).toHaveLength(3); + const reply = await send(4, { kind: "event", phase: "running", nativeCursor: null, events: [terminal], + terminalInput: { ...execution, providerId: "codex", providerIdentities: [], outcome: "errored", + projection: { kind: "writer-staged" } } }); + expect(reply.result).toMatchObject({ kind: "committed", parentEvent: { + terminal: { outcome: "errored", assistant: { content: "Partial answer" } }, + } }); + expect(writer.operations).toHaveLength(4); + expect(writer.operations[3]).toMatchObject({ mutation: { kind: "finish-live-event", + providerEvent: { events: [terminal] }, outcome: "errored", + projection: { assistant: { content: "Partial answer" } } }, + livePublication: [{ after: "terminal", event: { type: "error", error: "Disconnected" } }], + }); + expect((await send(5, { kind: "release" })).result).toEqual({ kind: "released" }); + }); +}); diff --git a/apps/server/src/features/agents/execution/__tests__/provider-live-event-effects.test.ts b/apps/server/src/features/agents/execution/__tests__/provider-live-event-effects.test.ts new file mode 100644 index 000000000..63e584e20 --- /dev/null +++ b/apps/server/src/features/agents/execution/__tests__/provider-live-event-effects.test.ts @@ -0,0 +1,90 @@ +import { AgentEventSchema, type AgentEvent } from "@mcode/contracts"; +import { describe, expect, it } from "vitest"; + +import { OtherProviderLiveEventEffects } from "../provider-live-event-effects.js"; + +const execution = { + threadId: "provider-thread", + turnId: "provider-turn", + executionId: "11111111-1111-4111-8111-111111111111", +}; + +function event(type: AgentEvent["type"], fields: Record = {}): AgentEvent { + return AgentEventSchema().parse({ type, threadId: execution.threadId, + turnExecutionId: execution.executionId, ...fields }); +} + +describe("OtherProviderLiveEventEffects", () => { + it("prepares Claude text, assistant message, cursor, and terminal writes for one execution", () => { + const owner = new OtherProviderLiveEventEffects("claude", execution, "user-message"); + const start = owner.prepare(event("turnStarted")); + expect(start).toMatchObject({ kind: "prepared", providerId: "claude", execution, + runtime: [{ kind: "turn-started" }] }); + + const cursor = owner.prepare(event("system", { subtype: "sdk_session_id:claude-session" })); + expect(cursor).toMatchObject({ kind: "prepared", effects: { + systemIntents: [{ kind: "session-cursor", event: { subtype: "sdk_session_id:claude-session" } }], + } }); + + const delta = owner.prepare(event("textDelta", { delta: "Answer" })); + expect(delta).toMatchObject({ kind: "prepared", effects: { + text: { kind: "append", inputs: [{ ...execution, sequence: 1, text: "Answer" }] }, + } }); + expect(owner.prepare(event("assistantMessageBoundary", { isFinalResponse: true }))).toMatchObject({ + kind: "prepared", publication: { after: "writer" }, + }); + const message = owner.prepare(event("message", { content: "Answer", tokens: null })); + expect(message).toMatchObject({ kind: "prepared", effects: { + message: { content: "Answer", precedingMessageId: "user-message" }, + }, runtime: [{ kind: "assistant-message-feature", providerId: "claude", + event: { type: "message", content: "Answer" } }] }); + const terminal = owner.prepare(event("turnComplete", { + reason: "completed", costUsd: null, tokensIn: 4, tokensOut: 2, + }), "2026-09-25T00:00:00.000Z"); + expect(terminal).toMatchObject({ kind: "prepared", publication: { after: "terminal" }, + terminal: { threadId: execution.threadId, executionId: execution.executionId, + outcome: "completed", endedAt: "2026-09-25T00:00:00.000Z", + assistant: { content: "Answer" } } }); + expect(structuredClone(terminal)).toEqual(terminal); + expect(owner.prepare(event("hookStarted", { hookName: "stop", hookType: "stop" }))) + .toMatchObject({ kind: "prepared", publication: { after: "terminal" }, + runtime: [{ kind: "hook-started", late: true }] }); + expect(owner.prepare(event("message", { content: "Goal achieved in 2s.", tokens: null }))) + .toMatchObject({ kind: "unsupported", reason: expect.stringContaining("goal receipt") }); + }); + + it("prepares Cursor unknown text and its final message without a boundary", () => { + const owner = new OtherProviderLiveEventEffects("cursor", execution, "user-message"); + expect(owner.prepare(event("turnStarted"))).toMatchObject({ kind: "prepared" }); + expect(owner.prepare(event("textDelta", { delta: "Working" }))).toMatchObject({ + kind: "prepared", effects: { text: { kind: "append" } }, + }); + expect(owner.prepare(event("message", { content: "Done", tokens: null }))).toMatchObject({ + kind: "prepared", effects: { message: { content: "Done" } }, + runtime: [{ kind: "assistant-message-feature", providerId: "cursor" }], + }); + expect(owner.prepare(event("turnComplete", { + reason: "completed", costUsd: null, tokensIn: 0, tokensOut: 0, + }))).toMatchObject({ kind: "prepared", terminal: { outcome: "completed", + assistant: { content: "Done" } } }); + }); + + it("rejects an unbound or stale event without advancing the owner", () => { + const owner = new OtherProviderLiveEventEffects("claude", execution, "user-message"); + expect(owner.prepare(event("turnStarted", { turnExecutionId: undefined }))).toMatchObject({ + kind: "unsupported", reason: expect.stringContaining("without execution identity"), + }); + expect(owner.prepare(event("turnStarted", { + turnExecutionId: "22222222-2222-4222-8222-222222222222", + }))).toMatchObject({ + kind: "unsupported", reason: "different execution", + }); + expect(owner.prepare(event("turnStarted"))).toMatchObject({ kind: "prepared" }); + expect(owner.prepare(event("turnComplete", { + reason: "completed", costUsd: null, tokensIn: 0, tokensOut: 0, providerId: "codex", + }))).toMatchObject({ kind: "unsupported", reason: "different provider" }); + expect(owner.prepare(event("turnComplete", { + reason: "completed", costUsd: null, tokensIn: 0, tokensOut: 0, providerId: "claude", + }))).toMatchObject({ kind: "prepared", terminal: { outcome: "completed" } }); + }); +}); diff --git a/apps/server/src/features/agents/execution/__tests__/worker-owned-turn-runtime.test.ts b/apps/server/src/features/agents/execution/__tests__/worker-owned-turn-runtime.test.ts new file mode 100644 index 000000000..67720c9f2 --- /dev/null +++ b/apps/server/src/features/agents/execution/__tests__/worker-owned-turn-runtime.test.ts @@ -0,0 +1,73 @@ +import "reflect-metadata"; +import * as NodeFS from "node:fs"; +import * as NodeOS from "node:os"; +import * as NodePath from "node:path"; +import { afterEach, expect, it } from "vitest"; + +import { openDatabase } from "../../../../runtime/persistence/sqlite/database.js"; +import { AgentEventPublicationRegistry } from "../../orchestration/agent-event-publication-registry.js"; +import { WorkerOwnedTurnRuntime } from "../worker-owned-turn-runtime.js"; + +const NOW = "2026-09-25T10:00:00.000Z"; +const identities = Array.from({ length: 5 }, (_, index) => ({ + threadId: `failure-isolation-thread-${index}`, + turnId: `failure-isolation-turn-${index}`, + executionId: `00000000-0000-4000-8000-0000000002${String(index).padStart(2, "0")}`, +})); + +let directory: string | undefined; +let database: ReturnType | undefined; +let runtime: WorkerOwnedTurnRuntime | undefined; + +afterEach(async () => { + await runtime?.close(); + database?.close(true); + if (directory) NodeFS.rmSync(directory, { recursive: true, force: true }); + runtime = undefined; + database = undefined; + directory = undefined; +}); + +it("contains a rejected writer operation to its execution while a slot peer keeps running", async () => { + directory = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "mcode-rejected-execution-")); + const dbPath = NodePath.join(directory, "app.sqlite"); + database = openDatabase({ dbPath }); + database.prepare("INSERT INTO workspaces (id, name, path, created_at, updated_at) VALUES (?, ?, ?, ?, ?)") + .run("failure-isolation-workspace", "Workspace", directory, NOW, NOW); + const insertThread = database.prepare("INSERT INTO threads (id, workspace_id, title, branch, provider, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?)"); + for (const identity of identities) { + insertThread.run(identity.threadId, "failure-isolation-workspace", "Thread", "main", "codex", NOW, NOW); + } + runtime = new WorkerOwnedTurnRuntime(dbPath, new AgentEventPublicationRegistry()); + await runtime.whenReady(); + for (const identity of identities) { + await runtime.owner.start({ execution: identity, ownerEpoch: 1, providerId: "codex", parentTurn: { + thread: { id: identity.threadId, workspaceId: "failure-isolation-workspace", providerId: "codex", createdAt: NOW }, + turnId: identity.turnId, executionId: identity.executionId, permissionMode: "supervised", + providerIdentities: [], userMessage: { kind: "create", messageId: `${identity.threadId}-user`, content: "Question", sequence: 1 }, + } }); + } + const failed = identities[0]!; + const peer = identities[4]!; + expect(runtime.owner.current(failed.threadId)?.lease.workerIndex) + .toBe(runtime.owner.current(peer.threadId)?.lease.workerIndex); + database.run(`CREATE TRIGGER fail_one_append BEFORE INSERT ON canonical_writer_operation_receipts + WHEN NEW.kind = 'semantic:append-events' AND NEW.execution_id = '${failed.executionId}' + BEGIN SELECT RAISE(ABORT, 'injected append failure'); END`); + await expect(runtime.owner.submit(failed, { kind: "event", phase: "running", nativeCursor: null, events: [{ + eventId: `${failed.executionId}:event:1`, routing: { ...failed, itemId: "failed-tool" }, + sourceProviderId: "codex", sourceIdentities: [], sourceSequence: 1, + payload: { type: "item.recorded", item: { id: "failed-tool", threadId: failed.threadId, turnId: failed.turnId, + kind: "tool-call", providerIdentities: [], payload: { projection: "toolCall", toolName: "Read", path: "CONTEXT.md" }, + createdAt: NOW, updatedAt: NOW } }, + }] })).rejects.toThrow("writer-failure"); + await runtime.recoverRejected(failed); + expect(runtime.owner.current(failed.threadId)).toBeUndefined(); + expect(database.prepare("SELECT terminal_outcome FROM canonical_agent_ingest_checkpoints WHERE execution_id = ?") + .get(failed.executionId)).toEqual({ terminal_outcome: "interrupted" }); + expect(database.prepare("SELECT terminal_outcome FROM canonical_agent_ingest_checkpoints WHERE execution_id = ?") + .get(peer.executionId)).toEqual({ terminal_outcome: null }); + await expect(runtime.owner.submit(peer, { kind: "checkpoint", phase: "running", nativeCursor: null })) + .resolves.toMatchObject({ kind: "committed" }); + expect(runtime.owner.isStarted(peer)).toBe(true); +}); diff --git a/apps/server/src/features/agents/execution/codex-live-event-effects.ts b/apps/server/src/features/agents/execution/codex-live-event-effects.ts new file mode 100644 index 000000000..ecc28ae40 --- /dev/null +++ b/apps/server/src/features/agents/execution/codex-live-event-effects.ts @@ -0,0 +1,157 @@ +import type { DataOnlyParentTerminalProjectionInput } from "../canonical/canonical-parent-turn-write.js"; +import type { CodexSystemWriterIntent } from "../canonical/canonical-codex-system-error-projection.js"; +import { taskToolWriteIntents, type TaskToolCall } from "../tasks/task-tool-intent-reducer.js"; +import { NarrativeRecoveryDelta } from "../turns/narrative-recovery-delta.js"; +import { deriveTurnAssistantMessageId } from "../turns/turn-assistant-message-id.js"; +import type { ParentAssistantTextCheckpointInput } from "../turns/parent-assistant-text-checkpoint-service.js"; +import type { CodexLiveReduction, CodexLiveWriterIntent } from "./codex-live-event-reducer.js"; +import type { ExecutionIdentity } from "./execution-mailbox-protocol.js"; +import type { ExecutionLivePublicationIntent, ParentLiveEffects } from "./execution-worker-handler.js"; + +type ReducedEvent = Extract; +type TerminalIntent = Extract; + +/** Effects that still require an execution runtime or a canonical projection owner. */ +export type CodexLiveRuntimeIntent = Extract | { readonly kind: "feature-event"; readonly feature: "goal-refresh"; readonly event: ReducedEvent["publication"]["event"] }; + +/** Cloneable parent writes and the remaining work required before publication. */ +export interface PreparedCodexLiveEvent { + readonly effects: ParentLiveEffects; + readonly publication: ExecutionLivePublicationIntent; + readonly runtime: readonly CodexLiveRuntimeIntent[]; + readonly terminal?: DataOnlyParentTerminalProjectionInput; +} + +/** + * Maps one execution's reduced events to writer data. Advance only in mailbox order; + * discard this mapper and its reducer if a prepared operation fails to commit. + */ +export class CodexLiveEventEffects { + private sequence = 0; + private readonly narrative = new NarrativeRecoveryDelta(); + private readonly calls = new Map(); + private assignedMessageId: string | undefined; + + constructor(private readonly execution: ExecutionIdentity, private readonly precedingMessageId: string) {} + + /** Prepare every parent write while retaining intents owned by other execution collaborators. */ + prepare(reduction: ReducedEvent, endedAt = new Date().toISOString()): PreparedCodexLiveEvent { + this.requireExecution(reduction); + let effects: ParentLiveEffects = { text: { kind: "unchanged" } }; + const runtime: CodexLiveRuntimeIntent[] = []; + let terminal: DataOnlyParentTerminalProjectionInput | undefined; + for (const intent of reduction.writer) { + if (intent.kind === "terminal-projection") terminal = this.terminalProjection(intent, endedAt); + else effects = this.applyIntent(intent, effects, runtime); + } + const event = reduction.publication.event; + const publication = event.type === "message" && effects.message + ? { ...reduction.publication, event: { ...event, messageId: effects.message.messageId } } + : reduction.publication; + return structuredClone({ effects, publication, runtime, ...(terminal ? { terminal } : {}) }); + } + + private requireExecution(reduction: ReducedEvent): void { + const actual = reduction.execution; + const event = reduction.publication.event; + if (actual.threadId !== this.execution.threadId || actual.turnId !== this.execution.turnId + || actual.executionId !== this.execution.executionId || event.threadId !== actual.threadId + || event.turnExecutionId !== actual.executionId) throw new Error("Codex effects belong to another execution"); + } + + private applyIntent( + intent: Exclude, + effects: ParentLiveEffects, + runtime: CodexLiveRuntimeIntent[], + ): ParentLiveEffects { + switch (intent.kind) { + case "assistant-text-delta": + return intent.delta ? { ...effects, text: { kind: "append", inputs: [this.checkpoint(intent.delta)] } } : effects; + case "assistant-text-promote": + return { ...effects, text: { kind: "promote", input: this.checkpoint(intent.text) } }; + case "assistant-text-reclassify": + this.sequence = 0; + return { ...effects, text: { kind: "reclassify", expectedText: intent.text } }; + default: return this.applyNonTextIntent(intent, effects, runtime); + } + } + + private applyNonTextIntent( + intent: Exclude, + effects: ParentLiveEffects, + runtime: CodexLiveRuntimeIntent[], + ): ParentLiveEffects { + switch (intent.kind) { + case "assistant-body": return this.messageEffects(intent, effects); + case "narrative-recovery": return this.narrativeEffects(intent, effects); + case "feature-event": return this.featureEffects(intent, effects, runtime); + case "plan-questions": return { ...effects, planQuestions: intent.questions }; + case "plan-output": return { ...effects, planOutput: intent.output }; + case "notice-session": + case "system-notice": + case "session-cursor": return this.systemEffects(intent, effects); + default: return this.runtimeEffects(intent, effects, runtime); + } + } + + private checkpoint(text: string): ParentAssistantTextCheckpointInput { + return { ...this.execution, sequence: ++this.sequence, text }; + } + + private messageEffects(intent: Extract, effects: ParentLiveEffects): ParentLiveEffects { + this.assignedMessageId ??= deriveTurnAssistantMessageId(this.execution.threadId, this.precedingMessageId); + return { ...effects, message: { precedingMessageId: this.precedingMessageId, messageId: this.assignedMessageId, + content: intent.content, model: intent.model, attachments: intent.attachments } }; + } + + private narrativeEffects(intent: Extract, effects: ParentLiveEffects): ParentLiveEffects { + const delta = this.narrative.prepare(intent.items); + if (!delta) return effects; + delta.acknowledge(); + return { ...effects, narrative: { executionId: this.execution.executionId, + items: delta.items, discardedItemIds: delta.discardedItemIds } }; + } + + private featureEffects( + intent: Extract, + effects: ParentLiveEffects, + runtime: CodexLiveRuntimeIntent[], + ): ParentLiveEffects { + if (intent.feature === "goal-refresh") runtime.push({ ...intent, feature: "goal-refresh" }); + // The reducer emits parsed plans and assistant bodies as their own intents. + if (intent.feature !== "task-tool") return effects; + const event = intent.event; + const bufferedCalls = [...this.calls.values()]; + const taskIntents = event.type === "toolUse" + ? taskToolWriteIntents({ kind: "tool-use", ...event, bufferedCalls }) + : event.type === "toolResult" + ? taskToolWriteIntents({ kind: "tool-result", ...event, bufferedCalls }) : []; + return taskIntents.length ? { ...effects, taskIntents } : effects; + } + + private systemEffects(intent: CodexSystemWriterIntent, effects: ParentLiveEffects): ParentLiveEffects { + return { ...effects, systemIntents: [...effects.systemIntents ?? [], intent] }; + } + + private runtimeEffects(intent: CodexLiveRuntimeIntent, effects: ParentLiveEffects, runtime: CodexLiveRuntimeIntent[]): ParentLiveEffects { + if (intent.kind === "tool-use") { + const event = intent.event; + this.calls.set(event.toolCallId, { toolCallId: event.toolCallId, toolName: event.toolName, + parentToolCallId: event.parentToolCallId, _rawToolInput: event.toolInput }); + } + runtime.push(intent); + return effects; + } + + private terminalProjection(intent: TerminalIntent, endedAt: string): DataOnlyParentTerminalProjectionInput { + return { threadId: this.execution.threadId, executionId: this.execution.executionId, + outcome: intent.outcome, endedAt, + assistant: { content: intent.assistant.content, model: intent.assistant.model, + attachments: intent.assistant.attachments, ...(this.assignedMessageId ? { messageId: this.assignedMessageId } : {}) }, + narrative: intent.narrative }; + } +} diff --git a/apps/server/src/features/agents/execution/codex-live-event-reducer.ts b/apps/server/src/features/agents/execution/codex-live-event-reducer.ts index 9ff0fda9e..b10d0bec5 100644 --- a/apps/server/src/features/agents/execution/codex-live-event-reducer.ts +++ b/apps/server/src/features/agents/execution/codex-live-event-reducer.ts @@ -1,4 +1,4 @@ -import type { AgentEvent, ParentNarrativeRecoveryItem, PlanQuestion, StoredAttachment } from "@mcode/contracts"; +import type { AgentEvent, ParentNarrativeRecoveryItem, PlanQuestion, StoredAttachment, TurnOutcome } from "@mcode/contracts"; import { NarrativeTurnState, type NarrativeTurnStateEffect } from "../conversation/narrative/narrative-turn-state.js"; import { AssistantExecutionState, type AssistantMaterializationInput } from "../turns/assistant-execution-state.js"; import { PlanExecutionState, type PlanPersistenceReady } from "../planning/plan-execution-state.js"; @@ -15,6 +15,11 @@ const MAX_PLAN_TEXT_BYTES = 256 * 1024; /** Which plan parser, if any, was armed when this execution began. */ export type CodexPlanFeature = "none" | "questions" | "output"; +/** A runtime termination carries its actual failure or cancellation outcome. */ +export type SyntheticTerminalInput = + | { readonly outcome: "errored"; readonly error: string } + | { readonly outcome: "cancelled" | "interrupted" }; + const BEFORE_START_EVENTS = new Set([ "system", "quotaUpdate", "goalUpdated", "goalCleared", "mcpServerStartupStatus", ]); @@ -83,7 +88,7 @@ export type CodexLiveWriterIntent = | { readonly kind: "system-notice"; readonly event: SystemEvent } | { readonly kind: "session-cursor"; readonly event: SystemEvent } | { readonly kind: "turn-error"; readonly error: string } - | { readonly kind: "terminal-projection"; readonly source: "turnComplete" | "error" | "ended"; readonly outcome: "completed" | "errored" | "interrupted"; readonly assistant: AssistantMaterializationInput; readonly narrative: ParentNarrativeRecoveryItem[] } + | { readonly kind: "terminal-projection"; readonly source: "turnComplete" | "error" | "ended"; readonly outcome: TurnOutcome; readonly assistant: AssistantMaterializationInput; readonly narrative: ParentNarrativeRecoveryItem[] } | { readonly kind: "turn-ended" }; /** A publication is released only after the writer accepts every intent for the event. */ @@ -147,6 +152,22 @@ export class CodexLiveEventReducer { }); } + /** Preserve partial text and narrative when the runtime must finish without a provider terminal event. */ + finishFromState(input: SyntheticTerminalInput): CodexLiveReduction { + const event: AgentEvent = input.outcome === "errored" + ? { type: "error", threadId: this.execution.threadId, turnExecutionId: this.execution.executionId, error: input.error } + : { type: "ended", threadId: this.execution.threadId, turnExecutionId: this.execution.executionId, outcome: "interrupted" }; + if (this.phase === "completed" || this.phase === "ended") return this.unsupported(event, "execution already terminal"); + this.phase = "completed"; + const writer: CodexLiveWriterIntent[] = [{ kind: "terminal-projection", + source: input.outcome === "errored" ? "error" : "ended", outcome: input.outcome, + assistant: this.assistant.materializationInput(null), narrative: this.narrative.recoverySnapshot(this.execution.threadId) }]; + if (input.outcome === "errored") writer.push({ kind: "turn-error", error: input.error }); + else writer.push({ kind: "turn-ended" }); + return structuredClone({ kind: "reduced", execution: this.execution, writer, + publication: { event, after: "terminal" } }); + } + private identityRejection(event: AgentEvent): string | undefined { if (event.threadId !== this.execution.threadId) return "different thread"; if (!event.turnExecutionId) return "event without execution identity needs the provider event routing owner"; diff --git a/apps/server/src/features/agents/execution/execution-file-evidence-coordinator.ts b/apps/server/src/features/agents/execution/execution-file-evidence-coordinator.ts new file mode 100644 index 000000000..7ac0a5afe --- /dev/null +++ b/apps/server/src/features/agents/execution/execution-file-evidence-coordinator.ts @@ -0,0 +1,152 @@ +import type { AgentEvent, ProviderFileMutationStart, TurnOutcome } from "@mcode/contracts"; + +import type { SnapshotService } from "../../projects/diffs/snapshots/snapshot-service.js"; +import type { PreparedTurnDiffEvidence, TurnDiffService } from "../turns/turn-diff-service.js"; +import { + prepareExecutionFileEvidence, + type PreparedExecutionFileEvidence, +} from "../turns/turn-execution-file-evidence.js"; +import type { + CapturedToolUseObservation, + FileTurnHandoff, + TurnFileTracker, +} from "../turns/turn-file-tracker.js"; +import { ExecutionFileObservationHandoff } from "./execution-file-observation-handoff.js"; + +type ToolUse = Extract; +type DiffHandoff = Pick; + +/** Cloneable terminal file data retained until the writer acknowledges finalization. */ +export interface FrozenExecutionFileEvidence { + readonly handoff: FileTurnHandoff; + readonly turnId: string; + readonly deliveryAttempt: number; + readonly outcome: TurnOutcome; + readonly nativeDiff: PreparedTurnDiffEvidence | null; +} + +interface ActiveFileExecution { + readonly handoff: FileTurnHandoff; + readonly turnId: string; + readonly deliveryAttempt: number; + readonly observations: ExecutionFileObservationHandoff; + frozen?: FrozenExecutionFileEvidence; +} + +/** Captures pre-edit evidence on the provider callback and seals one attempt for worker settlement. */ +export class ExecutionFileEvidenceCoordinator { + private readonly active = new Map(); + + constructor( + private readonly captureTracker: TurnFileTracker, + private readonly diffs: DiffHandoff, + ) {} + + /** Open the exact execution only after its durable start has been acknowledged. */ + begin(input: { + readonly threadId: string; + readonly turnId: string; + readonly executionId: string; + readonly deliveryAttempt: number; + readonly cwd: string; + readonly baselineRef: string | null; + }): FileTurnHandoff { + if (!input.turnId || !Number.isSafeInteger(input.deliveryAttempt) || input.deliveryAttempt < 1) { + throw new Error("File evidence requires a turn ID and positive delivery attempt"); + } + const previous = this.active.get(input.threadId); + if (previous?.frozen) throw new Error("Retire sealed file evidence before opening another attempt"); + const handoff = this.captureTracker.beginExecutionTurn(input); + const observations = new ExecutionFileObservationHandoff( + this.captureTracker, handoff, input.deliveryAttempt, + ); + previous?.observations.close(); + if (previous) this.captureTracker.clearTurn(input.threadId, previous.handoff.generation); + this.diffs.begin({ threadId: input.threadId, turnId: input.turnId, + turnExecutionId: input.executionId, deliveryAttempt: input.deliveryAttempt }); + this.active.set(input.threadId, { + handoff, turnId: input.turnId, deliveryAttempt: input.deliveryAttempt, observations, + }); + return handoff; + } + + /** Run synchronously before the provider is allowed to mutate a named file. */ + capture(event: ProviderFileMutationStart): boolean { + return this.active.get(event.threadId)?.observations.capture(event) ?? false; + } + + /** Transfer the original pre-edit observation with its matching admitted tool event. */ + take(event: ToolUse, deliveryAttempt: number): CapturedToolUseObservation | null { + return this.active.get(event.threadId)?.observations.take(event, deliveryAttempt) ?? null; + } + + /** Reject later callbacks at the Stop or retry cut while retaining settled evidence. */ + fence(threadId: string, executionId: string, deliveryAttempt: number): boolean { + const active = this.match(threadId, executionId, deliveryAttempt); + if (!active) return false; + active.observations.close(); + return true; + } + + /** Freeze native evidence before asynchronous file settlement or terminal publication. */ + seal(input: { + readonly threadId: string; + readonly turnId: string; + readonly executionId: string; + readonly deliveryAttempt: number; + readonly outcome: TurnOutcome; + }): FrozenExecutionFileEvidence | null { + const active = this.match(input.threadId, input.executionId, input.deliveryAttempt); + if (!active || active.turnId !== input.turnId) return null; + if (active.frozen) { + if (active.frozen.outcome !== input.outcome) throw new Error("File evidence outcome changed after sealing"); + return active.frozen; + } + active.observations.close(); + const nativeDiff = this.diffs.takeFinalizationEvidence(input.threadId, input.executionId); + if (nativeDiff && (nativeDiff.turnId !== input.turnId + || nativeDiff.turnExecutionId !== input.executionId + || nativeDiff.deliveryAttempt !== input.deliveryAttempt)) { + throw new Error("Native diff does not belong to the sealed execution attempt"); + } + active.frozen = Object.freeze({ + handoff: active.handoff, turnId: input.turnId, + deliveryAttempt: input.deliveryAttempt, outcome: input.outcome, nativeDiff, + }); + return active.frozen; + } + + /** Release the capture tracker only after the terminal writer receipt or explicit abandonment. */ + retire(threadId: string, executionId: string, deliveryAttempt: number): boolean { + const active = this.match(threadId, executionId, deliveryAttempt); + if (!active) return false; + active.observations.close(); + this.active.delete(threadId); + this.diffs.clearExecution(threadId, executionId); + this.captureTracker.clearTurn(threadId, active.handoff.generation); + return true; + } + + private match(threadId: string, executionId: string, deliveryAttempt: number): ActiveFileExecution | null { + const active = this.active.get(threadId); + return active?.handoff.executionId === executionId && active.deliveryAttempt === deliveryAttempt + ? active : null; + } +} + +/** Settle a frozen attempt on its execution worker before the terminal writer operation. */ +export function prepareFrozenExecutionFileEvidence( + frozen: FrozenExecutionFileEvidence, + tracker: TurnFileTracker, + snapshots: SnapshotService, +): Promise { + return prepareExecutionFileEvidence({ + handoff: frozen.handoff, + turnId: frozen.turnId, + deliveryAttempt: frozen.deliveryAttempt, + outcome: frozen.outcome, + nativeDiff: frozen.nativeDiff, + tracker, + snapshots, + }); +} diff --git a/apps/server/src/features/agents/execution/execution-mailbox-owner.ts b/apps/server/src/features/agents/execution/execution-mailbox-owner.ts index 6c4f456e0..c10872910 100644 --- a/apps/server/src/features/agents/execution/execution-mailbox-owner.ts +++ b/apps/server/src/features/agents/execution/execution-mailbox-owner.ts @@ -1,6 +1,7 @@ import type { DataOnlyParentTurnStartInput } from "../canonical/canonical-parent-turn-write.js"; +import type { ExecutionParentStartContext } from "./provider-execution-event-state.js"; import type { ExecutionIdentity, ExecutionLease, ExecutionMailboxCompletion } from "./execution-mailbox-protocol.js"; -import type { ExecutionMailboxScheduler } from "./execution-mailbox-scheduler.js"; +import type { ExecutionMailboxScheduler, ExecutionRecoveryReceipt } from "./execution-mailbox-scheduler.js"; import type { ExecutionWorkCommand, ExecutionWorkerResult } from "./execution-worker-handler.js"; type Scheduler = ExecutionMailboxScheduler; @@ -23,6 +24,8 @@ export class ExecutionMailboxOwner { readonly ownerEpoch: number; readonly providerId: string; readonly parentTurn: DataOnlyParentTurnStartInput; + readonly parentLive?: ExecutionParentStartContext; + readonly publishParentStart?: boolean; }): Promise { if (this.active.has(input.execution.threadId)) throw new Error("Thread already has an execution owner"); const claim = this.scheduler.claim(input.execution, input.ownerEpoch); @@ -30,7 +33,11 @@ export class ExecutionMailboxOwner { const owner = { execution: input.execution, lease: claim.lease, started: false }; this.active.set(input.execution.threadId, owner); try { - const result = await this.submitOwned(owner, { kind: "start", providerId: input.providerId, input: input.parentTurn }); + const result = await this.submitOwned(owner, { + kind: "start", providerId: input.providerId, input: input.parentTurn, + ...(input.parentLive ? { parentLive: input.parentLive } : {}), + ...(input.publishParentStart ? { publishParentStart: true } : {}), + }); if (result.kind !== "committed") throw new Error(`Execution start was not committed: ${input.execution.executionId}`); owner.started = true; } catch (error) { @@ -61,6 +68,20 @@ export class ExecutionMailboxOwner { this.active.delete(execution.threadId); } + /** Release one rejected execution only after the writer has durably settled that exact lease. */ + async releaseRecovered(execution: ExecutionIdentity, recovery: ExecutionRecoveryReceipt): Promise { + const owner = this.requireOwner(execution); + if (recovery.operationId !== `${owner.lease.leaseId}:worker-lost` + || recovery.kind === "conflict" && recovery.recoveryState !== "already-terminal") { + throw new Error("Recovered release has no matching durable evidence"); + } + const result = await this.submitOwned(owner, { kind: "release", recovery }); + if (result.kind !== "released" || !this.scheduler.release(owner.execution, owner.lease)) { + throw new Error(`Recovered execution release was not acknowledged: ${execution.executionId}`); + } + this.active.delete(execution.threadId); + } + /** Forget a worker-lost execution only after the recovery coordinator released its lease. */ forgetRecovered(execution: ExecutionIdentity, lease: ExecutionLease): void { const owner = this.active.get(execution.threadId); diff --git a/apps/server/src/features/agents/execution/execution-mailbox-scheduler.ts b/apps/server/src/features/agents/execution/execution-mailbox-scheduler.ts index c896ae584..a6dcb95f4 100644 --- a/apps/server/src/features/agents/execution/execution-mailbox-scheduler.ts +++ b/apps/server/src/features/agents/execution/execution-mailbox-scheduler.ts @@ -17,6 +17,7 @@ export type ExecutionMailboxCommand = /** Lifecycle results that must be allowed to settle after Stop is admitted. */ export const EXECUTION_CONTROL_KINDS = [ "checkpoint", "effect-result", "provider-outcome", "stage-terminal", "finalize", "release", + "finish-from-state", "finish-live-event", "post-terminal-event", ] as const; export type ExecutionControlKind = typeof EXECUTION_CONTROL_KINDS[number]; const CONTROL_KINDS: ReadonlySet = new Set(EXECUTION_CONTROL_KINDS); diff --git a/apps/server/src/features/agents/execution/execution-provider-event-ownership.ts b/apps/server/src/features/agents/execution/execution-provider-event-ownership.ts index 0c43422d6..c421034b5 100644 --- a/apps/server/src/features/agents/execution/execution-provider-event-ownership.ts +++ b/apps/server/src/features/agents/execution/execution-provider-event-ownership.ts @@ -1,6 +1,7 @@ import type { ExecutionIdentity } from "./execution-mailbox-protocol.js"; import type { ExecutionMailboxOwner } from "./execution-mailbox-owner.js"; import type { ProviderEventCommitReceipt } from "@mcode/providers"; +import type { ExecutionWorkCommand, ExecutionWorkerResult } from "./execution-worker-handler.js"; import type { ProviderEventOwnership, ProviderEventOwnershipRoute, WorkerOwnedProviderEventBatch, WorkerOwnedProviderEventResult } from "../../providers/composition/provider-host-ports.js"; interface ActiveRoute { @@ -25,9 +26,27 @@ type Route = ActiveRoute | SwitchingRoute; */ export class ExecutionProviderEventOwnership implements ProviderEventOwnership { private readonly routes = new Map(); + private readonly workerProviders = new Set(["codex"]); + private onCommitted: ((execution: ExecutionIdentity, result: Extract) => Promise) | undefined; + private prepareCommand: ((execution: ExecutionIdentity, batch: WorkerOwnedProviderEventBatch) => Promise>) | undefined; constructor(private readonly owner: Pick) {} + /** Run execution lifecycle effects after the writer acknowledgement and before the provider receives its batch receipt. */ + bindCommitted(onCommitted: (execution: ExecutionIdentity, result: Extract) => Promise): void { + this.onCommitted = onCommitted; + } + + /** Prepare exact file and terminal inputs before admitting the provider batch to its mailbox. */ + bindCommandPreparation(prepare: (execution: ExecutionIdentity, batch: WorkerOwnedProviderEventBatch) => Promise>): void { + this.prepareCommand = prepare; + } + + /** Activate another provider only after its worker live effects are installed. */ + enableProvider(providerId: string): void { + this.workerProviders.add(providerId); + } + /** Admit one provider attempt only while the exact mailbox still owns the thread. */ async bind(execution: ExecutionIdentity, deliveryAttempt: number): Promise { this.requireOwner(execution, deliveryAttempt); @@ -80,10 +99,10 @@ export class ExecutionProviderEventOwnership implements ProviderEventOwnership { } /** Select the exact mailbox, then check again before asynchronous admission. */ - resolve(executionId: string): ProviderEventOwnershipRoute { + resolve(executionId: string, sourceProviderId?: string): ProviderEventOwnershipRoute { const route = this.routes.get(executionId); if (route?.kind !== "active" || !this.owner.isStarted(route.execution)) { - return { kind: "rejected" }; + return { kind: sourceProviderId && !this.workerProviders.has(sourceProviderId) ? "legacy" : "rejected" }; } return { kind: "worker", @@ -97,22 +116,21 @@ export class ExecutionProviderEventOwnership implements ProviderEventOwnership { if (!this.isCurrent(route, batch)) { throw new Error("Provider batch belongs to a retired execution attempt"); } - const pending = this.owner.submit(route.execution, { - kind: "event", - phase: batch.phase, - nativeCursor: batch.nativeCursor ?? null, - events: batch.events, - }); + const pending = this.commitAndNotify(route, batch); route.pending.add(pending); - let result: Awaited; try { - result = await pending; + return await pending; } finally { route.pending.delete(pending); } + } + + private async commitAndNotify(route: ActiveRoute, batch: WorkerOwnedProviderEventBatch): Promise { + const result = await this.submitPrepared(route, batch); if (result.kind !== "committed" || !result.providerCommit) { throw new Error("Execution worker did not commit provider batch"); } + await this.notifyCommitted(route.execution, result); const commit: ProviderEventCommitReceipt = { ...result.providerCommit, outcome: result.providerCommit.outcome === "terminal-outcome-confirmed" @@ -129,12 +147,30 @@ export class ExecutionProviderEventOwnership implements ProviderEventOwnership { }; } + private async submitPrepared(route: ActiveRoute, batch: WorkerOwnedProviderEventBatch): Promise { + const command = this.prepareCommand + ? await this.prepareCommand(route.execution, batch) + : { kind: "event" as const, phase: batch.phase, nativeCursor: batch.nativeCursor ?? null, events: batch.events }; + if (!this.isCurrent(route, batch) || command.kind !== "event" + || command.events !== batch.events || command.phase !== batch.phase) { + throw new Error("Prepared provider event lost its execution owner"); + } + return await this.owner.submit(route.execution, command); + } + private isCurrent(route: ActiveRoute, batch: WorkerOwnedProviderEventBatch): boolean { return this.routes.get(route.execution.executionId) === route && this.owner.isStarted(route.execution) && batch.deliveryAttempt === route.deliveryAttempt && sameExecution(batch, route.execution); } + + private async notifyCommitted( + execution: ExecutionIdentity, + result: Extract, + ): Promise { + await this.onCommitted?.(execution, result); + } } function sameExecution(left: ExecutionIdentity | undefined, right: ExecutionIdentity): boolean { diff --git a/apps/server/src/features/agents/execution/execution-worker-file-evidence.ts b/apps/server/src/features/agents/execution/execution-worker-file-evidence.ts new file mode 100644 index 000000000..17644b4a8 --- /dev/null +++ b/apps/server/src/features/agents/execution/execution-worker-file-evidence.ts @@ -0,0 +1,156 @@ +import "reflect-metadata"; +import type { AgentEvent, TurnFileEffectSummary } from "@mcode/contracts"; +import { hostRuntime } from "@mcode/shared/node/host-runtime"; + +import { RealGitExecutor } from "../../projects/git/execution/real-git-executor.js"; +import { SnapshotService } from "../../projects/diffs/snapshots/snapshot-service.js"; +import { TurnFileTracker, type CapturedToolUseObservation, type FileTurnHandoff } from "../turns/turn-file-tracker.js"; +import type { PreparedExecutionFileEvidence } from "../turns/turn-execution-file-evidence.js"; +import type { ExecutionIdentity } from "./execution-mailbox-protocol.js"; +import { + prepareFrozenExecutionFileEvidence, + type FrozenExecutionFileEvidence, +} from "./execution-file-evidence-coordinator.js"; + +type ToolUse = Extract; +type ToolResult = Extract; +const MAX_OPEN_TOOL_CALLS = 64; + +interface ActiveFileExecution { + readonly execution: ExecutionIdentity; + readonly deliveryAttempt: number; + readonly handoff: FileTurnHandoff; + readonly openToolCalls: Set; + sealed: boolean; + settlement?: Promise; +} + +/** Runs file attribution and terminal settlement on the execution worker, with no database access. */ +export class ExecutionWorkerFileEvidence { + private readonly active = new Map(); + private readonly tracker: TurnFileTracker; + + constructor(private readonly snapshots = new SnapshotService(new RealGitExecutor())) { + this.tracker = new TurnFileTracker( + (cwd, ref, path) => snapshots.getFileAtRef(cwd, ref, path), + () => {}, + hostRuntime.platform, + ); + } + + /** Install the main-loop handoff only against the scheduler's independent cwd and execution. */ + begin(input: { + readonly execution: ExecutionIdentity; + readonly deliveryAttempt: number; + readonly cwd: string; + readonly handoff: FileTurnHandoff; + }): boolean { + if (!Number.isSafeInteger(input.deliveryAttempt) || input.deliveryAttempt < 1) return false; + const { execution, handoff } = input; + const current = this.active.get(execution.threadId); + if (current) return this.matches(current, execution, input.deliveryAttempt) + && current.handoff.generationToken === handoff.generationToken; + if (!this.tracker.beginTurnFromHandoff(handoff, { + threadId: execution.threadId, executionId: execution.executionId, cwd: input.cwd, + })) return false; + this.active.set(execution.threadId, { + execution, deliveryAttempt: input.deliveryAttempt, handoff, + openToolCalls: new Set(), sealed: false, + }); + return true; + } + + /** Attach the captured pre-edit state before processing the matching tool event. */ + async observeToolUse(input: { + readonly execution: ExecutionIdentity; + readonly deliveryAttempt: number; + readonly event: ToolUse; + readonly captured: CapturedToolUseObservation | null; + }): Promise { + const active = this.activeFor(input.execution, input.deliveryAttempt); + if (!active || active.sealed || !input.captured || !this.matchesEvent(active, input.event) + || active.openToolCalls.has(input.event.toolCallId) + || active.openToolCalls.size >= MAX_OPEN_TOOL_CALLS) return false; + const observed = await this.tracker.observeCapturedToolUse(input.event, input.captured); + if (observed) active.openToolCalls.add(input.event.toolCallId); + return observed; + } + + /** Recompute the tracked net effect after a previously captured tool finishes. */ + async observeToolResult(input: { + readonly execution: ExecutionIdentity; + readonly deliveryAttempt: number; + readonly event: ToolResult; + }): Promise { + const active = this.activeFor(input.execution, input.deliveryAttempt); + if (!active || active.sealed || !this.matchesEvent(active, input.event) + || !active.openToolCalls.has(input.event.toolCallId)) return null; + await this.tracker.observeToolResult(active.execution.threadId, input.event.toolCallId); + active.openToolCalls.delete(input.event.toolCallId); + return this.tracker.finalizeTurn(active.execution.threadId, active.handoff.generation); + } + + /** Apply a baseline captured after start to this attempt's tracker generation only. */ + async setBaselineRef(execution: ExecutionIdentity, deliveryAttempt: number, ref: string): Promise { + const active = this.activeFor(execution, deliveryAttempt); + if (!active || active.sealed) return false; + await this.tracker.setBaselineRef(execution.threadId, active.handoff.generation, ref); + return true; + } + + /** Freeze exact attempt evidence before the one terminal writer operation. */ + settle(frozen: FrozenExecutionFileEvidence): Promise { + const active = this.active.get(frozen.handoff.threadId); + if (!active || !this.matches(active, { + threadId: frozen.handoff.threadId, turnId: frozen.turnId, + executionId: frozen.handoff.executionId, + }, frozen.deliveryAttempt) + || active.handoff.generation !== frozen.handoff.generation + || active.handoff.generationToken !== frozen.handoff.generationToken) return Promise.resolve(null); + if (active.settlement) return active.settlement; + active.sealed = true; + const settlement = this.settleActive(active, frozen); + active.settlement = settlement; + void settlement.catch(() => { + if (active.settlement === settlement) active.settlement = undefined; + }); + return settlement; + } + + /** Clear a settled generation only after its final writer receipt. */ + retire(execution: ExecutionIdentity, deliveryAttempt: number): boolean { + const active = this.activeFor(execution, deliveryAttempt); + if (!active) return false; + this.active.delete(execution.threadId); + this.tracker.clearTurn(execution.threadId, active.handoff.generation); + return true; + } + + private activeFor(execution: ExecutionIdentity, deliveryAttempt: number): ActiveFileExecution | null { + const active = this.active.get(execution.threadId); + return active && this.matches(active, execution, deliveryAttempt) ? active : null; + } + + private matches(active: ActiveFileExecution, execution: ExecutionIdentity, deliveryAttempt: number): boolean { + return active.execution.threadId === execution.threadId + && active.execution.turnId === execution.turnId + && active.execution.executionId === execution.executionId + && active.deliveryAttempt === deliveryAttempt; + } + + private matchesEvent(active: ActiveFileExecution, event: ToolUse | ToolResult): boolean { + return event.threadId === active.execution.threadId + && event.turnExecutionId === active.execution.executionId; + } + + private async settleActive( + active: ActiveFileExecution, + frozen: FrozenExecutionFileEvidence, + ): Promise { + for (const toolCallId of active.openToolCalls) { + await this.tracker.observeToolResult(active.execution.threadId, toolCallId); + } + active.openToolCalls.clear(); + return prepareFrozenExecutionFileEvidence(frozen, this.tracker, this.snapshots); + } +} diff --git a/apps/server/src/features/agents/execution/execution-worker-handler.ts b/apps/server/src/features/agents/execution/execution-worker-handler.ts index d113b42a5..bc1c51889 100644 --- a/apps/server/src/features/agents/execution/execution-worker-handler.ts +++ b/apps/server/src/features/agents/execution/execution-worker-handler.ts @@ -1,4 +1,4 @@ -import type { AgentEvent, PlanQuestion, PlanRecord, TurnOutcome } from "@mcode/contracts"; +import type { AgentEvent, ParentNarrativeRecoveryItem, PlanQuestion, PlanRecord, TurnFileEffectSummary, TurnOutcome } from "@mcode/contracts"; import type { ProviderEventDraft } from "@mcode/providers"; import type { @@ -23,13 +23,19 @@ import type { ExecutionWorkerReply, ExecutionWorkerRequest, } from "./execution-mailbox-protocol.js"; -import type { ExecutionMailboxCommand } from "./execution-mailbox-scheduler.js"; +import type { ExecutionMailboxCommand, ExecutionRecoveryReceipt } from "./execution-mailbox-scheduler.js"; +import { ProviderExecutionEventState, type ExecutionParentStartContext, type PreparedProviderLiveEvent } from "./provider-execution-event-state.js"; +import { ExecutionWorkerFileEvidence } from "./execution-worker-file-evidence.js"; +import type { FrozenExecutionFileEvidence } from "./execution-file-evidence-coordinator.js"; +import type { CapturedToolUseObservation, FileTurnHandoff } from "../turns/turn-file-tracker.js"; +import type { SyntheticTerminalInput } from "./codex-live-event-reducer.js"; /** Data that an execution worker may receive without a server dependency container. */ export type ExecutionWorkCommand = - | { readonly kind: "start"; readonly providerId: string; readonly input: DataOnlyParentTurnStartInput; readonly livePublication?: readonly ExecutionLivePublicationIntent[] } + | { readonly kind: "start"; readonly providerId: string; readonly input: DataOnlyParentTurnStartInput; readonly parentLive?: ExecutionParentStartContext; readonly publishParentStart?: boolean; readonly livePublication?: readonly ExecutionLivePublicationIntent[] } | { readonly kind: "resume"; readonly providerId: string; readonly checkpointId: string } - | { readonly kind: "event"; readonly phase: string; readonly nativeCursor: unknown | null; readonly events: readonly ProviderEventDraft[]; readonly livePublication?: readonly ExecutionLivePublicationIntent[] } + | { readonly kind: "begin-files"; readonly cwd: string; readonly handoff: FileTurnHandoff; readonly deliveryAttempt: number } + | { readonly kind: "event"; readonly phase: string; readonly nativeCursor: unknown | null; readonly events: readonly ProviderEventDraft[]; readonly parentLive?: ParentLiveEffects; readonly terminalInput?: DataOnlyParentTurnFinishInput; readonly deliveryAttempt?: number; readonly capturedFileObservation?: CapturedToolUseObservation | null; readonly frozenFileEvidence?: FrozenExecutionFileEvidence; readonly livePublication?: readonly ExecutionLivePublicationIntent[] } | { readonly kind: "assistant-text"; readonly inputs: readonly ParentAssistantTextCheckpointInput[] } | { readonly kind: "narrative-delta"; readonly input: ParentNarrativeRecoveryCommit } | { @@ -52,7 +58,27 @@ export type ExecutionWorkCommand = | { readonly kind: "provider-outcome"; readonly outcome: TurnOutcome } | { readonly kind: "stage-terminal"; readonly input: DataOnlyParentTerminalProjectionInput } | { readonly kind: "finalize"; readonly outcome: TurnOutcome; readonly input: DataOnlyParentTurnFinishInput; readonly livePublication?: readonly ExecutionLivePublicationIntent[] } - | { readonly kind: "release" }; + | { readonly kind: "finish-live-event"; readonly outcome: TurnOutcome; readonly projection: DataOnlyParentTerminalProjectionInput; readonly input: DataOnlyParentTurnFinishInput; readonly providerEvent?: TerminalProviderEventInput; readonly frozenFileEvidence?: FrozenExecutionFileEvidence; readonly livePublication?: readonly ExecutionLivePublicationIntent[] } + | { readonly kind: "finish-from-state"; readonly outcome: SyntheticTerminalInput["outcome"]; readonly input: DataOnlyParentTurnFinishInput; readonly frozenFileEvidence?: FrozenExecutionFileEvidence } + | ({ readonly kind: "post-terminal-event"; readonly publication: ExecutionLivePublicationIntent } & PostTerminalEffects) + | { readonly kind: "release"; readonly recovery?: ExecutionRecoveryReceipt }; + +/** Parent effects prepared from the same provider event as its canonical draft. */ +export type ParentLiveEffects = Omit, "kind" | "publication">; + +/** The only effects admitted after an execution's terminal projection has committed. */ +export interface PostTerminalEffects { + readonly hooks?: readonly Extract[]; + readonly systemIntents?: readonly CodexSystemWriterIntent[]; + readonly providerEvent?: TerminalProviderEventInput; +} + +/** Provider evidence committed with a terminal projection; synthetic Stop completion omits it. */ +export interface TerminalProviderEventInput { + readonly phase: string; + readonly nativeCursor: unknown | null; + readonly events: readonly ProviderEventDraft[]; +} /** One complete semantic mutation. The single writer decides its SQL transaction. */ export interface ExecutionSemanticOperation { @@ -65,7 +91,7 @@ export interface ExecutionSemanticOperation { readonly mutation: | { readonly kind: "begin"; readonly providerId: string; readonly input: DataOnlyParentTurnStartInput } | { readonly kind: "resume"; readonly providerId: string; readonly checkpointId: string } - | { readonly kind: "append-events"; readonly phase: string; readonly nativeCursor: unknown | null; readonly events: readonly ProviderEventDraft[] } + | { readonly kind: "append-events"; readonly phase: string; readonly nativeCursor: unknown | null; readonly events: readonly ProviderEventDraft[]; readonly parentLive?: ParentLiveEffects } | { readonly kind: "append-assistant-text"; readonly inputs: readonly ParentAssistantTextCheckpointInput[] } | { readonly kind: "narrative-delta"; readonly input: ParentNarrativeRecoveryCommit } | { @@ -84,7 +110,9 @@ export interface ExecutionSemanticOperation { | { readonly kind: "provider-outcome"; readonly outcome: TurnOutcome } | { readonly kind: "stage-terminal"; readonly input: DataOnlyParentTerminalProjectionInput } | { readonly kind: "worker-lost"; readonly reason: string; readonly recoveryIncidentId: string } - | { readonly kind: "finish"; readonly outcome: TurnOutcome; readonly input: DataOnlyParentTurnFinishInput }; + | { readonly kind: "finish"; readonly outcome: TurnOutcome; readonly input: DataOnlyParentTurnFinishInput } + | { readonly kind: "finish-live-event"; readonly outcome: TurnOutcome; readonly projection: DataOnlyParentTerminalProjectionInput; readonly input: DataOnlyParentTurnFinishInput; readonly providerEvent?: TerminalProviderEventInput } + | ({ readonly kind: "post-terminal-event" } & PostTerminalEffects); } /** A live event has one durability barrier and stays inert until its semantic effects commit. */ @@ -120,9 +148,18 @@ export type ProjectedCommittedProviderEvent = Omit< readonly canonicalReceipt: NonNullable; }; +/** Writer-confirmed terminal projection surfaced after its durable finish. */ +export interface ExecutionTerminalPersistenceReceipt { + readonly messageId: string | null; + readonly outcome: TurnOutcome; + readonly toolCallCount: number; + readonly filesChanged: readonly string[]; + readonly fileEffects?: TurnFileEffectSummary; +} + /** A writer reply is valid only after the semantic operation commits durably. */ export type ExecutionWriteReceipt = - | { readonly kind: "committed"; readonly operationId: string; readonly durableRevision: number; readonly providerCommit?: ExecutionProviderCommitReceipt; readonly providerEvents?: readonly ProjectedCommittedProviderEvent[]; readonly assistantTextCheckpoint?: ParentAssistantTextCheckpointResult; readonly livePublication?: readonly ExecutionLivePublicationReceipt[]; readonly planQuestions?: ExecutionPlanQuestionsReceipt; readonly planOutput?: PlanRecord } + | { readonly kind: "committed"; readonly operationId: string; readonly durableRevision: number; readonly providerCommit?: ExecutionProviderCommitReceipt; readonly providerEvents?: readonly ProjectedCommittedProviderEvent[]; readonly assistantTextCheckpoint?: ParentAssistantTextCheckpointResult; readonly livePublication?: readonly ExecutionLivePublicationReceipt[]; readonly planQuestions?: ExecutionPlanQuestionsReceipt; readonly planOutput?: PlanRecord; readonly terminalPersistence?: ExecutionTerminalPersistenceReceipt } | { readonly kind: "conflict"; readonly operationId: string; readonly recoveryState?: "not-started" | "already-terminal" }; /** @@ -134,11 +171,14 @@ export interface ExecutionSemanticWriter { transact(operation: ExecutionSemanticOperation): Promise; } +/** Execution runtime effects returned only after their associated parent write commits. */ +export type ExecutionParentEventResult = Pick; + /** A command result that never calls an uncommitted mutation successful. */ export type ExecutionWorkerResult = - | { readonly kind: "committed"; readonly operationId: string; readonly durableRevision: number; readonly providerCommit?: ExecutionProviderCommitReceipt; readonly providerEvents?: readonly ProjectedCommittedProviderEvent[]; readonly assistantTextCheckpoint?: ParentAssistantTextCheckpointResult; readonly livePublication?: readonly ExecutionLivePublicationReceipt[]; readonly planQuestions?: ExecutionPlanQuestionsReceipt; readonly planOutput?: PlanRecord } + | { readonly kind: "committed"; readonly operationId: string; readonly durableRevision: number; readonly providerCommit?: ExecutionProviderCommitReceipt; readonly providerEvents?: readonly ProjectedCommittedProviderEvent[]; readonly assistantTextCheckpoint?: ParentAssistantTextCheckpointResult; readonly livePublication?: readonly ExecutionLivePublicationReceipt[]; readonly planQuestions?: ExecutionPlanQuestionsReceipt; readonly planOutput?: PlanRecord; readonly terminalPersistence?: ExecutionTerminalPersistenceReceipt; readonly parentEvent?: ExecutionParentEventResult } | { readonly kind: "released" } - | { readonly kind: "rejected"; readonly reason: "no-execution" | "stale-execution" | "out-of-order" | "invalid-transition" | "invalid-event-routing" | "invalid-text-routing" | "invalid-narrative-routing" | "invalid-stop-watermark" | "writer-conflict" }; + | { readonly kind: "rejected"; readonly reason: "no-execution" | "stale-execution" | "out-of-order" | "invalid-transition" | "invalid-event-routing" | "invalid-text-routing" | "invalid-narrative-routing" | "invalid-stop-watermark" | "writer-conflict" | "writer-failure" }; type WorkerCommand = ExecutionMailboxCommand; const METADATA_MUTATIONS: ReadonlySet = new Set([ @@ -149,9 +189,16 @@ interface ExecutionState { readonly execution: ExecutionIdentity; readonly lease: ExecutionLease; readonly providerId: string; + readonly parentEvents?: ProviderExecutionEventState; nextOrdinal: number; - phase: "running" | "stopping" | "finalized"; + phase: "running" | "stopping" | "finalized" | "poisoned"; durableRevision: number; + fileAttempt?: number; +} + +interface PreparedExecutionRequest { + readonly request: ExecutionWorkerRequest; + readonly parentEvent?: ExecutionParentEventResult; } /** @@ -162,7 +209,7 @@ interface ExecutionState { export class ExecutionWorkerHandler { private readonly states = new Map(); - constructor(private readonly writer: ExecutionSemanticWriter) {} + constructor(private readonly writer: ExecutionSemanticWriter, private readonly files = new ExecutionWorkerFileEvidence()) {} /** Process one mailbox command and echo its exact execution and lease. */ async handle(request: ExecutionWorkerRequest): Promise> { @@ -192,35 +239,184 @@ export class ExecutionWorkerHandler { const rejection = commandRejection(request, state); if (rejection) return { kind: "rejected", reason: rejection }; if (request.command.kind === "release") return this.release(request, state); - if (state.phase === "finalized") return { kind: "rejected", reason: "invalid-transition" }; - const mutation = mutationFor(request, state); - if (!mutation) return { kind: "rejected", reason: invalidReason(request) }; - const receipt = await this.writer.transact(operationFor(request, mutation)); + if (state.phase === "poisoned" || state.phase === "finalized" + && request.command.kind !== "event" && request.command.kind !== "post-terminal-event") { + return { kind: "rejected", reason: "invalid-transition" }; + } + const prepared = await this.prepareOwnedRequest(request, state); + if (!prepared) return { kind: "rejected", reason: "invalid-event-routing" }; + const mutation = mutationFor(prepared.request, state); + if (!mutation) { + if (prepared.parentEvent) state.phase = "poisoned"; + return { kind: "rejected", reason: invalidReason(request) }; + } + return await this.commitRequest(prepared, state, mutation); + } + + private async prepareOwnedRequest(request: ExecutionWorkerRequest, state: ExecutionState): Promise { + try { + const prepared = this.prepareRequest(request, state); + if (!prepared) return undefined; + const withFiles = await this.prepareFileEffects(prepared, state); + if (!withFiles && prepared.parentEvent) state.phase = "poisoned"; + return withFiles; + } catch (error) { + if (state.parentEvents) state.phase = "poisoned"; + throw error; + } + } + + private async commitRequest( + prepared: PreparedExecutionRequest, + state: ExecutionState, + mutation: ExecutionSemanticOperation["mutation"], + ): Promise { + const { request } = prepared; + let receipt: ExecutionWriteReceipt; + try { + receipt = await this.writer.transact(operationFor(request, mutation)); + } catch (error) { + if (state.parentEvents) state.phase = "poisoned"; + throw error; + } if (!isDurableReceipt(receipt, request, state.durableRevision)) { + if (state.parentEvents) state.phase = "poisoned"; return { kind: "rejected", reason: "writer-conflict" }; } state.nextOrdinal += 1; state.durableRevision = receipt.durableRevision; if (request.command.kind === "stop") state.phase = "stopping"; - if (request.command.kind === "finalize") state.phase = "finalized"; - return committedResult(receipt); + if (request.command.kind === "finalize" || request.command.kind === "finish-live-event") { + state.phase = "finalized"; + if (state.fileAttempt !== undefined) this.files.retire(state.execution, state.fileAttempt); + } + const result = committedResult(receipt); + return prepared.parentEvent ? { ...result, parentEvent: prepared.parentEvent } : result; + } + + private prepareRequest(request: ExecutionWorkerRequest, state: ExecutionState): PreparedExecutionRequest | undefined { + const command = request.command; + if (command.kind === "finish-from-state") return this.prepareSyntheticFinish(request, command, state); + if (!state.parentEvents || command.kind !== "event") return { request }; + if (state.phase === "finalized") return this.preparePostTerminalEvent(request, command, state); + if (!validOwnedEventCommand(command, state)) return undefined; + const reduction = state.parentEvents.prepare(command.events, command.terminalInput !== undefined); + if (reduction.kind === "rejected") return undefined; + if (reduction.kind === "writer-owned") return { request }; + const { effects, ...parentEvent } = reduction.prepared; + if (parentEvent.publication.after === "terminal") { + const terminal = prepareTerminalEvent(command, parentEvent, state.providerId); + if (!terminal) { state.phase = "poisoned"; return undefined; } + return { request: { ...request, command: terminal }, parentEvent }; + } + return { request: { ...request, command: { ...command, + parentLive: effects, livePublication: [parentEvent.publication] } }, parentEvent }; + } + + + private prepareSyntheticFinish( + request: ExecutionWorkerRequest, + command: Extract, + state: ExecutionState, + ): PreparedExecutionRequest | undefined { + const terminalInput = syntheticTerminalInput(command, state); + if (!terminalInput || !state.parentEvents) return undefined; + const reduction = state.parentEvents.finishFromState(terminalInput); + if (reduction.kind !== "parent" || !reduction.prepared.terminal) return undefined; + const { effects: _effects, ...parentEvent } = reduction.prepared; + const projection = reduction.prepared.terminal; + const finish: Extract = { + kind: "finish-live-event", outcome: command.outcome, projection, + input: { ...command.input, projection: { kind: "writer-staged", + ...(projection.assistant.messageId ? { messageId: projection.assistant.messageId } : {}) } }, + ...(command.frozenFileEvidence ? { frozenFileEvidence: command.frozenFileEvidence } : {}), + livePublication: [parentEvent.publication], + }; + return { request: { ...request, command: finish }, parentEvent }; + } + + private preparePostTerminalEvent( + request: ExecutionWorkerRequest, + command: Extract, + state: ExecutionState, + ): PreparedExecutionRequest | undefined { + if (!validLateOwnedEventCommand(command, state) || !state.parentEvents) return undefined; + const reduction = state.parentEvents.prepare(command.events, true); + if (reduction.kind !== "parent" || reduction.prepared.terminal) return undefined; + const { effects, ...prepared } = reduction.prepared; + const parentEvent = { ...prepared, publication: { ...prepared.publication, after: "terminal" as const } }; + const hooks = effects.narrative?.items.filter((item) => item.kind === "hook"); + return { parentEvent, request: { ...request, command: { + kind: "post-terminal-event", publication: parentEvent.publication, + ...(hooks?.length ? { hooks } : {}), ...(effects.systemIntents ? { systemIntents: effects.systemIntents } : {}), + providerEvent: { phase: command.phase, nativeCursor: command.nativeCursor, events: command.events }, + } } }; + } + + private async prepareFileEffects(prepared: PreparedExecutionRequest, state: ExecutionState): Promise { + const command = prepared.request.command; + if (command.kind === "begin-files") { + return this.beginFileEffects(command, state) ? prepared : undefined; + } + if (command.kind === "finish-live-event") { + if (state.fileAttempt !== undefined) return this.prepareTerminalFiles(prepared, command, state); + return command.frozenFileEvidence ? undefined : prepared; + } + if (command.kind !== "event" || state.fileAttempt === undefined) return prepared; + return await this.observeEventFiles(prepared, command, state) ? prepared : undefined; + } + + private beginFileEffects(command: Extract, state: ExecutionState): boolean { + if (!state.parentEvents || state.nextOrdinal !== 2 || !this.files.begin({ + execution: state.execution, deliveryAttempt: command.deliveryAttempt, cwd: command.cwd, handoff: command.handoff, + })) return false; + state.fileAttempt = command.deliveryAttempt; + return true; + } + + private async observeEventFiles( + prepared: PreparedExecutionRequest, + command: Extract, + state: ExecutionState, + ): Promise { + if (state.fileAttempt === undefined) return false; + const event = prepared.parentEvent?.publication.event; + if (event?.type === "toolUse" && command.capturedFileObservation) { + return this.files.observeToolUse({ execution: state.execution, deliveryAttempt: state.fileAttempt, + event, captured: command.capturedFileObservation }); + } + if (event?.type === "toolResult") await this.files.observeToolResult({ + execution: state.execution, deliveryAttempt: state.fileAttempt, event, + }); + return true; + } + + private async prepareTerminalFiles( + prepared: PreparedExecutionRequest, + command: Extract, + state: ExecutionState, + ): Promise { + const frozen = command.frozenFileEvidence; + if (!frozen || frozen.outcome !== command.outcome || frozen.deliveryAttempt !== state.fileAttempt) return undefined; + const evidence = await this.files.settle(frozen); + if (!evidence) return undefined; + return { ...prepared, request: { ...prepared.request, command: { ...command, + input: { ...command.input, deliveryAttempt: state.fileAttempt, fileEvidence: evidence } } } }; } private async begin( request: ExecutionWorkerRequest, existing: ExecutionState | undefined, ): Promise { - if (existing) return { kind: "rejected", reason: "invalid-transition" }; - if (request.ordinal !== 1) return { kind: "rejected", reason: "out-of-order" }; + const rejection = beginRejection(request, existing); + if (rejection) return { kind: "rejected", reason: rejection }; const command = request.command; - if (command.kind !== "start" && command.kind !== "resume") return { kind: "rejected", reason: "invalid-transition" }; - if (command.kind === "start" && !validStartInput(command, request.execution)) { - return { kind: "rejected", reason: "invalid-transition" }; - } - const mutation: ExecutionSemanticOperation["mutation"] = command.kind === "start" - ? { kind: "begin", providerId: command.providerId, input: command.input } - : { kind: "resume", providerId: command.providerId, checkpointId: command.checkpointId }; - const receipt = await this.writer.transact(operationFor(request, mutation)); + if (command.kind !== "start" && command.kind !== "resume") throw new Error("Invalid begin command escaped validation"); + const parent = parentReducerState(command, request.execution); + const started = this.admissionStart(command, parent); + const receipt = await this.writer.transact(operationFor( + withParentStartPublication(request, started), beginMutation(command), + )); if (!isDurableReceipt(receipt, request, 0)) { return { kind: "rejected", reason: "writer-conflict" }; } @@ -228,26 +424,76 @@ export class ExecutionWorkerHandler { execution: request.execution, lease: request.lease, providerId: command.providerId, + ...parent, nextOrdinal: 2, phase: "running", durableRevision: receipt.durableRevision, }); - return committedResult(receipt); + return started + ? { ...committedResult(receipt), parentEvent: started } + : committedResult(receipt); + } + + private admissionStart( + command: Extract, + parent: Pick, + ): PreparedProviderLiveEvent | undefined { + if (command.kind !== "start" || !command.publishParentStart) return undefined; + const started = parent.parentEvents?.startFromAdmission(); + if (started?.kind !== "parent" || started.prepared.publication.event.type !== "turnStarted") { + throw new Error("Worker-owned parent start could not be prepared"); + } + return started.prepared; } private release(request: ExecutionWorkerRequest, state: ExecutionState): ExecutionWorkerResult { - if (state.phase !== "finalized") return { kind: "rejected", reason: "invalid-transition" }; + if (state.phase !== "finalized" && !validRecoveryRelease(request.command, state.lease)) { + return { kind: "rejected", reason: "invalid-transition" }; + } + if (state.fileAttempt !== undefined) this.files.retire(state.execution, state.fileAttempt); this.states.delete(request.execution.threadId); return { kind: "released" }; } } +function beginRejection( + request: ExecutionWorkerRequest, + existing: ExecutionState | undefined, +): "invalid-transition" | "out-of-order" | null { + if (existing) return "invalid-transition"; + if (request.ordinal !== 1) return "out-of-order"; + const command = request.command; + if (command.kind !== "start" && command.kind !== "resume") return "invalid-transition"; + return command.kind === "start" && !validStartInput(command, request.execution) + ? "invalid-transition" : null; +} + +function beginMutation(command: Extract): ExecutionSemanticOperation["mutation"] { + return command.kind === "start" + ? { kind: "begin", providerId: command.providerId, input: command.input } + : { kind: "resume", providerId: command.providerId, checkpointId: command.checkpointId }; +} + +function withParentStartPublication( + request: ExecutionWorkerRequest, + started: PreparedProviderLiveEvent | undefined, +): ExecutionWorkerRequest { + return started && request.command.kind === "start" + ? { ...request, command: { ...request.command, livePublication: [started.publication] } } + : request; +} + function mutationFor( request: ExecutionWorkerRequest, state: ExecutionState, ): ExecutionSemanticOperation["mutation"] | undefined { const command = request.command; switch (command.kind) { + case "post-terminal-event": { + if (state.phase !== "finalized") return undefined; + const { publication: _publication, ...mutation } = command; + return mutation; + } case "event": return eventMutation(command, request.execution, state); case "narrative-delta": @@ -257,15 +503,51 @@ function mutationFor( case "stop": return stopMutation(command, request, state); case "finalize": + case "finish-live-event": return finishMutation(command, request.execution, state.providerId); - case "start": - case "resume": - case "release": - return undefined; - default: return metadataOrTextMutation(command, request.execution, state); + default: return auxiliaryMutation(command, request.execution, state); + } +} + +function auxiliaryMutation( + command: Extract, + execution: ExecutionIdentity, + state: ExecutionState, +): ExecutionSemanticOperation["mutation"] | undefined { + if (command.kind === "start" || command.kind === "resume" || command.kind === "release" || command.kind === "finish-from-state") return undefined; + // File handoff is the first command after durable start, before a provider cursor exists. + if (command.kind === "begin-files") return { kind: "checkpoint", phase: "running", nativeCursor: null }; + return metadataOrTextMutation(command, execution, state); +} + +function syntheticTerminalInput( + command: Extract, + state: ExecutionState, +): SyntheticTerminalInput | undefined { + if (state.phase !== "running" && state.phase !== "stopping") return undefined; + if (command.input.outcome !== command.outcome || command.input.providerId !== state.providerId + || !sameIdentity(command.input, state.execution)) return undefined; + switch (command.outcome) { + case "cancelled": return { outcome: "cancelled" }; + case "interrupted": return { outcome: "interrupted" }; + case "errored": return errorTerminalInput(command.input.error); } } +function errorTerminalInput(error: unknown): Extract | undefined { + return typeof error === "string" && error.length > 0 ? { outcome: "errored", error } : undefined; +} + +function validOwnedEventCommand(command: Extract, state: ExecutionState): boolean { + return command.parentLive === undefined && command.livePublication === undefined && state.phase === "running" + && (state.fileAttempt === undefined || command.deliveryAttempt === state.fileAttempt); +} + +function validLateOwnedEventCommand(command: Extract, state: ExecutionState): boolean { + return command.parentLive === undefined && command.livePublication === undefined + && (state.fileAttempt === undefined || command.deliveryAttempt === state.fileAttempt); +} + function metadataOrTextMutation( command: Extract, execution: ExecutionIdentity, @@ -288,8 +570,13 @@ function eventMutation( execution: ExecutionIdentity, state: ExecutionState, ): ExecutionSemanticOperation["mutation"] | undefined { - if (state.phase !== "running" || !validEventRouting(command.events, execution)) return undefined; - return { kind: "append-events", phase: command.phase, nativeCursor: command.nativeCursor, events: command.events }; + if (state.phase !== "running" || !validEventRouting(command.events, execution) + || (command.parentLive !== undefined && (!validLiveEventRouting(command.parentLive, execution) + || command.livePublication?.length !== 1))) return undefined; + return { + kind: "append-events", phase: command.phase, nativeCursor: command.nativeCursor, events: command.events, + ...(command.parentLive ? { parentLive: command.parentLive } : {}), + }; } function narrativeMutation( @@ -328,11 +615,18 @@ function stopMutation( } function finishMutation( - command: Extract, + command: Extract, execution: ExecutionIdentity, providerId: string, ): ExecutionSemanticOperation["mutation"] | undefined { if (!validFinishInput(command, execution, providerId)) return undefined; + if (command.kind === "finish-live-event") { + if (command.projection.threadId !== execution.threadId || command.projection.executionId !== execution.executionId + || command.projection.outcome !== command.outcome) return undefined; + if (command.providerEvent && !validEventRouting(command.providerEvent.events, execution)) return undefined; + return { kind: "finish-live-event", outcome: command.outcome, projection: command.projection, input: command.input, + ...(command.providerEvent ? { providerEvent: command.providerEvent } : {}) }; + } return { kind: "finish", outcome: command.outcome, input: command.input }; } @@ -341,10 +635,18 @@ function commandRejection( state: ExecutionState, ): Extract["reason"] | null { if (!sameIdentity(state.execution, request.execution) || !sameLease(state.lease, request.lease)) return "stale-execution"; + if (validRecoveryRelease(request.command, state.lease)) return null; if (request.ordinal !== state.nextOrdinal) return "out-of-order"; return null; } +function validRecoveryRelease(command: WorkerCommand, lease: ExecutionLease): boolean { + if (command.kind !== "release" || !command.recovery + || command.recovery.operationId !== `${lease.leaseId}:worker-lost`) return false; + return command.recovery.kind === "committed" && Number.isSafeInteger(command.recovery.durableRevision) + || command.recovery.kind === "conflict" && command.recovery.recoveryState === "already-terminal"; +} + function invalidReason(request: ExecutionWorkerRequest): Extract["reason"] { if (request.command.kind === "stop" && request.stopWatermark !== request.ordinal - 1) return "invalid-stop-watermark"; if (request.command.kind === "live-event") { @@ -359,7 +661,7 @@ function invalidRoutingReason( ): "invalid-event-routing" | "invalid-text-routing" | "invalid-narrative-routing" | null { switch (command.kind) { case "event": - return validEventRouting(command.events, execution) ? null : "invalid-event-routing"; + return invalidEventRoutingReason(command, execution); case "assistant-text": return validTextRouting(command.inputs, execution) ? null : "invalid-text-routing"; case "narrative-delta": @@ -390,7 +692,7 @@ function validTextRouting(inputs: readonly ParentAssistantTextCheckpointInput[], } function validLiveEventRouting( - command: Extract, + command: ParentLiveEffects, execution: ExecutionIdentity, ): boolean { if (command.narrative !== undefined && command.narrative?.executionId !== execution.executionId) return false; @@ -408,7 +710,7 @@ function validLiveEventRouting( } } -function validPlanMessageRouting(command: Extract): boolean { +function validPlanMessageRouting(command: ParentLiveEffects): boolean { return command.planOutput === undefined || command.message !== undefined; } @@ -419,11 +721,64 @@ function validStartInput( return command.providerId === command.input.thread.providerId && command.input.thread.id === execution.threadId && command.input.turnId === execution.turnId - && command.input.executionId === execution.executionId; + && command.input.executionId === execution.executionId + && validParentStartContext(command); +} + +function invalidEventRoutingReason( + command: Extract, + execution: ExecutionIdentity, +): "invalid-event-routing" | "invalid-text-routing" | null { + if (!validEventRouting(command.events, execution)) return "invalid-event-routing"; + return command.parentLive !== undefined && (!validLiveEventRouting(command.parentLive, execution) + || command.livePublication?.length !== 1) ? "invalid-text-routing" : null; +} + +function parentReducerState( + command: Extract, + execution: ExecutionIdentity, +): Pick { + if (command.kind !== "start" || !command.parentLive) return {}; + switch (command.providerId) { + case "codex": return { parentEvents: new ProviderExecutionEventState("codex", execution, command.parentLive) }; + case "claude": return { parentEvents: new ProviderExecutionEventState("claude", execution, command.parentLive) }; + case "cursor": return { parentEvents: new ProviderExecutionEventState("cursor", execution, command.parentLive) }; + default: throw new Error("Unsupported execution parent provider"); + } +} + +function validParentStartContext(command: Extract): boolean { + const context = command.parentLive; + if (!context) return true; + return supportedParentProvider(command.providerId) && typeof context.precedingMessageId === "string" + && context.precedingMessageId.length > 0 && context.precedingMessageId === command.input.userMessage.messageId + && (context.planFeature === "none" || context.planFeature === "questions" || context.planFeature === "output"); +} + +function supportedParentProvider(providerId: string): boolean { + return providerId === "codex" || providerId === "claude" || providerId === "cursor"; +} + +function prepareTerminalEvent( + command: Extract, + parentEvent: ExecutionParentEventResult, + providerId: string, +): Extract | undefined { + const projection = parentEvent.terminal; + const input = command.terminalInput; + if (!projection || !input || input.outcome !== projection.outcome || input.providerId !== providerId + || input.threadId !== projection.threadId || input.executionId !== projection.executionId) return undefined; + return { kind: "finish-live-event", outcome: projection.outcome, projection, + input: { ...input, ...(parentEvent.publication.event.type === "error" ? { error: parentEvent.publication.event.error } : {}), + projection: { kind: "writer-staged", + ...(projection.assistant.messageId ? { messageId: projection.assistant.messageId } : {}) } }, + providerEvent: { phase: command.phase, nativeCursor: command.nativeCursor, events: command.events }, + ...(command.frozenFileEvidence ? { frozenFileEvidence: command.frozenFileEvidence } : {}), + livePublication: [parentEvent.publication] }; } function validFinishInput( - command: Extract, + command: Extract, execution: ExecutionIdentity, providerId: string, ): boolean { @@ -453,8 +808,10 @@ function livePublicationFor(command: WorkerCommand): readonly ExecutionLivePubli switch (command.kind) { case "start": case "event": + case "finish-live-event": case "finalize": return command.livePublication; case "live-event": return [command.publication]; + case "post-terminal-event": return [command.publication]; default: return undefined; } } @@ -472,7 +829,7 @@ function isDurableReceipt( && Number.isSafeInteger(receipt.durableRevision) && receipt.durableRevision >= previousRevision; } -function committedResult(receipt: Extract): ExecutionWorkerResult { +function committedResult(receipt: Extract): Extract { return { kind: "committed", operationId: receipt.operationId, durableRevision: receipt.durableRevision, ...(receipt.providerCommit ? { providerCommit: receipt.providerCommit } : {}), @@ -481,6 +838,7 @@ function committedResult(receipt: Extract, options: SchedulerOptions, + private readonly onRecovered?: (assignment: ExecutionLostAssignment) => void, ) { this.scheduler = new ExecutionMailboxScheduler({ ...options, @@ -86,12 +87,13 @@ export class ExecutionWorkerLossCoordinator { const receipt = await this.writer.interruptWorkerLoss({ ...assignment, reason: WORKER_LOSS_REASON, - recoveryIncidentId: incidentId(assignment), + recoveryIncidentId: workerLossIncidentId(assignment), }); const evidence = recoveryEvidence(receipt); if (!evidence || !this.scheduler.reconcileLost(assignment.execution, assignment.lease, evidence)) { throw new Error(`Lost execution has no matching durable recovery evidence: ${assignment.execution.executionId}`); } + this.onRecovered?.(assignment); slot.unresolved.shift(); } if (!this.scheduler.replaceWorker(slot.workerIndex)) { @@ -112,7 +114,8 @@ function recoveryEvidence(receipt: ExecutionWriteReceipt): ExecutionRecoveryRece return { ...receipt, recoveryState: receipt.recoveryState }; } -function incidentId(assignment: ExecutionLostAssignment): string { +/** Stable incident identity for a writer-backed interruption of one exact lease. */ +export function workerLossIncidentId(assignment: ExecutionLostAssignment): string { const hash = NodeCrypto.createHash("sha256") .update(JSON.stringify([assignment.execution, assignment.lease])).digest("hex"); return `worker-loss:${hash}`; diff --git a/apps/server/src/features/agents/execution/execution.worker.ts b/apps/server/src/features/agents/execution/execution.worker.ts index 95342481d..bda8f86d8 100644 --- a/apps/server/src/features/agents/execution/execution.worker.ts +++ b/apps/server/src/features/agents/execution/execution.worker.ts @@ -13,6 +13,8 @@ interface PendingWrite { readonly reject: () => void; } +class ExecutionWriterRpcFailure extends Error {} + let nextRpcId = 1; let pendingWrite: PendingWrite | undefined; let commandActive = false; @@ -27,7 +29,7 @@ const writer: ExecutionSemanticWriter = { rpcId, operationId: operation.operationId, resolve, - reject: () => reject(new Error("Execution writer RPC failed")), + reject: () => reject(new ExecutionWriterRpcFailure("Execution writer RPC failed")), }; post({ kind: "writer-request", rpcId, operation }); }); @@ -70,7 +72,15 @@ function handleCommand(request: Extract { if (!closed) post({ kind: "command-reply", reply }); }) - .catch(() => failWorker()) + .catch((error: unknown) => { + if (error instanceof ExecutionWriterRpcFailure) { + post({ kind: "command-reply", reply: { requestId: request.requestId, + execution: request.execution, lease: request.lease, ordinal: request.ordinal, + result: { kind: "rejected", reason: "writer-failure" } } }); + } else { + failWorker(); + } + }) .finally(() => { commandActive = false; }); } diff --git a/apps/server/src/features/agents/execution/provider-execution-event-state.ts b/apps/server/src/features/agents/execution/provider-execution-event-state.ts new file mode 100644 index 000000000..0550b928b --- /dev/null +++ b/apps/server/src/features/agents/execution/provider-execution-event-state.ts @@ -0,0 +1,149 @@ +import { ProviderRuntimeEventSchema, type AgentEvent, type ProviderRuntimeEvent } from "@mcode/contracts"; +import type { ProviderEventDraft } from "@mcode/providers"; +import { processProviderEventWorkerTask } from "../../providers/composition/provider-event-worker-protocol.js"; +import { CodexLiveEventReducer, type CodexPlanFeature, type SyntheticTerminalInput } from "./codex-live-event-reducer.js"; +import { CodexLiveEventEffects, type PreparedCodexLiveEvent } from "./codex-live-event-effects.js"; +import type { ExecutionIdentity } from "./execution-mailbox-protocol.js"; +import { OtherProviderLiveEventEffects, type OtherProviderRuntimeIntent } from "./provider-live-event-effects.js"; + +/** Providers whose parent event effects have an execution worker implementation. */ +export type ExecutionLiveProviderId = "codex" | "claude" | "cursor"; + +/** Immutable reducer setup supplied with the same durable parent start. */ +export interface ExecutionParentStartContext { + readonly planFeature: CodexPlanFeature; + readonly precedingMessageId: string; +} + +/** A provider preparation retains all runtime work alongside its writer data. */ +export type PreparedProviderLiveEvent = Omit & { + readonly runtime: readonly OtherProviderRuntimeIntent[]; +}; + +/** Parent effects are prepared only for ordinary, explicitly routed provider events. */ +export type PreparedExecutionParentEvent = + | { readonly kind: "writer-owned" } + | { readonly kind: "parent"; readonly prepared: PreparedProviderLiveEvent } + | { readonly kind: "rejected" }; + +/** Owns the volatile parent reducer; the execution must be poisoned after a failed write. */ +export class ProviderExecutionEventState { + private readonly parent: ParentReducer; + + constructor(private readonly providerId: ExecutionLiveProviderId, private readonly execution: ExecutionIdentity, context: ExecutionParentStartContext) { + this.parent = createParentReducer(providerId, execution, context); + } + + /** Validate the whole draft before any reducer state changes. Child evidence stays writer-owned. */ + prepare(drafts: readonly ProviderEventDraft[], allowTerminal = false): PreparedExecutionParentEvent { + const parents: AgentEvent[] = []; + for (const draft of drafts) { + const result = this.parentEvent(draft); + if (result.kind === "rejected") return result; + if (result.kind === "parent") parents.push(result.event); + } + if (parents.length === 0) return { kind: "writer-owned" }; + if (parents.length !== 1 || drafts.length !== 1) return { kind: "rejected" }; + const event = parents[0]; + if (!event) return { kind: "rejected" }; + return this.reduceParent(event, allowTerminal); + } + + /** Synthesize terminal projection from this execution's buffers without host-side reconstruction. */ + finishFromState(input: SyntheticTerminalInput): PreparedExecutionParentEvent { + switch (this.parent.providerId) { + case "codex": { + const reduction = this.parent.reducer.finishFromState(input); + return reduction.kind === "reduced" + ? { kind: "parent", prepared: this.parent.effects.prepare(reduction) } : { kind: "rejected" }; + } + case "claude": return preparedOtherResult(this.parent.effects.finishFromState(input)); + case "cursor": return preparedOtherResult(this.parent.effects.finishFromState(input)); + } + } + + /** Publish the admission-owned start before the provider can stream its first event. */ + startFromAdmission(): PreparedExecutionParentEvent { + return this.reduceParent({ type: "turnStarted", threadId: this.execution.threadId, + turnExecutionId: this.execution.executionId }, false); + } + + private reduceParent(event: AgentEvent, allowTerminal: boolean): PreparedExecutionParentEvent { + if (!allowTerminal && isTerminalEvent(event)) return { kind: "rejected" }; + switch (this.parent.providerId) { + case "codex": { + const reduction = this.parent.reducer.reduce(event); + return reduction.kind === "reduced" + ? { kind: "parent", prepared: this.parent.effects.prepare(reduction) } : { kind: "rejected" }; + } + case "claude": return prepareOtherParent(this.parent.effects, event); + case "cursor": return prepareOtherParent(this.parent.effects, event); + } + } + + private parentEvent(draft: ProviderEventDraft): + | { kind: "parent"; event: AgentEvent } + | { kind: "writer-owned" } + | { kind: "rejected" } { + if (draft.sourceProviderId !== this.providerId || !this.matchesExecution(draft)) { + return { kind: "rejected" }; + } + if (draft.payload.type !== "item.recorded") return { kind: "writer-owned" }; + const item = draft.payload.item; + if (item.payload.projection !== "providerRuntimeEvent") return { kind: "writer-owned" }; + const runtime = this.validatedRuntimeEvent(draft, item); + if (!runtime) return { kind: "rejected" }; + if (hasWriterOwnedExtension(runtime)) return { kind: "writer-owned" }; + return { kind: "parent", event: runtime.event }; + } + + private validatedRuntimeEvent( + draft: ProviderEventDraft, + item: Extract["item"], + ): ProviderRuntimeEvent | undefined { + const runtime = ProviderRuntimeEventSchema().safeParse(item.payload.runtimeEvent); + if (!runtime.success || item.threadId !== draft.routing.threadId || item.turnId !== draft.routing.turnId + || item.id !== draft.routing.itemId) return undefined; + const parsed = processProviderEventWorkerTask({ kind: "provider-runtime", providerId: this.providerId, runtimeEvent: runtime.data }); + if (parsed.status === "rejected") return undefined; + if (parsed.status !== "accepted" || parsed.event.event.threadId !== this.execution.threadId + || parsed.event.event.turnExecutionId !== this.execution.executionId) return undefined; + return { ...runtime.data, event: parsed.event.event }; + } + + private matchesExecution(draft: ProviderEventDraft): boolean { + return draft.routing.threadId === this.execution.threadId && draft.routing.turnId === this.execution.turnId + && draft.routing.executionId === this.execution.executionId; + } +} + +type ParentReducer = + | { readonly providerId: "codex"; readonly reducer: CodexLiveEventReducer; readonly effects: CodexLiveEventEffects } + | { readonly providerId: "claude" | "cursor"; readonly effects: OtherProviderLiveEventEffects }; + +function createParentReducer(providerId: ExecutionLiveProviderId, execution: ExecutionIdentity, context: ExecutionParentStartContext): ParentReducer { + switch (providerId) { + case "codex": return { providerId, reducer: new CodexLiveEventReducer(execution, context.planFeature), + effects: new CodexLiveEventEffects(execution, context.precedingMessageId) }; + case "claude": return { providerId, effects: new OtherProviderLiveEventEffects("claude", execution, context.precedingMessageId, context.planFeature) }; + case "cursor": return { providerId, effects: new OtherProviderLiveEventEffects("cursor", execution, context.precedingMessageId, context.planFeature) }; + } +} + +function prepareOtherParent(effects: OtherProviderLiveEventEffects, event: AgentEvent): PreparedExecutionParentEvent { + return preparedOtherResult(effects.prepare(event)); +} + +function preparedOtherResult(prepared: ReturnType): PreparedExecutionParentEvent { + return prepared.kind === "prepared" ? { kind: "parent", prepared } : { kind: "rejected" }; +} + +function isTerminalEvent(event: AgentEvent): boolean { + return event.type === "turnComplete" || event.type === "error" + || event.type === "ended" && event.outcome !== undefined; +} + +function hasWriterOwnedExtension(runtime: ProviderRuntimeEvent): boolean { + const extension = runtime.extension; + return Boolean(extension?.child || extension?.collaboration || extension?.continuation); +} diff --git a/apps/server/src/features/agents/execution/provider-live-event-effects.ts b/apps/server/src/features/agents/execution/provider-live-event-effects.ts new file mode 100644 index 000000000..d98017aef --- /dev/null +++ b/apps/server/src/features/agents/execution/provider-live-event-effects.ts @@ -0,0 +1,100 @@ +import type { AgentEvent } from "@mcode/contracts"; + +import type { DataOnlyParentTerminalProjectionInput } from "../canonical/canonical-parent-turn-write.js"; +import type { ExecutionIdentity } from "./execution-mailbox-protocol.js"; +import { CodexLiveEventEffects, type CodexLiveRuntimeIntent } from "./codex-live-event-effects.js"; +import { CodexLiveEventReducer, type CodexPlanFeature, type CodexLiveReduction, type SyntheticTerminalInput } from "./codex-live-event-reducer.js"; +import type { ExecutionLivePublicationIntent, ParentLiveEffects } from "./execution-worker-handler.js"; + +/** Providers whose AgentEvents use the shared parent turn projection. */ +export type OtherLiveProviderId = "claude" | "cursor"; + +/** Runtime work retained after the event and its parent effects are durable. */ +export type OtherProviderRuntimeIntent = CodexLiveRuntimeIntent | { + readonly kind: "assistant-message-feature"; + readonly providerId: OtherLiveProviderId; + readonly event: Extract; +}; + +/** One execution-bound provider event prepared for a single semantic write. */ +export type OtherProviderLivePreparation = + | { + readonly kind: "prepared"; + readonly providerId: OtherLiveProviderId; + readonly execution: ExecutionIdentity; + readonly effects: ParentLiveEffects; + readonly publication: ExecutionLivePublicationIntent; + readonly runtime: readonly OtherProviderRuntimeIntent[]; + readonly terminal?: DataOnlyParentTerminalProjectionInput; + } + | { + readonly kind: "unsupported"; + readonly providerId: OtherLiveProviderId; + readonly execution: ExecutionIdentity; + readonly eventType: AgentEvent["type"]; + readonly reason: string; + }; + +/** + * Prepares Claude or Cursor parent writes from their normalized AgentEvents. + * Keep one instance per execution and discard it after any failed writer commit. + */ +export class OtherProviderLiveEventEffects { + private readonly reducer: CodexLiveEventReducer; + private readonly effects: CodexLiveEventEffects; + + constructor( + readonly providerId: OtherLiveProviderId, + readonly execution: ExecutionIdentity, + precedingMessageId: string, + planFeature: CodexPlanFeature = "none", + ) { + this.reducer = new CodexLiveEventReducer(execution, planFeature); + this.effects = new CodexLiveEventEffects(execution, precedingMessageId); + } + + /** Reduce exactly one event; unsupported input never yields a publication. */ + prepare(event: AgentEvent, endedAt?: string): OtherProviderLivePreparation { + if (event.type === "turnComplete" && event.providerId && event.providerId !== this.providerId) { + return { + kind: "unsupported", + providerId: this.providerId, + execution: this.execution, + eventType: event.type, + reason: "different provider", + }; + } + return this.prepareReduction(this.reducer.reduce(event), endedAt); + } + + /** Prepare an interrupted execution from worker-owned buffers, including unclassified partial text. */ + finishFromState(input: SyntheticTerminalInput, endedAt?: string): OtherProviderLivePreparation { + return this.prepareReduction(this.reducer.finishFromState(input), endedAt); + } + + private prepareReduction(reduction: CodexLiveReduction, endedAt?: string): OtherProviderLivePreparation { + if (reduction.kind === "unsupported") { + return { + kind: "unsupported", + providerId: this.providerId, + execution: this.execution, + eventType: reduction.eventType, + reason: reduction.reason, + }; + } + const prepared = this.effects.prepare(reduction, endedAt); + const messageFeature = reduction.writer.find((intent) => intent.kind === "feature-event" + && intent.feature === "assistant-message"); + const runtime: OtherProviderRuntimeIntent[] = [...prepared.runtime]; + if (messageFeature?.kind === "feature-event" && messageFeature.event.type === "message") { + runtime.push({ kind: "assistant-message-feature", providerId: this.providerId, event: messageFeature.event }); + } + return structuredClone({ + kind: "prepared", + providerId: this.providerId, + execution: this.execution, + ...prepared, + runtime, + }); + } +} diff --git a/apps/server/src/features/agents/execution/worker-owned-turn-runtime.ts b/apps/server/src/features/agents/execution/worker-owned-turn-runtime.ts new file mode 100644 index 000000000..6c3367a83 --- /dev/null +++ b/apps/server/src/features/agents/execution/worker-owned-turn-runtime.ts @@ -0,0 +1,116 @@ +import { broadcast } from "../../../application/transport/push.js"; +import { CanonicalAgentWriterClient } from "../canonical/canonical-agent-writer-client.js"; +import { publishCanonicalAgentEvents } from "../canonical/canonical-agent-boundary.js"; +import { CanonicalExecutionWriterPort } from "../canonical/canonical-execution-writer-port.js"; +import { ExecutionLivePublicationRelease } from "../canonical/execution-live-publication-release.js"; +import { ExecutionPlanQuestionRelease } from "../canonical/execution-plan-question-release.js"; +import { AgentEventPublicationRegistry } from "../orchestration/agent-event-publication-registry.js"; +import { ExecutionMailboxOwner } from "./execution-mailbox-owner.js"; +import { ExecutionMailboxScheduler, type ExecutionMailboxLimits } from "./execution-mailbox-scheduler.js"; +import { ExecutionProviderEventOwnership } from "./execution-provider-event-ownership.js"; +import { ExecutionThreadWorkerPort } from "./execution-worker-port.js"; +import type { ExecutionIdentity } from "./execution-mailbox-protocol.js"; +import { ExecutionWorkerLossCoordinator, workerLossIncidentId } from "./execution-worker-loss-coordinator.js"; +import type { ExecutionWorkCommand, ExecutionWorkerResult } from "./execution-worker-handler.js"; + +const EXECUTION_WORKER_COUNT = 4; +const EXECUTION_MAILBOX_LIMITS: ExecutionMailboxLimits = { + maxPending: 256, + maxPendingBytes: 16 * 1024 * 1024, + reservedControl: 32, + reservedControlBytes: 2 * 1024 * 1024, + maxPerExecutionPending: 64, + maxPerExecutionBytes: 4 * 1024 * 1024, + reservedPerExecutionControl: 8, + reservedPerExecutionControlBytes: 512 * 1024, +}; + +/** Owns the fixed execution pool and sole semantic writer for this server process. */ +export class WorkerOwnedTurnRuntime { + readonly writer: CanonicalAgentWriterClient; + readonly writerPort: CanonicalExecutionWriterPort; + readonly workerLoss: ExecutionWorkerLossCoordinator; + readonly scheduler: ExecutionMailboxScheduler; + readonly owner: ExecutionMailboxOwner; + readonly providerEvents: ExecutionProviderEventOwnership; + private readonly initialWorkers: ExecutionThreadWorkerPort[] = []; + private readonly rejectedRecoveries = new Map>(); + private onRecovered: ((execution: ExecutionIdentity) => void) | undefined; + + constructor(dbPath: string, publication: AgentEventPublicationRegistry) { + this.writer = new CanonicalAgentWriterClient(dbPath); + this.writerPort = new CanonicalExecutionWriterPort( + this.writer, + publishCanonicalAgentEvents, + new ExecutionLivePublicationRelease(publication), + new ExecutionPlanQuestionRelease((threadId, questions) => { + broadcast("plan.questions", { threadId, questions: [...questions] }); + }), + ); + this.workerLoss = new ExecutionWorkerLossCoordinator(this.writerPort, { + workerCount: EXECUTION_WORKER_COUNT, + limits: EXECUTION_MAILBOX_LIMITS, + createWorker: () => { + const worker = new ExecutionThreadWorkerPort(this.writerPort); + if (this.initialWorkers.length < EXECUTION_WORKER_COUNT) this.initialWorkers.push(worker); + return worker; + }, + }, (assignment) => { + this.owner.forgetRecovered(assignment.execution, assignment.lease); + this.onRecovered?.(assignment.execution); + }); + this.scheduler = this.workerLoss.scheduler; + this.owner = new ExecutionMailboxOwner(this.scheduler); + this.providerEvents = new ExecutionProviderEventOwnership(this.owner); + } + + /** Notify the runtime controller only after writer-backed worker-loss recovery releases the lease. */ + bindRecovered(onRecovered: (execution: ExecutionIdentity) => void): void { + this.onRecovered = onRecovered; + } + + /** Interrupt and release only the rejected execution; other assignments keep their worker. */ + async recoverRejected(execution: ExecutionIdentity): Promise { + const existing = this.rejectedRecoveries.get(execution.executionId); + if (existing) return existing; + const owned = this.owner.current(execution.threadId); + if (!owned || owned.execution.executionId !== execution.executionId) return; + const task = this.recoverRejectedOwned(execution, owned.lease); + this.rejectedRecoveries.set(execution.executionId, task); + try { + await task; + } finally { + this.rejectedRecoveries.delete(execution.executionId); + } + } + + private async recoverRejectedOwned(execution: ExecutionIdentity, lease: NonNullable>["lease"]): Promise { + const receipt = await this.writerPort.interruptWorkerLoss({ + execution, lease, + reason: "The provider event could not be durably applied to this execution.", + recoveryIncidentId: workerLossIncidentId({ execution, lease }), + }); + const recovery = receipt.kind === "committed" ? receipt + : receipt.recoveryState === "already-terminal" + ? { ...receipt, recoveryState: "already-terminal" as const } + : null; + if (!recovery) throw new Error("Rejected execution has no durable recovery evidence"); + await this.owner.releaseRecovered(execution, recovery); + this.onRecovered?.(execution); + } + + /** Wait for the writer and every initial slot before accepting a provider turn. */ + async whenReady(): Promise { + await this.writer.whenReady(); + const ready = await Promise.all(this.initialWorkers.map((worker) => worker.whenReady())); + if (ready.length !== EXECUTION_WORKER_COUNT || ready.some((value) => !value)) { + throw new Error("Execution worker pool did not start"); + } + } + + /** Stop mailbox admission before closing its durable writer. */ + async close(): Promise { + this.scheduler.shutdown(); + await this.writer.close(); + } +} diff --git a/apps/server/src/features/agents/orchestration/turn-runtime-controller.ts b/apps/server/src/features/agents/orchestration/turn-runtime-controller.ts index cb4b8682e..b6c6edb17 100644 --- a/apps/server/src/features/agents/orchestration/turn-runtime-controller.ts +++ b/apps/server/src/features/agents/orchestration/turn-runtime-controller.ts @@ -5,9 +5,11 @@ * Extracted from apps/desktop/src/main/app-state.ts. */ +import * as NodeCrypto from "node:crypto"; import { injectable, inject, delay } from "tsyringe"; import { logger } from "@mcode/shared"; -import { AgentEventType, isSessionEvictable } from "@mcode/contracts"; +import { AgentEventType, ProviderRuntimeEventSchema, isSessionEvictable } from "@mcode/contracts"; +import type { CodexProviderBoundary } from "@mcode/providers"; import type { Thread, IProviderRegistry, @@ -17,6 +19,7 @@ import type { ProviderId, TurnRuntimeSnapshot, AgentStopResult, + ProviderRuntimeEvent, } from "@mcode/contracts"; import { TURN_FILE_EFFECTS, TurnFileEffects } from "../turns/turn-file-effects.js"; import type { WorkspaceEnvironmentAutomaticSetupDispatch } from "../../projects/environment/workspace-environment-service.js"; @@ -31,6 +34,13 @@ import { TurnErrorPolicy } from "../turns/turn-error-policy.js"; import { TurnRuntimeRegistry } from "../turns/turn-runtime.js"; import type { TurnOutcome } from "../turns/turn-outcome.js"; import { ProviderEventIngress } from "../../providers/composition/provider-event-ingress.js"; +import { ExecutionFileEvidenceCoordinator } from "../execution/execution-file-evidence-coordinator.js"; +import type { ExecutionIdentity } from "../execution/execution-mailbox-protocol.js"; +import type { ExecutionWorkCommand, ExecutionWorkerResult } from "../execution/execution-worker-handler.js"; +import { WorkerOwnedTurnRuntime } from "../execution/worker-owned-turn-runtime.js"; +import type { WorkerOwnedProviderEventBatch } from "../../providers/composition/provider-host-ports.js"; +import { SnapshotService } from "../../projects/diffs/snapshots/snapshot-service.js"; +import type { DataOnlyParentTurnFinishInput } from "../canonical/canonical-parent-turn-write.js"; import { TurnEventPipeline, type FinalizeTurnCommand, @@ -95,6 +105,21 @@ type PreparedStop = { runtime: TurnRuntimeSnapshot; }; +interface WorkerOwnedTurn { + readonly execution: ExecutionIdentity; + readonly prepared: PreparedTurnDispatch; + readonly provider: CodexProviderBoundary; + deliveryAttempt: number; + terminalCommitted: boolean; + persistedPublished: boolean; + terminalOutcome?: TurnOutcome; + releaseStarted: boolean; + deliveryFailed?: boolean; + deliveryFailureHandling?: boolean; + stopInProgress?: boolean; + stopRequested?: boolean; +} + /** Read-only runtime state available to server-owned diagnostics and recovery infrastructure. */ export interface AgentRuntimeAccess { /** Return the number of active agent sessions. */ @@ -184,6 +209,7 @@ export class TurnRuntimeController implements TurnLifecycleControl, TurnRuntimeE private readonly turnGenerations = new Map(); /** Single ordered pipeline for validated provider envelopes and terminal materialization. */ private readonly turnEventPipeline: TurnEventPipeline; + private readonly workerTurns = new Map(); constructor( @inject(TURN_RUNTIME_PERSISTENCE) private readonly runtimePersistence: TurnRuntimePersistence, @@ -220,9 +246,19 @@ export class TurnRuntimeController implements TurnLifecycleControl, TurnRuntimeE @inject(delay(() => ProviderTurnEventApplication)) private readonly eventApplication: ProviderTurnEventApplication, @inject(TurnDiffService) turnDiffs: TurnDiffService, + @inject("WorkerOwnedTurnRuntime", { isOptional: true }) + private readonly workerRuntime?: WorkerOwnedTurnRuntime, + @inject("ExecutionFileEvidenceCoordinator", { isOptional: true }) + private readonly workerFiles?: ExecutionFileEvidenceCoordinator, + @inject("WorkerTurnSnapshotService", { isOptional: true }) + private readonly snapshots?: SnapshotService, ) { this.turnDiffs = turnDiffs; - this.turnEventPipeline = new TurnEventPipeline(this, eventApplication, turnDiffs, providerEventIngress); + this.turnEventPipeline = new TurnEventPipeline(this, eventApplication, turnDiffs, providerEventIngress, + workerRuntime && workerFiles ? { + ownsFileMutation: (event) => Boolean(workerRuntime.owner.current(event.threadId)?.started), + capture: (event) => workerFiles.capture(event), + } : undefined); runtimeCommands.bind({ sendMessage: (command) => this.sendMessage(command), runtimeSnapshots: () => this.runtimeSnapshots(), @@ -237,6 +273,9 @@ export class TurnRuntimeController implements TurnLifecycleControl, TurnRuntimeE }); continuation.bind((executionId) => eventApplication.continueWithoutSaving(executionId)); reliability.bind((threadId) => eventApplication.streamReliabilityAssistantText(threadId)); + workerRuntime?.providerEvents.bindCommandPreparation((execution, batch) => this.prepareWorkerEvent(execution, batch)); + workerRuntime?.providerEvents.bindCommitted((execution, result) => this.applyWorkerReceipt(execution, result)); + workerRuntime?.bindRecovered((execution) => this.handleWorkerRecovery(execution)); this.eventPublication.registerPipelineStart(() => this.initializeProviderEvents()); } @@ -488,6 +527,12 @@ export class TurnRuntimeController implements TurnLifecycleControl, TurnRuntimeE /** Terminalize an admitted turn when durable admission cannot complete. */ private async abortTurnAdmission(lease: TurnRuntimeLease): Promise { + if (await this.finishUnsentWorkerTurn(lease.threadId, lease.turnExecutionId)) { + this.turnRuntime.terminalize(lease.threadId, lease.turnExecutionId, "errored"); + this.disarmTurnRetryWindow(lease.threadId); + this.trackSessionEnded(lease.threadId, lease.turnExecutionId); + return; + } if (!this.turnRuntime.terminalize(lease.threadId, lease.turnExecutionId, "errored")) return; await (this.finalizeTerminalTurn(lease.threadId, "errored", "turn admission failure") ?? Promise.resolve()); this.disarmTurnRetryWindow(lease.threadId); @@ -514,6 +559,8 @@ export class TurnRuntimeController implements TurnLifecycleControl, TurnRuntimeE await this.activatePreparedCommandEffect(prepared); await this.startPreparedProviderDispatch(prepared); } catch (error) { + logger.error("Prepared turn dispatch failed", { threadId: prepared.lease.threadId, + error: error instanceof Error ? error.message : String(error) }); await this.failPreparedTurnDispatch(prepared, error); } } @@ -521,6 +568,10 @@ export class TurnRuntimeController implements TurnLifecycleControl, TurnRuntimeE /** Initialize pipeline state after the parent-turn transaction has committed. */ private async prepareRuntimeDispatch(prepared: PreparedTurnDispatch): Promise { const { lease, request } = prepared; + if (prepared.workerOwned) { + await this.prepareWorkerRuntimeDispatch(prepared); + return; + } this.eventApplication.beginPreparedTurn(lease.threadId, lease.turnExecutionId, request.turnId); this.turnAdmissions.markDispatchActive(lease.threadId); this.emitProviderEvent(prepared.provider, { @@ -533,6 +584,255 @@ export class TurnRuntimeController implements TurnLifecycleControl, TurnRuntimeE this.eventApplication.recordContextSeed(lease.threadId, prepared.contextSeed, prepared.contextWindow ?? undefined); } + /** Bind file evidence and the exact provider route after the parent start receipt. */ + private async prepareWorkerRuntimeDispatch(prepared: PreparedTurnDispatch): Promise { + const worker = this.requireWorkerRuntime(); + const files = this.requireWorkerFiles(); + const snapshots = this.snapshots; + if (!snapshots) throw new Error("Worker-owned turn requires snapshot service"); + const execution = this.executionFor(prepared); + const provider = this.codexProvider(prepared.provider); + let baselineRef: string | null = null; + try { + baselineRef = await snapshots.captureRef(prepared.cwd); + } catch (error) { + logger.warn("Could not capture initial turn snapshot", { + threadId: execution.threadId, + error: error instanceof Error ? error.message : String(error), + }); + } + if (!this.ownsTurnAdmission(prepared.lease)) throw new Error("Worker turn lost admission during file preparation"); + const handoff = files.begin({ ...execution, deliveryAttempt: 1, cwd: prepared.cwd, baselineRef }); + const began = await worker.owner.submit(execution, { + kind: "begin-files", cwd: prepared.cwd, handoff, deliveryAttempt: 1, + }); + if (began.kind !== "committed") throw new Error("Worker file handoff was not committed"); + this.markProviderTurnActive(execution.threadId); + await worker.providerEvents.bind(execution, 1); + this.workerTurns.set(execution.threadId, { + execution, prepared, provider, deliveryAttempt: 1, + terminalCommitted: false, persistedPublished: false, releaseStarted: false, + }); + provider.setCanonicalTurnDeliveryFailureHandler((routing, error) => this.handleWorkerDeliveryFailure(routing, error)); + provider.setCanonicalTurnEventDeliveryEnabled(true); + this.turnAdmissions.markDispatchActive(execution.threadId); + } + + private executionFor(prepared: PreparedTurnDispatch): ExecutionIdentity { + return { + threadId: prepared.lease.threadId, + turnId: prepared.request.turnId, + executionId: prepared.lease.turnExecutionId, + }; + } + + private codexProvider(provider: IAgentProvider): CodexProviderBoundary { + if (provider.id !== "codex" || !("setCanonicalTurnEventDeliveryEnabled" in provider) + || !("fenceCanonicalTurnEvents" in provider)) { + throw new Error("Codex provider has no worker-owned event route"); + } + return provider as CodexProviderBoundary; + } + + private requireWorkerRuntime(): WorkerOwnedTurnRuntime { + if (!this.workerRuntime) throw new Error("Worker-owned turn runtime is unavailable"); + return this.workerRuntime; + } + + private requireWorkerFiles(): ExecutionFileEvidenceCoordinator { + if (!this.workerFiles) throw new Error("Worker-owned file capture is unavailable"); + return this.workerFiles; + } + + /** Attach host-captured evidence before the exact provider batch enters its mailbox. */ + private async prepareWorkerEvent( + execution: ExecutionIdentity, + batch: WorkerOwnedProviderEventBatch, + ): Promise> { + const active = this.requireActiveWorkerTurn(execution, batch.deliveryAttempt); + const event = this.parentRuntimeEvent(batch); + const fileObservation = event?.type === "toolUse" + ? this.requireWorkerFiles().take(event, batch.deliveryAttempt) : null; + const outcome = event ? workerTerminalOutcome(event) : null; + const terminal = outcome ? this.prepareWorkerTerminal(active, outcome) : null; + return { + kind: "event", phase: batch.phase, nativeCursor: batch.nativeCursor ?? null, + events: batch.events, deliveryAttempt: batch.deliveryAttempt, + ...(fileObservation ? { capturedFileObservation: fileObservation } : {}), + ...(terminal ? { terminalInput: terminal.input, frozenFileEvidence: terminal.files } : {}), + }; + } + + private requireActiveWorkerTurn(execution: ExecutionIdentity, attempt: number): WorkerOwnedTurn { + const active = this.workerTurns.get(execution.threadId); + if (!active || active.execution.executionId !== execution.executionId + || active.deliveryAttempt !== attempt) { + throw new Error("Provider event has no matching worker-owned turn"); + } + return active; + } + + private parentRuntimeEvent(batch: WorkerOwnedProviderEventBatch): AgentEvent | null { + if (batch.events.length !== 1) return null; + const draft = batch.events[0]; + if (!draft || draft.payload.type !== "item.recorded") return null; + const payload = draft.payload.item.payload; + if (payload.projection !== "providerRuntimeEvent") return null; + const parsed = ProviderRuntimeEventSchema().safeParse(payload.runtimeEvent); + if (!parsed.success) throw new Error("Worker event has an invalid provider runtime payload"); + if (writerOwnedProviderExtension(parsed.data.extension)) return null; + const event = parsed.data.event; + if (!matchesWorkerEvent(event, batch)) { + throw new Error("Worker event belongs to a different execution"); + } + return event; + } + + private prepareWorkerTerminal(active: WorkerOwnedTurn, outcome: TurnOutcome): { + input: DataOnlyParentTurnFinishInput; + files: NonNullable>; + } { + const files = this.requireWorkerFiles().seal({ ...active.execution, + deliveryAttempt: active.deliveryAttempt, outcome }); + if (!files) throw new Error("Terminal event lost its exact file evidence generation"); + return { + files, + input: { + ...active.execution, + providerId: active.prepared.providerId, + providerIdentities: active.prepared.parentStartInput.providerIdentities, + outcome, + deliveryAttempt: active.deliveryAttempt, + projection: { kind: "writer-staged" }, + }, + }; + } + + /** Update volatile runtime state only after the semantic writer has acknowledged the event. */ + private async applyWorkerReceipt( + execution: ExecutionIdentity, + result: Extract, + ): Promise { + const active = this.workerTurns.get(execution.threadId); + if (!active || active.execution.executionId !== execution.executionId) return; + const event = result.parentEvent?.publication.event; + if (!event) return; + this.applyWorkerFeatureReceipt(active, result); + this.publishWorkerPersistence(active, result); + this.applyWorkerTerminalReceipt(active, event); + if (event.type === "ended") { + // The callback still belongs to the route's pending set. Release on the next task. + setTimeout(() => { void this.releaseWorkerTurn(active).catch((error: unknown) => { + logger.error("Worker-owned terminal release failed", { threadId: execution.threadId, + error: error instanceof Error ? error.message : String(error) }); + }); }, 0); + } + } + + private applyWorkerFeatureReceipt( + active: WorkerOwnedTurn, + result: Extract, + ): void { + const event = result.parentEvent?.publication.event; + if (event?.type === "message" && active.prepared.providerId === "codex") { + this.featureEffects.onAssistantMessage("codex", event); + } + for (const intent of result.parentEvent?.runtime ?? []) { + if (intent.kind === "assistant-message-feature") { + this.featureEffects.onAssistantMessage(intent.providerId, intent.event); + } + } + } + + private applyWorkerTerminalReceipt(active: WorkerOwnedTurn, event: AgentEvent): void { + if (event.type === "turnStarted") this.markProviderTurnActive(event.threadId); + const outcome = workerTerminalOutcome(event); + if (!outcome) return; + active.terminalCommitted = true; + active.terminalOutcome = outcome; + this.turnRuntime.terminalize(active.execution.threadId, active.execution.executionId, outcome); + } + + private publishWorkerPersistence( + active: WorkerOwnedTurn | undefined, + result: Extract, + ): void { + if (!active) return; + const persisted = result.terminalPersistence; + if (!persisted || active.persistedPublished) return; + active.persistedPublished = true; + broadcast("thread.status", { threadId: active.execution.threadId, + status: persisted.outcome === "cancelled" ? "interrupted" : persisted.outcome }); + broadcast("turn.persisted", { + threadId: active.execution.threadId, + turnId: active.execution.turnId, + messageId: persisted.messageId, + toolCallCount: persisted.toolCallCount, + filesChanged: [...persisted.filesChanged], + outcome: persisted.outcome, + executionId: active.execution.executionId, + ...(persisted.fileEffects ? { fileEffects: persisted.fileEffects } : {}), + }); + } + + private async releaseWorkerTurn(active: WorkerOwnedTurn): Promise { + if (active.releaseStarted || active.stopInProgress || !active.terminalCommitted) return; + active.releaseStarted = true; + const { execution, provider, deliveryAttempt } = active; + try { + const routing = { ...execution, deliveryAttempt }; + if (active.deliveryFailed) { + await provider.fenceCanonicalTurnEvents(routing).catch(() => {}); + await provider.retireCanonicalTurnEvents(routing).catch(() => {}); + } else { + await provider.fenceCanonicalTurnEvents(routing); + await provider.retireCanonicalTurnEvents(routing); + } + await this.requireWorkerRuntime().providerEvents.retire(execution); + await this.requireWorkerRuntime().owner.release(execution); + this.requireWorkerFiles().retire(execution.threadId, execution.executionId, deliveryAttempt); + if (this.workerTurns.get(execution.threadId) === active) this.workerTurns.delete(execution.threadId); + this.trackSessionEnded(execution.threadId, execution.executionId); + if (active.terminalOutcome === "completed") this.featureEffects.refreshAfterTurn(execution.threadId); + this.disarmTurnRetryWindow(execution.threadId); + this.clearTurnEndedState(execution.threadId); + } catch (error) { + active.releaseStarted = false; + await this.requireWorkerRuntime().recoverRejected(execution); + throw error; + } + } + + private handleWorkerRecovery(execution: ExecutionIdentity): void { + if (this.turnRuntime.snapshot(execution.threadId)?.turnExecutionId !== execution.executionId) return; + const active = this.workerTurns.get(execution.threadId); + if (active && active.execution.executionId !== execution.executionId) return; + if (active) this.workerTurns.delete(execution.threadId); + if (active) this.recoverWorkerProvider(active); + this.turnRuntime.terminalize(execution.threadId, execution.executionId, "interrupted"); + this.trackSessionEnded(execution.threadId, execution.executionId); + this.disarmTurnRetryWindow(execution.threadId); + this.clearTurnEndedState(execution.threadId); + } + + private recoverWorkerProvider(active: WorkerOwnedTurn): void { + const { execution } = active; + void active.provider.fenceCanonicalTurnEvents({ ...execution, + deliveryAttempt: active.deliveryAttempt }).catch((error: unknown) => { + logger.warn("Recovered worker provider fence failed", { threadId: execution.threadId, + error: error instanceof Error ? error.message : String(error) }); + }); + void this.requireWorkerRuntime().providerEvents.retire(execution).catch((error: unknown) => { + logger.warn("Recovered worker provider route cleanup failed", { threadId: execution.threadId, + error: error instanceof Error ? error.message : String(error) }); + }); + this.requireWorkerFiles().retire(execution.threadId, execution.executionId, active.deliveryAttempt); + void Promise.resolve(active.provider.stopSession(active.prepared.request.sessionId)).catch((error: unknown) => { + logger.warn("Recovered worker provider stop failed", { threadId: execution.threadId, + error: error instanceof Error ? error.message : String(error) }); + }); + } + /** Activate command-specific state only after the runtime can still dispatch. */ private async activatePreparedCommandEffect(prepared: PreparedTurnDispatch): Promise { if (!this.ownsTurnAdmission(prepared.lease)) { @@ -618,7 +918,9 @@ export class TurnRuntimeController implements TurnLifecycleControl, TurnRuntimeE "activeTurn", () => { dispatch.dispatchStarted = true; - this.turnDiffs.begin({ ...dispatch.turnRequest, deliveryAttempt: dispatch.turnRequest.deliveryAttempt ?? 1 }); + if (!this.workerTurns.has(threadId)) { + this.turnDiffs.begin({ ...dispatch.turnRequest, deliveryAttempt: dispatch.turnRequest.deliveryAttempt ?? 1 }); + } return dispatch.resolvedProvider.sendTurn(dispatch.turnRequest); }, ); @@ -644,10 +946,69 @@ export class TurnRuntimeController implements TurnLifecycleControl, TurnRuntimeE error: unknown, ): Promise { if (!this.mutationReservations.owns(threadId, dispatch.mutationReservationToken, "activeTurn")) return; + const workerTurn = this.workerTurns.get(threadId); + if (workerTurn) { + try { + await this.failWorkerDispatch(workerTurn, error); + } catch (failure) { + await this.requireWorkerRuntime().recoverRejected(workerTurn.execution); + throw failure; + } + return; + } if (await this.runTransientTurnRetry(threadId, error)) return; await this.giveUpTransientTurnRetry(threadId, error); } + private async failWorkerDispatch(active: WorkerOwnedTurn, error: unknown): Promise { + const { execution, deliveryAttempt, provider } = active; + this.requireWorkerFiles().fence(execution.threadId, execution.executionId, deliveryAttempt); + if (active.deliveryFailed) { + await provider.fenceCanonicalTurnEvents({ ...execution, deliveryAttempt }).catch(() => {}); + } else { + await provider.fenceCanonicalTurnEvents({ ...execution, deliveryAttempt }); + } + if (active.terminalCommitted) return this.releaseWorkerTurn(active); + const frozen = this.requireWorkerFiles().seal({ ...execution, deliveryAttempt, outcome: "errored" }); + if (!frozen) throw new Error("Failed worker dispatch lost its file evidence generation"); + const result = await this.requireWorkerRuntime().owner.submit(execution, { + kind: "finish-from-state", outcome: "errored", frozenFileEvidence: frozen, + input: { ...execution, providerId: active.prepared.providerId, + providerIdentities: active.prepared.parentStartInput.providerIdentities, + outcome: "errored", error: error instanceof Error ? error.message : String(error), + deliveryAttempt, projection: { kind: "writer-staged" } }, + }); + if (result.kind !== "committed") throw new Error("Failed worker dispatch terminal did not commit"); + this.publishWorkerPersistence(active, result); + active.terminalCommitted = true; + active.terminalOutcome = "errored"; + this.turnRuntime.terminalize(execution.threadId, execution.executionId, "errored"); + await this.releaseWorkerTurn(active); + } + + private async handleWorkerDeliveryFailure( + routing: { threadId: string; turnId: string; executionId: string; deliveryAttempt: number }, + error: Error, + ): Promise { + const active = this.workerTurns.get(routing.threadId); + if (!active || active.execution.turnId !== routing.turnId + || active.execution.executionId !== routing.executionId + || active.deliveryAttempt !== routing.deliveryAttempt || active.deliveryFailureHandling) return; + active.deliveryFailed = true; + active.deliveryFailureHandling = true; + logger.error("Worker-owned canonical delivery failed", { threadId: routing.threadId, + executionId: routing.executionId, deliveryAttempt: routing.deliveryAttempt, error: error.message }); + try { + await this.failWorkerDispatch(active, error); + } catch (failure) { + logger.error("Worker-owned canonical delivery recovery failed", { threadId: routing.threadId, + executionId: routing.executionId, error: failure instanceof Error ? failure.message : String(failure) }); + await this.requireWorkerRuntime().recoverRejected(active.execution); + } finally { + active.deliveryFailureHandling = false; + } + } + /** Terminalize setup failures while preserving an explicit user-stop winner. */ private async failPreparedTurnDispatch(prepared: PreparedTurnDispatch, error: unknown): Promise { const runtime = this.turnRuntime.snapshot(prepared.lease.threadId); @@ -659,6 +1020,7 @@ export class TurnRuntimeController implements TurnLifecycleControl, TurnRuntimeE this.releaseTurnAdmission(prepared.lease); return; } + if (prepared.workerOwned) return this.failWorkerPreparedDispatch(prepared, error); if (this.turnRuntime.terminalize(prepared.lease.threadId, prepared.lease.turnExecutionId, "errored")) { await (this.finalizeTerminalTurn(prepared.lease.threadId, "errored", "send setup failure") ?? Promise.resolve()); this.disarmTurnRetryWindow(prepared.lease.threadId); @@ -669,6 +1031,79 @@ export class TurnRuntimeController implements TurnLifecycleControl, TurnRuntimeE throw error; } + private async failWorkerPreparedDispatch(prepared: PreparedTurnDispatch, error: unknown): Promise { + if (this.turnRetryDispatchByThread.get(prepared.lease.threadId)?.dispatchStarted) throw error; + await this.finishUnsentWorkerTurn(prepared.lease.threadId, prepared.lease.turnExecutionId); + this.turnRuntime.terminalize(prepared.lease.threadId, prepared.lease.turnExecutionId, "errored"); + this.disarmTurnRetryWindow(prepared.lease.threadId); + this.trackSessionEnded(prepared.lease.threadId, prepared.lease.turnExecutionId); + this.releaseTurnAdmission(prepared.lease); + await this.turnAdmissions.rollbackCommandEffect(prepared.commandEffect); + throw error; + } + + /** Finish a committed parent start that never reached provider dispatch. */ + private async finishUnsentWorkerTurn(threadId: string, executionId: string, outcome: "errored" | "cancelled" = "errored"): Promise { + const worker = this.workerRuntime; + const owned = worker?.owner.current(threadId); + if (!worker || !owned?.started || owned.execution.executionId !== executionId) return false; + const active = this.workerTurns.get(threadId); + const execution = owned.execution; + const result = await this.submitUnsentWorkerFinish(worker, execution, active, outcome); + if (!result) return true; + if (result.kind !== "committed") throw new Error("Unsent worker turn did not finish durably"); + this.publishWorkerPersistence(active, result); + await worker.providerEvents.retire(execution); + await worker.owner.release(execution); + if (active) this.requireWorkerFiles().retire(execution.threadId, execution.executionId, active.deliveryAttempt); + this.workerTurns.delete(execution.threadId); + return true; + } + + private async submitUnsentWorkerFinish( + worker: WorkerOwnedTurnRuntime, + execution: ExecutionIdentity, + active: WorkerOwnedTurn | undefined, + outcome: "errored" | "cancelled", + ): Promise { + try { + return await worker.owner.submit(execution, this.unsentFinishCommand(execution, active, outcome)); + } catch (error) { + await worker.recoverRejected(execution); + logger.error("Unsent worker turn recovered after rejected finish", { threadId: execution.threadId, + error: error instanceof Error ? error.message : String(error) }); + return null; + } + } + + private unsentFinishCommand( + execution: ExecutionIdentity, + active: WorkerOwnedTurn | undefined, + outcome: "errored" | "cancelled", + ): Extract { + const frozen = this.unsentFileEvidence(execution, active, outcome); + const providerId = active?.prepared.providerId + ?? this.runtimePersistence.load(execution.threadId)?.provider ?? "codex"; + return { + kind: "finish-from-state", outcome, + input: { ...execution, providerId, + providerIdentities: active?.prepared.parentStartInput.providerIdentities ?? [], + outcome, projection: { kind: "writer-staged" }, + ...(outcome === "errored" ? { error: "Provider turn failed before dispatch" } : {}), + ...(active ? { deliveryAttempt: active.deliveryAttempt } : {}) }, + ...(frozen ? { frozenFileEvidence: frozen } : {}), + }; + } + + private unsentFileEvidence( + execution: ExecutionIdentity, + active: WorkerOwnedTurn | undefined, + outcome: "errored" | "cancelled", + ): ReturnType { + return active ? this.requireWorkerFiles().seal({ ...execution, + deliveryAttempt: active.deliveryAttempt, outcome }) : null; + } + /** Start a prepared first-turn command and return its authoritative admission outcome. */ private async sendInitialMessageAndSnapshot( command: SendMessageCommand, @@ -715,6 +1150,10 @@ export class TurnRuntimeController implements TurnLifecycleControl, TurnRuntimeE */ private async stopSessionInternal(threadId: string): Promise { const prepared = this.prepareStop(threadId); + const owned = this.workerRuntime?.owner.current(threadId); + if (owned?.started && owned.execution.executionId === prepared.runtime.turnExecutionId) { + return this.stopWorkerSession(prepared); + } if (!isRunningRuntime(prepared.runtime)) return this.alreadyTerminal(prepared); this.finishStopCheckpoint(prepared); void this.stopProviderForTurn(prepared).catch((error: unknown) => { @@ -727,6 +1166,106 @@ export class TurnRuntimeController implements TurnLifecycleControl, TurnRuntimeE return this.finalizeStoppedTurn(prepared); } + private async stopWorkerSession(prepared: PreparedStop): Promise { + const executionId = prepared.runtime.turnExecutionId!; + const active = this.workerTurns.get(prepared.threadId); + if (!active || !this.turnRetryDispatchByThread.get(prepared.threadId)?.dispatchStarted) { + await this.finishUnsentWorkerTurn(prepared.threadId, executionId, "cancelled"); + this.turnRuntime.terminalize(prepared.threadId, executionId, "cancelled"); + this.trackSessionEnded(prepared.threadId, executionId); + this.disarmTurnRetryWindow(prepared.threadId); + return this.cancelledWorkerStop(prepared); + } + if (!isRunningRuntime(prepared.runtime)) { + await this.releaseWorkerTurn(active); + return this.alreadyTerminal(prepared); + } + active.stopInProgress = true; + this.requireWorkerFiles().fence(prepared.threadId, executionId, active.deliveryAttempt); + const drained = active.provider.fenceCanonicalTurnEvents({ ...active.execution, + deliveryAttempt: active.deliveryAttempt }, { discardQueued: true }); + void this.stopWorkerProvider(prepared, active.provider); + try { + return await this.completeWorkerStop(prepared, active, drained); + } catch (error) { + await this.requireWorkerRuntime().recoverRejected(active.execution); + throw error; + } finally { + active.stopInProgress = false; + } + } + + private async completeWorkerStop( + prepared: PreparedStop, + active: WorkerOwnedTurn, + drained: Promise, + ): Promise { + await drained; + if (active.terminalCommitted) { + active.stopInProgress = false; + await this.releaseWorkerTurn(active); + return this.alreadyTerminal(prepared); + } + if (!active.stopRequested) { + const stopped = await this.requireWorkerRuntime().owner.stop(active.execution, NodeCrypto.randomUUID()); + if (stopped.kind !== "committed") throw new Error("Worker stop request did not commit"); + active.stopRequested = true; + } + const frozen = this.requireWorkerFiles().seal({ ...active.execution, + deliveryAttempt: active.deliveryAttempt, outcome: "cancelled" }); + if (!frozen) throw new Error("Worker stop lost its file evidence generation"); + await this.finishWorkerStop(active, frozen); + return this.cancelledWorkerStop(prepared); + } + + private async finishWorkerStop( + active: WorkerOwnedTurn, + frozen: NonNullable>, + ): Promise { + const result = await this.requireWorkerRuntime().owner.submit(active.execution, { + kind: "finish-from-state", outcome: "cancelled", frozenFileEvidence: frozen, + input: { ...active.execution, providerId: active.prepared.providerId, + providerIdentities: active.prepared.parentStartInput.providerIdentities, + outcome: "cancelled", deliveryAttempt: active.deliveryAttempt, + projection: { kind: "writer-staged" } }, + }); + if (result.kind !== "committed") throw new Error("Worker stop terminal did not commit"); + this.publishWorkerPersistence(active, result); + active.terminalCommitted = true; + active.terminalOutcome = "cancelled"; + this.turnRuntime.terminalize(active.execution.threadId, active.execution.executionId, "cancelled"); + active.stopInProgress = false; + await this.releaseWorkerTurn(active); + } + + private cancelledWorkerStop(prepared: PreparedStop): AgentStopResult { + return { threadId: prepared.threadId, turnExecutionId: prepared.runtime.turnExecutionId, + snapshot: this.turnRuntime.snapshot(prepared.threadId) ?? prepared.runtime, + status: "cancelled", dispatchState: prepared.dispatchState }; + } + + private async stopWorkerProvider(prepared: PreparedStop, provider: CodexProviderBoundary): Promise { + void Promise.resolve(this.featureEffects.stopDescendants(prepared.threadId)).catch((error: unknown) => { + logger.warn("Worker-owned descendant stop failed", { threadId: prepared.threadId, + error: error instanceof Error ? error.message : String(error) }); + }); + const stop = Promise.resolve(provider.stopSession(prepared.sessionId)); + let timer: ReturnType | undefined; + try { + const timedOut = await Promise.race([ + stop.then(() => false), + new Promise((resolve) => { timer = setTimeout(() => resolve(true), PROVIDER_STOP_SETTLE_TIMEOUT_MS); }), + ]); + if (timedOut) this.evictUnresponsiveSession(prepared); + } catch (error) { + this.evictUnresponsiveSession(prepared); + logger.warn("Worker-owned provider stop failed", { threadId: prepared.threadId, + error: error instanceof Error ? error.message : String(error) }); + } finally { + clearTimeout(timer); + } + } + private prepareStop(threadId: string): PreparedStop { const reservationToken = this.activeMutationReservations.get(threadId); const runtime = this.turnRuntime.snapshot(threadId) ?? idleRuntime(threadId); @@ -1512,3 +2051,18 @@ export class TurnRuntimeController implements TurnLifecycleControl, TurnRuntimeE this.activeMutationReservations.clear(); } } + +function workerTerminalOutcome(event: AgentEvent): TurnOutcome | null { + if (event.type === "turnComplete") return "completed"; + if (event.type === "error") return "errored"; + if (event.type !== "ended" || event.outcome === undefined) return null; + return event.outcome === "cancelled" ? "interrupted" : event.outcome; +} + +function writerOwnedProviderExtension(extension: ProviderRuntimeEvent["extension"]): boolean { + return Boolean(extension?.child || extension?.collaboration || extension?.continuation); +} + +function matchesWorkerEvent(event: AgentEvent, batch: WorkerOwnedProviderEventBatch): boolean { + return event.threadId === batch.threadId && event.turnExecutionId === batch.executionId; +} diff --git a/apps/server/src/features/agents/turns/__tests__/turn-event-pipeline.test.ts b/apps/server/src/features/agents/turns/__tests__/turn-event-pipeline.test.ts index 0b3be33b6..0ebc8ea2c 100644 --- a/apps/server/src/features/agents/turns/__tests__/turn-event-pipeline.test.ts +++ b/apps/server/src/features/agents/turns/__tests__/turn-event-pipeline.test.ts @@ -8,9 +8,11 @@ import { type TurnEventApplication, type TurnEventIngressFence, type TurnLifecycleControl, + type WorkerFileObservationPort, } from "../turn-event-pipeline.js"; import type { ProviderEventIngressEvent } from "../../../providers/composition/provider-event-ingress.js"; import { PARENT_ASSISTANT_TEXT_RETAINED_LIMITS } from "../parent-assistant-text-checkpoint-service.js"; +import type { TurnDiffService } from "../turn-diff-service.js"; const EXECUTION_ID = "00000000-0000-4000-8000-000000000001"; @@ -57,31 +59,66 @@ function createPipeline( previousFileFinalization: TurnEventApplication["previousFileFinalization"] = () => undefined, rejectForQueueCapacity = vi.fn(), ingressFence?: TurnEventIngressFence, + workerFileObserver?: WorkerFileObservationPort, + turnDiffs?: Pick, ): { pipeline: TurnEventPipeline; finalize: ReturnType; rejectForQueueCapacity: ReturnType; publishCommitted: ReturnType; + observeFileMutation: ReturnType; } { const lifecycle: TurnLifecycleControl = { normalize: (event) => event, finalize, }; const publishCommitted = vi.fn(); + const observeFileMutation = vi.fn(); const application: TurnEventApplication = { apply, publishCommitted, - observeFileMutation: vi.fn(), + observeFileMutation, rejectForQueueCapacity, previousFileFinalization, beginResumedFileTracking: vi.fn(), observeToolUse: vi.fn(), observeToolResult: vi.fn(), }; - return { pipeline: new TurnEventPipeline(lifecycle, application, undefined, ingressFence), finalize, rejectForQueueCapacity, publishCommitted }; + return { pipeline: new TurnEventPipeline(lifecycle, application, turnDiffs, ingressFence, workerFileObserver), + finalize, rejectForQueueCapacity, publishCommitted, observeFileMutation }; } describe("TurnEventPipeline", () => { + it("claims worker file callbacks before legacy attribution and keeps native diff routing", () => { + const workerFileObserver: WorkerFileObservationPort = { + ownsFileMutation: vi.fn((event) => event.threadId === "worker-thread"), + capture: vi.fn(() => false), + }; + const push = vi.fn((): "accepted" => "accepted"); + const { pipeline, observeFileMutation } = createPipeline( + () => true, undefined, undefined, undefined, undefined, workerFileObserver, { push }, + ); + const mutation = { + threadId: "worker-thread", turnExecutionId: EXECUTION_ID, deliveryAttempt: 1, + toolCallId: "tool-1", toolName: "Edit", toolInput: { file_path: "tracked.txt" }, + }; + + pipeline.handleProviderFileMutation(mutation); + pipeline.handleProviderFileMutation({ ...mutation, deliveryAttempt: 0 }); + pipeline.handleProviderFileMutation({ ...mutation, threadId: "legacy-thread" }); + pipeline.handleProviderTurnDiff({ + turnId: "turn-1", turnExecutionId: EXECUTION_ID, + deliveryAttempt: 1, revision: 1, state: "invalidated", + }); + + expect(workerFileObserver.capture).toHaveBeenCalledTimes(2); + expect(observeFileMutation).toHaveBeenCalledExactlyOnceWith({ ...mutation, threadId: "legacy-thread" }); + expect(push).toHaveBeenCalledExactlyOnceWith({ + turnId: "turn-1", turnExecutionId: EXECUTION_ID, + deliveryAttempt: 1, revision: 1, state: "invalidated", + }); + }); + it("publishes projected worker output without legacy event application", () => { const apply = vi.fn(() => true); const { pipeline, publishCommitted } = createPipeline(apply); diff --git a/apps/server/src/features/agents/turns/turn-admission-dispatch-coordinator.ts b/apps/server/src/features/agents/turns/turn-admission-dispatch-coordinator.ts index 883724a69..3f5ea663c 100644 --- a/apps/server/src/features/agents/turns/turn-admission-dispatch-coordinator.ts +++ b/apps/server/src/features/agents/turns/turn-admission-dispatch-coordinator.ts @@ -133,6 +133,8 @@ export interface TurnParentStartOwner { readonly ownerEpoch: number; readonly providerId: string; readonly parentTurn: DataOnlyParentTurnStartInput; + readonly parentLive?: import("../execution/provider-execution-event-state.js").ExecutionParentStartContext; + readonly publishParentStart?: boolean; }): Promise; } @@ -436,18 +438,7 @@ export class TurnAdmissionDispatchCoordinator { const attachmentData = await this.persistAttachments(prepared); const sourceTurnId = prepared.command.sourceTurnId ?? NodeCrypto.randomUUID(); const parentStartInput = this.prepareParentTurnStartInput(prepared, lease, sourceTurnId, attachmentData, review); - if (this.parentStartOwner) { - await this.parentStartOwner.start({ - execution: { threadId: lease.threadId, turnId: sourceTurnId, executionId: lease.turnExecutionId }, - ownerEpoch: lease.generation, - providerId: prepared.providerId, - parentTurn: parentStartInput, - }); - if (parentStartInput.reopenThread) { - const reopened = this.threads.findById(lease.threadId); - if (reopened) broadcast("thread.lifecycleChanged", { thread: reopened }); - } - } else this.startParentTurn(parentStartInput); + await this.commitParentStart(prepared, lease, sourceTurnId, parentStartInput); this.publishCommittedEffects(prepared, sourceTurnId); const wirePayload = this.buildWirePayload(prepared); const request = await this.buildTurnRequest(prepared, lease, sourceTurnId, attachmentData, cwd, wirePayload, review); @@ -463,7 +454,7 @@ export class TurnAdmissionDispatchCoordinator { threadControl: this.threadControlDirective(prepared, sourceTurnId), contextSeed: prepared.thread.last_context_tokens ?? 0, contextWindow: prepared.thread.context_window, - ...(this.parentStartOwner ? { workerOwned: true as const } : {}), + ...(this.parentStartOwner && prepared.providerId === "codex" ? { workerOwned: true as const } : {}), }; } @@ -730,6 +721,31 @@ export class TurnAdmissionDispatchCoordinator { if (reopenedThread) broadcast("thread.lifecycleChanged", { thread: reopenedThread }); } + private async commitParentStart( + prepared: PreparedCommand, + lease: TurnRuntimeLease, + sourceTurnId: string, + input: DataOnlyParentTurnStartInput, + ): Promise { + if (!this.parentStartOwner || prepared.providerId !== "codex") return this.startParentTurn(input); + const precedingMessageId = input.userMessage.messageId; + if (!precedingMessageId) throw new Error("Worker-owned turn needs its committed user message identity"); + const planFeature = prepared.command.planAction === "revise" ? "output" + : this.effectiveInteractionMode(prepared.command) === "plan" ? "questions" : "none"; + await this.parentStartOwner.start({ + execution: { threadId: lease.threadId, turnId: sourceTurnId, executionId: lease.turnExecutionId }, + ownerEpoch: lease.generation, + providerId: prepared.providerId, + parentTurn: input, + parentLive: { planFeature, precedingMessageId }, + publishParentStart: prepared.providerId === "codex", + }); + if (input.reopenThread) { + const reopened = this.threads.findById(lease.threadId); + if (reopened) broadcast("thread.lifecycleChanged", { thread: reopened }); + } + } + private prepareParentTurnStartInput( prepared: PreparedCommand, lease: TurnRuntimeLease, diff --git a/apps/server/src/features/agents/turns/turn-event-pipeline.ts b/apps/server/src/features/agents/turns/turn-event-pipeline.ts index 58e5b0d8d..eeca23996 100644 --- a/apps/server/src/features/agents/turns/turn-event-pipeline.ts +++ b/apps/server/src/features/agents/turns/turn-event-pipeline.ts @@ -62,6 +62,12 @@ export interface TurnEventIngressFence { waitForThread(threadId: string): Promise; } +/** Claims every worker-owned file callback, including one rejected by its attempt fence. */ +export interface WorkerFileObservationPort { + ownsFileMutation(event: ProviderFileMutationStart): boolean; + capture(event: ProviderFileMutationStart): boolean; +} + interface QueuedTurnEvent { input: ProviderEventIngressEvent; event: AgentEvent; @@ -93,6 +99,7 @@ export class TurnEventPipeline implements ProviderEventIngressConsumer { private readonly application: TurnEventApplication, private readonly turnDiffs?: Pick, private readonly ingressFence?: TurnEventIngressFence, + private readonly workerFileObserver?: WorkerFileObservationPort, ) {} /** Accept an ingress envelope and preserve its source receipt through the turn queue. */ @@ -118,6 +125,10 @@ export class TurnEventPipeline implements ProviderEventIngressConsumer { /** Observe one provider file mutation before public event attribution is available. */ handleProviderFileMutation(event: ProviderFileMutationStart): void { + if (this.workerFileObserver?.ownsFileMutation(event)) { + this.workerFileObserver.capture(event); + return; + } this.application.observeFileMutation(event); } diff --git a/apps/server/src/features/providers/adapters/claude/__tests__/claude-canonical-event-publisher.test.ts b/apps/server/src/features/providers/adapters/claude/__tests__/claude-canonical-event-publisher.test.ts index 732554da8..c545ec739 100644 --- a/apps/server/src/features/providers/adapters/claude/__tests__/claude-canonical-event-publisher.test.ts +++ b/apps/server/src/features/providers/adapters/claude/__tests__/claude-canonical-event-publisher.test.ts @@ -96,4 +96,22 @@ describe("ClaudeCanonicalEventPublisher", () => { `claude:${routing.executionId}:attempt:1:event:3`, ]); }); + + it("checks a completed turn without resetting sequence for late SDK events", async () => { + const submit = vi.fn<(batch: ProviderEventBatch) => Promise>().mockResolvedValue(undefined); + const publisher = new ClaudeCanonicalEventPublisher(createSink(submit)); + const event = providerRuntimeEvent({ + type: AgentEventType.System, + threadId: routing.threadId, + subtype: "late-hook", + turnExecutionId: routing.executionId, + }); + + publisher.publish(routing, event, []); + await publisher.flushForExecution(routing); + publisher.publish(routing, event, []); + await publisher.waitForExecution(routing); + + expect(submit.mock.calls.map(([batch]) => batch.events[0]?.sourceSequence)).toEqual([1, 2]); + }); }); diff --git a/apps/server/src/features/providers/adapters/claude/__tests__/claude-provider-result-error.test.ts b/apps/server/src/features/providers/adapters/claude/__tests__/claude-provider-result-error.test.ts index 1d98d92d0..e4967a1cb 100644 --- a/apps/server/src/features/providers/adapters/claude/__tests__/claude-provider-result-error.test.ts +++ b/apps/server/src/features/providers/adapters/claude/__tests__/claude-provider-result-error.test.ts @@ -291,4 +291,67 @@ describe("ClaudeProvider result is_error handling (#293)", () => { expect(submittedEvents.map((runtimeEvent) => runtimeEvent.event)) .not.toContainEqual(expect.objectContaining({ type: AgentEventType.TurnComplete })); }); + + it("reports canonical sink failure for the exact execution without direct event delivery", async () => { + const failure = vi.fn(async () => undefined); + const directEvents = vi.fn(); + provider = new ClaudeProvider( + stubEnvService(), + stubJobObject(), + undefined, + undefined, + undefined, + { + ...mockProviderHost(), + events: { submit: vi.fn(async () => { throw new Error("writer unavailable"); }) }, + }, + ); + provider.setCanonicalTurnDeliveryFailureHandler(failure); + provider.on("event", directEvents); + let releaseStream!: () => void; + const holdStream = new Promise((resolve) => { releaseStream = resolve; }); + const streamDone = vi.fn(); + provider.on("_streamDone:mcode-thread-failed", streamDone); + mockQuery.mockImplementation(({ prompt }: { prompt: AsyncIterable }) => Object.assign( + (async function* () { + await prompt[Symbol.asyncIterator]().next(); + yield { type: "system", subtype: "init", session_id: "sdk-abc" }; + yield { type: "result", is_error: true, errors: ["rate_limit_exceeded"] }; + await holdStream; + })(), + { ...queryMethodStubs(), close: vi.fn() }, + )); + + try { + await provider.sendTurn({ + turnId: "turn-failed", + turnExecutionId: "execution-failed", + deliveryAttempt: 2, + sessionId: "mcode-thread-failed", + workspaceId: "workspace-1", + threadId: "thread-failed", + message: "hi", + cwd: "/tmp", + model: "claude-sonnet-4-6", + permissionMode: "default", + interactionMode: "build", + providerOptions: {}, + }); + + await vi.waitFor(() => expect(failure).toHaveBeenCalledExactlyOnceWith( + { + threadId: "thread-failed", + turnId: "turn-failed", + executionId: "execution-failed", + deliveryAttempt: 2, + }, + expect.objectContaining({ message: "writer unavailable" }), + )); + expect(streamDone).not.toHaveBeenCalled(); + expect(directEvents).not.toHaveBeenCalled(); + } finally { + releaseStream(); + } + await vi.waitFor(() => expect(streamDone).toHaveBeenCalledOnce()); + }); }); diff --git a/apps/server/src/features/providers/adapters/claude/claude-provider.ts b/apps/server/src/features/providers/adapters/claude/claude-provider.ts index d787202c6..690c1eadd 100644 --- a/apps/server/src/features/providers/adapters/claude/claude-provider.ts +++ b/apps/server/src/features/providers/adapters/claude/claude-provider.ts @@ -509,6 +509,10 @@ export class ClaudeProvider /** Serializes canonical event submission when the adapter runs in the server composition. */ private readonly canonicalEventPublisher: ClaudeCanonicalEventPublisher | undefined; + private canonicalTurnDeliveryFailureHandler: + | ((routing: ClaudeCanonicalEventRouting, error: Error) => Promise) + | undefined; + private readonly reportedCanonicalDeliveryFailures = new WeakSet(); /** * Tail of the most recent stderr emitted by each session's Claude Code * subprocess, capped at {@link STDERR_CAPTURE_LIMIT}. The SDK's process-exit @@ -601,6 +605,13 @@ export class ClaudeProvider }); } + /** Reports canonical sink failure to the owner of the exact active turn. */ + setCanonicalTurnDeliveryFailureHandler( + handler: (routing: ClaudeCanonicalEventRouting, error: Error) => Promise, + ): void { + this.canonicalTurnDeliveryFailureHandler = handler; + } + /** * Merges {@link EnvService.getEnv} into `process.env` for the Claude SDK spawn window * only, then restores the previous environment. @@ -2054,7 +2065,12 @@ export class ClaudeProvider if (executionId && executionId !== state.currentTurnExecutionId) state.pendingPromptExecutionIds.push(executionId); }); - void this.consumeClaudeStream(sessionId, q, routing, state); + void this.consumeClaudeStream(sessionId, q, routing, state).catch((error: unknown) => { + logger.error("Claude stream finalization failed", { + executionId: state.currentTurnExecutionId, + error: error instanceof Error ? error.message : String(error), + }); + }); } private createClaudeStreamState( @@ -2151,6 +2167,7 @@ export class ClaudeProvider } const outcome = await state.mapper.map(message); if (outcome !== "none") { + await this.flushCanonicalExecution(state.currentTurnExecutionId); state.awaitingResume = outcome === "turn_complete"; state.resumedTurnStarted = false; } @@ -2410,15 +2427,43 @@ export class ClaudeProvider try { await this.canonicalEventPublisher.waitForExecution(routing); } catch (error: unknown) { - logger.error("Claude canonical event delivery failed", { - executionId, - error: error instanceof Error ? error.message : String(error), - }); + await this.reportCanonicalDeliveryFailure(routing, error); } finally { this.getCanonicalRoutings().delete(executionId); } } + private async flushCanonicalExecution(executionId: string): Promise { + const routing = this.getCanonicalRoutings().get(executionId); + if (!routing || !this.canonicalEventPublisher) return; + try { + await this.canonicalEventPublisher.flushForExecution(routing); + } catch (error: unknown) { + await this.reportCanonicalDeliveryFailure(routing, error); + } + } + + private async reportCanonicalDeliveryFailure( + routing: ClaudeCanonicalEventRouting, + error: unknown, + ): Promise { + if (this.reportedCanonicalDeliveryFailures.has(routing)) return; + this.reportedCanonicalDeliveryFailures.add(routing); + const deliveryError = error instanceof Error ? error : new Error(String(error)); + logger.error("Claude canonical event delivery failed", { + executionId: routing.executionId, + error: deliveryError.message, + }); + try { + await this.canonicalTurnDeliveryFailureHandler?.(routing, deliveryError); + } catch (handlerError: unknown) { + logger.error("Claude canonical failure handler failed", { + executionId: routing.executionId, + error: handlerError instanceof Error ? handlerError.message : String(handlerError), + }); + } + } + private getCanonicalRoutings(): Map { this.canonicalRoutings ??= new Map(); return this.canonicalRoutings; diff --git a/apps/server/src/features/providers/catalog/__tests__/provider-catalog-service.test.ts b/apps/server/src/features/providers/catalog/__tests__/provider-catalog-service.test.ts index b0d5505f0..6fc680344 100644 --- a/apps/server/src/features/providers/catalog/__tests__/provider-catalog-service.test.ts +++ b/apps/server/src/features/providers/catalog/__tests__/provider-catalog-service.test.ts @@ -1,12 +1,15 @@ import "reflect-metadata"; +import * as NodeFS from "node:fs"; +import * as NodeOS from "node:os"; +import * as NodePath from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; -import type { Database } from "bun:sqlite"; +import { Database } from "bun:sqlite"; import type { ProviderCatalogChange, ProviderCatalogRequest, ProviderCatalogSnapshot, } from "@mcode/contracts"; -import { openMemoryDatabase } from "../../../../runtime/persistence/sqlite/database.js"; +import { openDatabase, openMemoryDatabase } from "../../../../runtime/persistence/sqlite/database.js"; import { ProviderCatalogSnapshotRepo } from "../persistence/provider-catalog-snapshot-repo.js"; import { ProviderCatalogService, @@ -48,6 +51,46 @@ describe("ProviderCatalogService", () => { return { repo, service: new ProviderCatalogService(repo) }; } + it("keeps a background refresh failure contained while another SQLite writer holds the database", async () => { + const directory = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "mcode-catalog-busy-")); + const path = NodePath.join(directory, "app.sqlite"); + db = openDatabase({ dbPath: path }); + db.run("PRAGMA busy_timeout = 1"); + db.prepare("INSERT INTO workspaces (id, name, path) VALUES (?, ?, ?)") + .run("workspace-1", "Workspace 1", "C:/repo"); + const blocker = new Database(path, { strict: true }); + const repo = new ProviderCatalogSnapshotRepo(db); + const service = new ProviderCatalogService(repo); + const upsert = vi.spyOn(repo, "upsert"); + const changes: ProviderCatalogChange[] = []; + service.onChanged((change) => changes.push(change)); + const input = { + request: REQUEST, context: CACHED.context, cwd: "C:/repo", refresh: async () => CACHED, + }; + let locked = false; + + try { + blocker.run("BEGIN IMMEDIATE"); + locked = true; + service.request(input); + await vi.waitFor(() => expect(upsert).toHaveBeenCalledTimes(1)); + await new Promise((resolve) => setImmediate(resolve)); + expect(changes).toHaveLength(0); + + blocker.run("ROLLBACK"); + locked = false; + service.request(input); + await vi.waitFor(() => expect(changes).toHaveLength(1)); + expect(repo.get(providerCatalogContextKey(REQUEST, "C:/repo"))).toEqual(CACHED); + } finally { + if (locked) blocker.run("ROLLBACK"); + blocker.close(true); + db.close(true); + db = undefined; + NodeFS.rmSync(directory, { recursive: true, force: true }); + } + }); + it("returns a stale persisted snapshot before background refresh completes", async () => { const { repo, service } = createService(); const key = providerCatalogContextKey(REQUEST, "C:/repo"); diff --git a/apps/server/src/features/providers/catalog/persistence/provider-catalog-snapshot-repo.ts b/apps/server/src/features/providers/catalog/persistence/provider-catalog-snapshot-repo.ts index 3666bc85e..5fdca191c 100644 --- a/apps/server/src/features/providers/catalog/persistence/provider-catalog-snapshot-repo.ts +++ b/apps/server/src/features/providers/catalog/persistence/provider-catalog-snapshot-repo.ts @@ -51,6 +51,7 @@ export class ProviderCatalogSnapshotRepo { snapshot: ProviderCatalogSnapshot, ): boolean { const validated = ProviderCatalogSnapshotSchema().parse(snapshot); + // Reading the workspace before acquiring a write lock can fail a later WAL snapshot upgrade. return this.orm.transaction((tx) => { // The legacy INSERT...SELECT only wrote when the referenced workspace // still existed, so a dangling workspaceId must abort with no changes. @@ -101,6 +102,6 @@ export class ProviderCatalogSnapshotRepo { ) .run(); return true; - }); + }, { behavior: "immediate" }); } } diff --git a/apps/server/src/features/providers/catalog/provider-catalog-service.ts b/apps/server/src/features/providers/catalog/provider-catalog-service.ts index 8a03adcaa..333b827bf 100644 --- a/apps/server/src/features/providers/catalog/provider-catalog-service.ts +++ b/apps/server/src/features/providers/catalog/provider-catalog-service.ts @@ -327,11 +327,22 @@ export class ProviderCatalogService { pendingSubscribers: new Map(), }; this.inflight.set(persistenceKey, job); - void new Promise((resolve) => { - setImmediate(() => { - void this.refresh(persistenceKey, job).finally(resolve); + setImmediate(() => { + void this.runRefresh(persistenceKey, job).catch((error: unknown) => { + logger.warn("Provider catalog background refresh could not persist", { + providerId: input.request.providerId, + workspaceId: input.request.workspaceId, + error: error instanceof Error ? error.message : String(error), + }); }); - }).finally(() => { + }); + return true; + } + + private async runRefresh(persistenceKey: string, job: CatalogRefreshJob): Promise { + try { + await this.refresh(persistenceKey, job); + } finally { this.inflight.delete(persistenceKey); for (const [pendingRequestKey, pending] of job.pendingSubscribers) { const persisted = this.snapshotRepo.get(persistenceKey); @@ -348,8 +359,7 @@ export class ProviderCatalogService { false, ); } - }); - return true; + } } private emitChange(change: ProviderCatalogChange): void { diff --git a/apps/server/src/features/providers/composition/canonical-live-event-publisher.ts b/apps/server/src/features/providers/composition/canonical-live-event-publisher.ts index 8b87b2105..2a5b3c3a8 100644 --- a/apps/server/src/features/providers/composition/canonical-live-event-publisher.ts +++ b/apps/server/src/features/providers/composition/canonical-live-event-publisher.ts @@ -69,16 +69,24 @@ export class CanonicalLiveEventPublisher { }); } - /** Waits for all queued drafts, then reports a sink failure to the caller. */ - async waitForExecution(routing: CanonicalLiveEventRouting): Promise { + /** Checks delivery so a pooled provider can report turn failure before stream teardown. */ + async flushForExecution(routing: CanonicalLiveEventRouting): Promise { const key = this.queueKey(routing); const queue = this.queues.get(key); if (!queue) return; await queue.tail; - this.queues.delete(key); if (queue.failure) throw queue.failure; } + /** Waits for all queued drafts and retires the execution queue. */ + async waitForExecution(routing: CanonicalLiveEventRouting): Promise { + try { + await this.flushForExecution(routing); + } finally { + this.queues.delete(this.queueKey(routing)); + } + } + private queueFor(routing: CanonicalLiveEventRouting): ProviderExecutionQueue { const key = this.queueKey(routing); const existing = this.queues.get(key); diff --git a/apps/server/src/features/providers/composition/provider-host-ports.ts b/apps/server/src/features/providers/composition/provider-host-ports.ts index aee2205bf..400e9ebf3 100644 --- a/apps/server/src/features/providers/composition/provider-host-ports.ts +++ b/apps/server/src/features/providers/composition/provider-host-ports.ts @@ -45,7 +45,7 @@ export type ProviderEventOwnershipRoute = /** Selects one commit owner for every batch of an exact execution. */ export interface ProviderEventOwnership { - resolve(executionId: string): ProviderEventOwnershipRoute; + resolve(executionId: string, sourceProviderId?: string): ProviderEventOwnershipRoute; } /** Server services used to compose the narrow Provider host-port boundary. */ @@ -131,7 +131,9 @@ async function submitProviderEvents( dependencies: ProviderHostPortDependencies, batch: ProviderEventBatch, ): Promise { - const route: ProviderEventOwnershipRoute = dependencies.eventOwnership?.resolve(batch.executionId) + const route: ProviderEventOwnershipRoute = dependencies.eventOwnership?.resolve( + batch.executionId, batch.events[0]?.sourceProviderId, + ) ?? { kind: "legacy" }; if (route.kind === "rejected") throw new Error("Canonical event execution is no longer admitted"); if (route.kind === "worker") return await submitWorkerEvents(dependencies, route, batch); diff --git a/apps/server/src/features/providers/composition/register-providers.ts b/apps/server/src/features/providers/composition/register-providers.ts index 6f7011140..abf34f656 100644 --- a/apps/server/src/features/providers/composition/register-providers.ts +++ b/apps/server/src/features/providers/composition/register-providers.ts @@ -9,6 +9,7 @@ import { createProviderHostPorts } from "./provider-host-ports.js"; import { BrowserAutomationSessionLease } from "../../browser-automation/index.js"; import { InternalThreadControlMcpRuntime } from "../../thread-control/index.js"; import { CanonicalAgentBoundary } from "../../agents/index.js"; +import { WorkerOwnedTurnRuntime } from "../../agents/execution/worker-owned-turn-runtime.js"; import { ScopedPreGrantService } from "../../agents/permissions/scoped-pre-grant.js"; import { EnvService } from "../../../runtime/environment/env-service.js"; import type { JobObject } from "../../../runtime/process/containment/job-object.js"; @@ -90,6 +91,7 @@ export function registerProviderAdapters(container: DependencyContainer): void { grants: c.resolve(ScopedPreGrantService), events: c.resolve(CanonicalAgentBoundary), ingress: c.resolve(ProviderEventIngress), + eventOwnership: c.resolve(WorkerOwnedTurnRuntime).providerEvents, }), }); } diff --git a/apps/server/src/features/terminal/cleanup/__tests__/terminal-cleanup-ledger.test.ts b/apps/server/src/features/terminal/cleanup/__tests__/terminal-cleanup-ledger.test.ts index 22465569e..f4818d297 100644 --- a/apps/server/src/features/terminal/cleanup/__tests__/terminal-cleanup-ledger.test.ts +++ b/apps/server/src/features/terminal/cleanup/__tests__/terminal-cleanup-ledger.test.ts @@ -23,16 +23,19 @@ async function openLedgerTestDatabase(): Promise { `); return { prepare: (sql: string) => database.prepare(sql), - transaction: (action: () => Result) => () => { - database.exec("BEGIN"); - try { - const result = action(); - database.exec("COMMIT"); - return result; - } catch (error) { - database.exec("ROLLBACK"); - throw error; - } + transaction: (action: () => Result) => { + const run = (begin: "BEGIN" | "BEGIN IMMEDIATE") => { + database.exec(begin); + try { + const result = action(); + database.exec("COMMIT"); + return result; + } catch (error) { + database.exec("ROLLBACK"); + throw error; + } + }; + return Object.assign(() => run("BEGIN"), { immediate: () => run("BEGIN IMMEDIATE") }); }, close: () => database.close(), } as unknown as Database; diff --git a/apps/server/src/features/terminal/cleanup/terminal-cleanup-ledger.ts b/apps/server/src/features/terminal/cleanup/terminal-cleanup-ledger.ts index 84cb37f80..a96382daf 100644 --- a/apps/server/src/features/terminal/cleanup/terminal-cleanup-ledger.ts +++ b/apps/server/src/features/terminal/cleanup/terminal-cleanup-ledger.ts @@ -50,6 +50,8 @@ export class PtyHostCleanupLedger implements PtyHostCleanupLedgerStore { /** Adds one process identity or refreshes the matching generation record. */ record(record: PtyHostCleanupRecord): void { const parsed = validateRecord(record); + // Reserve the write lock before reading; a deferred read can lose its WAL snapshot + // when the execution writer commits between this ledger's SELECT and INSERT. this.db.transaction(() => { const existing = this.db .prepare( @@ -108,7 +110,7 @@ export class PtyHostCleanupLedger implements PtyHostCleanupLedgerStore { now, now, ); - })(); + }).immediate(); } /** Removes one record only when its session and host generation match. */ diff --git a/apps/web/src/__tests__/threadStore-turn-persist.test.ts b/apps/web/src/__tests__/threadStore-turn-persist.test.ts index 27894a2e3..48feaf163 100644 --- a/apps/web/src/__tests__/threadStore-turn-persist.test.ts +++ b/apps/web/src/__tests__/threadStore-turn-persist.test.ts @@ -450,6 +450,38 @@ describe("handleTurnPersisted", () => { expect(readThreadField(THREAD_ID, (record) => record.streaming)).toBe(""); }); + it("settles a stopped turn before any assistant message was written", () => { + const user = createMockMessage({ + id: "user-stop", + thread_id: THREAD_ID, + role: "user", + content: "Stop this turn", + }); + useThreadStore.setState({ + records: seedThreadRecord(THREAD_ID, { + messages: [user], + runtimePhase: "finalizing", + turnExecutionId: "execution-early-stop", + awaitingUserStopPersist: true, + }), + runningThreadIds: new Set([THREAD_ID]), + }); + + useThreadStore.getState().handleTurnPersisted({ + threadId: THREAD_ID, + messageId: null, + toolCallCount: 0, + filesChanged: [], + outcome: "cancelled", + executionId: "execution-early-stop", + }); + + expect(readThreadField(THREAD_ID, (record) => record.messages)).toEqual([user]); + expect(readThreadField(THREAD_ID, (record) => record.awaitingUserStopPersist)).toBeUndefined(); + expect(readThreadField(THREAD_ID, (record) => record.runtimePhase)).toBe("cancelled"); + expect(useThreadStore.getState().runningThreadIds.has(THREAD_ID)).toBe(false); + }); + it("materializes buffered streaming text when a terminal runtime snapshot arrives without a terminal event", () => { useThreadStore.setState({ records: seedThreadRecord(THREAD_ID, { diff --git a/apps/web/src/stores/threadStore.ts b/apps/web/src/stores/threadStore.ts index 4267ffc4a..a95f13b02 100644 --- a/apps/web/src/stores/threadStore.ts +++ b/apps/web/src/stores/threadStore.ts @@ -247,7 +247,7 @@ interface ThreadState { /** Handle server-side tool call persistence confirmation. */ handleTurnPersisted: (payload: { threadId: string; - messageId: string; + messageId: string | null; turnId?: string | null; executionId?: string | null; outcome?: TurnOutcome | null; @@ -2578,6 +2578,16 @@ export const useThreadStore = create((zustandSet, get) => { currentThreadId: string | null, terminalPhase: ThreadRecord["runtimePhase"] | undefined, ): Partial => { + if (payload.messageId === null) { + return { + ...turnPersistStopState(record, payload), + ...(terminalPhase ? terminalRuntimePatch(record, terminalPhase) : {}), + ...(payload.fileEffects && payload.turnId === record.fileEffectTurnId + && payload.fileEffects.revision >= record.fileEffectSummary.revision + ? { fileEffectSummary: payload.fileEffects } + : {}), + }; + } const localMessageId = resolveTurnPersistLocalMessageId(record, payload.messageId); const messages = persistedTurnMessages(record, payload, localMessageId); const stopState = turnPersistStopState(record, payload); diff --git a/apps/web/src/transport/ws-events.ts b/apps/web/src/transport/ws-events.ts index 1e9cba13e..2a8c4f556 100644 --- a/apps/web/src/transport/ws-events.ts +++ b/apps/web/src/transport/ws-events.ts @@ -468,16 +468,9 @@ export function startPushListeners(): void { // turn.persisted: server has persisted tool calls for a completed turn unsubs.push( pushEmitter.on("turn.persisted", (data) => { - const payload = data as { - threadId: string; - turnId?: string | null; - executionId?: string | null; - messageId: string; - outcome?: "completed" | "cancelled" | "interrupted" | "errored" | null; - toolCallCount: number; - filesChanged: string[]; - fileEffects?: TurnFileEffectSummary; - }; + const parsed = WS_CHANNELS["turn.persisted"].safeParse(data); + if (!parsed.success) return; + const payload = parsed.data; useThreadStore.getState().handleTurnPersisted(payload); refreshTurnSnapshotsAfterPersist(payload.threadId, payload.filesChanged); diff --git a/docs/internals/narrative-pipeline.md b/docs/internals/narrative-pipeline.md index 227437eca..88635c1a6 100644 --- a/docs/internals/narrative-pipeline.md +++ b/docs/internals/narrative-pipeline.md @@ -128,21 +128,12 @@ Provider-native event Provider runtime event keeps native evidence separate from AgentEvent data │ ▼ -Provider ingress selects a thread-affine worker for cloneable validation - │ - ▼ -Provider ingress queues accepted results fairly and selects a provider adapter - │ - ▼ -Adapter forwards a provider-neutral AgentEvent or consumes private provider work - │ - ▼ -Turn event pipeline ToolUse handler → narrative-turn-state.ts bufferToolCall - (writes to the per-turn tool-call buffer for later persist) - │ - ▼ -Turn event pipeline enriches missing parentToolCallId via - narrativeStore.getCurrentParentToolCallId (agentCallStack fallback) +Provider event ownership selects the admitted execution's route + ├─ Worker-owned: ordered task mailbox → parent event and file reduction + │ → single execution writer commits event, turn effects, and publication receipt + │ → server releases the acknowledged AgentEvent + └─ Legacy: canonical commit → provider ingress and adapter + → turn event pipeline and parent turn application │ ▼ broadcast("agent.event", enrichedEvent) diff --git a/docs/internals/provider-architecture.md b/docs/internals/provider-architecture.md index a0d45c2df..eb9431815 100644 --- a/docs/internals/provider-architecture.md +++ b/docs/internals/provider-architecture.md @@ -72,32 +72,33 @@ A provider emits a `ProviderRuntimeEvent`. Its `event` is provider-neutral data that may reach the renderer. Its optional extension contains provider-native evidence that must not reach the renderer. -The server accepts runtime events in this order: - -```text -Provider runtime event → provider ingress → event worker → provider adapter → turn event pipeline → AgentEvent -``` - -Ingress validates the provider identity and sends cloneable event validation to -a fixed worker pool. A thread stays on one worker until its queued work drains; -the pool limits concurrent work and returns results in that thread's order. -Ingress then queues accepted results fairly across threads and preserves the -receipt when it came from a canonical commit. It waits for both the worker and -the ingress queue before finalizing a turn. The worker does not own the -database: canonical commits and turn state still run in the server process. -An adapter may forward a generic event, consume private provider work, or -reject malformed native evidence with a diagnostic. Only a forwarded -`AgentEvent` enters narration, lifecycle, and -renderer publication. +An admitted worker-owned execution sends its canonical provider drafts to the +execution's ordered mailbox. One of a fixed number of workers owns that task +for the execution lifetime. It reduces parent events and file observations, +then awaits the single execution SQLite writer for the event, turn state, and +publication receipt. The server publishes the committed result without +applying that event again through `ProviderTurnEventApplication`. Stop, +terminalization, and late events use the same execution fence. The mailbox has +bounded capacity and schedules tasks fairly when they share a worker. +Worker ownership currently admits user-dispatched Codex turns. Claude, Cursor, +and provider-originated resumes still use the legacy route. + +Executions without a worker owner still use provider ingress, cloneable event +preprocessing, the provider adapter, and `TurnEventPipeline`. That route +preserves the existing canonical receipt and finalization fence. A provider +adapter may forward a generic event, consume private native evidence, or +reject malformed evidence with a diagnostic. Only a forwarded `AgentEvent` +reaches the renderer. Codex collaboration evidence uses a Codex adapter. Claude, Cursor, and Copilot send generic runtime events and do not invoke that adapter. A new provider adds an adapter only when it has private native evidence that requires server-side projection. -Canonical commits hand their committed runtime envelopes directly to ingress. -The resulting receipt means durable acceptance or queueing. It never means that -the event was published to the renderer. +Canonical commits on the legacy route hand their committed runtime envelopes +to ingress. A worker-owned receipt instead covers the parent event and its +turn effects together. Neither receipt alone proves the renderer displayed the +event. `AgentService` owns provider selection and narrow parent-turn durability. It does not import a concrete provider, a provider adapter, or a canonical diff --git a/packages/contracts/src/ws/channels.ts b/packages/contracts/src/ws/channels.ts index d6999c550..1a7ad67cc 100644 --- a/packages/contracts/src/ws/channels.ts +++ b/packages/contracts/src/ws/channels.ts @@ -163,7 +163,7 @@ export const WS_CHANNELS = { "turn.persisted": z.object({ threadId: z.string(), turnId: z.string().nullable().optional(), - messageId: z.string(), + messageId: z.string().nullable(), toolCallCount: z.number(), filesChanged: z.array(z.string()), fileEffects: TurnFileEffectSummarySchema().optional(), diff --git a/packages/providers/src/__tests__/codex/codex-canonical-event-publisher.test.ts b/packages/providers/src/__tests__/codex/codex-canonical-event-publisher.test.ts new file mode 100644 index 000000000..a869ff653 --- /dev/null +++ b/packages/providers/src/__tests__/codex/codex-canonical-event-publisher.test.ts @@ -0,0 +1,188 @@ +import { describe, expect, it, vi } from "vitest"; +import { AgentEventRoutingSchema } from "@mcode/agent-model"; +import { AgentEventType, providerRuntimeEvent } from "@mcode/contracts"; +import type { ProviderEventBatch, ProviderEventSinkPort, ProviderEventSubmissionReceipt } from "../../host-ports.js"; +import { + CodexCanonicalEventPublisher, + type CodexCanonicalEventRouting, +} from "../../private/codex/codex-canonical-event-publisher.js"; + +const routing: CodexCanonicalEventRouting = { + threadId: "thread-1", + turnId: "turn-1", + executionId: "00000000-0000-4000-8000-000000000001", + deliveryAttempt: 1, +}; + +const acceptedReceipt: ProviderEventSubmissionReceipt = { + commit: { + outcome: "committed", + conversationRevision: 1, + rosterRevision: 1, + acceptedThrough: 1, + durableThrough: 1, + eventCount: 1, + }, + delivery: { ingress: "queued" }, +}; + +function sink(submit: ProviderEventSinkPort["submit"]): ProviderEventSinkPort { + return { submit }; +} + +describe("CodexCanonicalEventPublisher", () => { + it("submits exact ordered drafts with stable batch, item and event IDs", async () => { + const submit = vi.fn<(batch: ProviderEventBatch) => Promise>().mockResolvedValue(acceptedReceipt); + const publisher = new CodexCanonicalEventPublisher(sink(submit)); + + publisher.publish(routing, { + ...providerRuntimeEvent({ + type: AgentEventType.TextDelta, + threadId: routing.threadId, + turnExecutionId: routing.executionId, + delta: "Hello", + }), + deliveryAttempt: 1, + }); + publisher.publish(routing, { + ...providerRuntimeEvent({ + type: AgentEventType.Ended, + threadId: routing.threadId, + turnExecutionId: routing.executionId, + }), + deliveryAttempt: 1, + }); + + await publisher.waitForExecution(routing); + expect(submit).toHaveBeenCalledTimes(2); + const [textBatch, endedBatch] = submit.mock.calls.map(([batch]) => batch); + expect(textBatch).toMatchObject({ + threadId: routing.threadId, + turnId: routing.turnId, + executionId: routing.executionId, + deliveryAttempt: 1, + phase: "running", + batchId: `codex:${routing.executionId}:attempt:1:event:1`, + events: [{ + eventId: `codex:${routing.executionId}:attempt:1:event:1`, + sourceProviderId: "codex", + sourceSequence: 1, + ingestClass: "volatile", + }], + }); + expect(textBatch.events[0]?.routing.itemId).toBe(`codex:${routing.executionId}:attempt:1:item:1`); + AgentEventRoutingSchema.parse(textBatch.events[0]?.routing); + expect(endedBatch.events[0]?.sourceSequence).toBe(2); + expect(endedBatch.events[0]?.payload).toMatchObject({ + type: "item.recorded", + item: { payload: { projection: "providerRuntimeEvent", runtimeEvent: { + deliveryAttempt: 1, + event: { type: AgentEventType.Ended, turnExecutionId: routing.executionId }, + } } }, + }); + }); + + it("uses a distinct sequence for a retry of the same execution", async () => { + const submit = vi.fn<(batch: ProviderEventBatch) => Promise>().mockResolvedValue(acceptedReceipt); + const publisher = new CodexCanonicalEventPublisher(sink(submit)); + const event = providerRuntimeEvent({ type: AgentEventType.System, threadId: routing.threadId, subtype: "notice" }); + + publisher.publish(routing, event); + await publisher.waitForExecution(routing); + const retry = { ...routing, deliveryAttempt: 2 }; + publisher.publish(retry, { ...event, deliveryAttempt: 2 }); + await publisher.waitForExecution(retry); + + expect(submit.mock.calls.map(([batch]) => batch.batchId)).toEqual([ + `codex:${routing.executionId}:attempt:1:event:1`, + `codex:${routing.executionId}:attempt:2:event:1`, + ]); + }); + + it("continues the sequence for an event after the first delivery drain", async () => { + const submit = vi.fn<(batch: ProviderEventBatch) => Promise>().mockResolvedValue(acceptedReceipt); + const publisher = new CodexCanonicalEventPublisher(sink(submit)); + const event = providerRuntimeEvent({ type: AgentEventType.System, threadId: routing.threadId, subtype: "notice" }); + + publisher.publish(routing, event); + await publisher.waitForExecution(routing); + publisher.publish(routing, event); + await publisher.retireExecution(routing); + + expect(submit.mock.calls.map(([batch]) => batch.batchId)).toEqual([ + `codex:${routing.executionId}:attempt:1:event:1`, + `codex:${routing.executionId}:attempt:1:event:2`, + ]); + }); + + it("discards queued events at Stop while awaiting the current sink call", async () => { + let completeFirst!: (receipt: ProviderEventSubmissionReceipt) => void; + const submit = vi.fn<(batch: ProviderEventBatch) => Promise>() + .mockImplementation(() => new Promise((resolve) => { completeFirst = resolve; })); + const publisher = new CodexCanonicalEventPublisher(sink(submit)); + const onFailure = vi.fn(); + publisher.setFailureHandler(onFailure); + const event = providerRuntimeEvent({ type: AgentEventType.System, threadId: routing.threadId, subtype: "notice" }); + + publisher.publish(routing, event); + await vi.waitFor(() => expect(submit).toHaveBeenCalledOnce()); + for (let index = 0; index < 40; index++) publisher.publish(routing, event); + const drained = publisher.discardQueuedForExecution(routing); + publisher.publish(routing, event); + completeFirst(acceptedReceipt); + await drained; + + expect(submit).toHaveBeenCalledOnce(); + expect(onFailure).not.toHaveBeenCalled(); + await publisher.retireExecution(routing); + }); + + it("stops submitting later events after the sink fails", async () => { + const submit = vi.fn<(batch: ProviderEventBatch) => Promise>() + .mockRejectedValueOnce(new Error("canonical sink unavailable")); + const publisher = new CodexCanonicalEventPublisher(sink(submit)); + const event = providerRuntimeEvent({ type: AgentEventType.System, threadId: routing.threadId, subtype: "notice" }); + + publisher.publish(routing, event); + publisher.publish(routing, event); + + await expect(publisher.waitForExecution(routing)).rejects.toThrow("canonical sink unavailable"); + expect(submit).toHaveBeenCalledTimes(1); + }); + + it("reports the first sink failure for the exact attempt before a delivery drain", async () => { + let rejectSubmit!: (error: Error) => void; + const submit = vi.fn<(batch: ProviderEventBatch) => Promise>() + .mockImplementation(() => new Promise((_, reject) => { rejectSubmit = reject; })); + const publisher = new CodexCanonicalEventPublisher(sink(submit)); + const onFailure = vi.fn(async () => undefined); + publisher.setFailureHandler(onFailure); + const event = providerRuntimeEvent({ type: AgentEventType.System, threadId: routing.threadId, subtype: "notice" }); + + publisher.publish(routing, event); + await vi.waitFor(() => expect(submit).toHaveBeenCalledOnce()); + rejectSubmit(new Error("writer unavailable")); + await vi.waitFor(() => expect(onFailure).toHaveBeenCalledExactlyOnceWith( + routing, + expect.objectContaining({ message: "writer unavailable" }), + )); + + publisher.publish(routing, event); + await expect(publisher.waitForExecution(routing)).rejects.toThrow("writer unavailable"); + expect(submit).toHaveBeenCalledOnce(); + expect(onFailure).toHaveBeenCalledOnce(); + }); + + it.each(["conflict", "ingest-overflow"] as const)("rejects a resolved %s receipt", async (outcome) => { + const submit = vi.fn<(batch: ProviderEventBatch) => Promise>() + .mockResolvedValue({ ...acceptedReceipt, commit: { ...acceptedReceipt.commit, outcome } }); + const publisher = new CodexCanonicalEventPublisher(sink(submit)); + const event = providerRuntimeEvent({ type: AgentEventType.System, threadId: routing.threadId, subtype: "notice" }); + + publisher.publish(routing, event); + publisher.publish(routing, event); + + await expect(publisher.waitForExecution(routing)).rejects.toThrow(outcome); + expect(submit).toHaveBeenCalledTimes(1); + }); +}); diff --git a/packages/providers/src/__tests__/codex/codex-provider-first-turn.test.ts b/packages/providers/src/__tests__/codex/codex-provider-first-turn.test.ts index 7cb192d16..5f897c873 100644 --- a/packages/providers/src/__tests__/codex/codex-provider-first-turn.test.ts +++ b/packages/providers/src/__tests__/codex/codex-provider-first-turn.test.ts @@ -82,8 +82,16 @@ vi.mock("../../private/codex/codex-app-server.js", async () => { import { BrowserAutomationSessionLease, CodexProvider, stubEnvService } from "./codex-provider-test-fixture.js"; import { AgentEventSchema, AgentEventType } from "@mcode/contracts"; import type { ProviderFileMutationStart, ProviderRuntimeEvent, ProviderTurnDiffUpdate } from "@mcode/contracts"; +import type { ProviderEventBatch, ProviderEventSinkPort, ProviderEventSubmissionReceipt } from "../../host-ports.js"; const schemaValidExecutionId = "00000000-0000-4000-8000-000000000001"; +const acceptedEventReceipt: ProviderEventSubmissionReceipt = { + commit: { + outcome: "committed", conversationRevision: 1, rosterRevision: 1, + acceptedThrough: 1, durableThrough: 1, eventCount: 1, + }, + delivery: { ingress: "queued" }, +}; function makeProvider( catalogService: { @@ -108,6 +116,7 @@ function makeProvider( createCodexConfiguration?: () => Promise; close?: (sessionId: string) => Promise; } = undefined as never, + eventSink?: ProviderEventSinkPort, ): CodexProvider { return new CodexProvider( { get: async () => ({ provider: { cli: { codex: "codex" } } }) } as never, @@ -116,6 +125,7 @@ function makeProvider( catalogService as never, browserAutomationLease, threadControlMcp as never, + eventSink, ); } @@ -642,6 +652,177 @@ describe("CodexProvider first turn on new session", () => { await ended; }); + it("submits an attempt-bound parent event through the canonical sink only when enabled", async () => { + const submit = vi.fn<(batch: ProviderEventBatch) => Promise>().mockResolvedValue(acceptedEventReceipt); + const provider = makeProvider(undefined, new BrowserAutomationSessionLease(), undefined, { submit }); + const legacy: ProviderRuntimeEvent[] = []; + provider.on("event", (event: ProviderRuntimeEvent) => legacy.push(event)); + provider.setCanonicalTurnEventDeliveryEnabled(true); + + await provider.sendTurn({ + turnId: "test-turn", + turnExecutionId: schemaValidExecutionId, + deliveryAttempt: 2, + sessionId, + workspaceId: "workspace-test", + threadId, + message: "hey", + cwd: process.cwd(), + model: "gpt-5.4", + interactionMode: "build", + providerOptions: {}, + permissionMode: "auto", + }); + await new Promise((resolve) => setImmediate(resolve)); + const server = appServers[0]!; + server.emit("notification", { + method: "item/agentMessage/delta", + params: { threadId: "sdk-thread-1", turnId: "turn-test-id", delta: "Hello" }, + }); + await vi.waitFor(() => expect(submit).toHaveBeenCalled()); + + const batches = submit.mock.calls.map(([batch]) => batch); + expect(batches.some((batch) => batch.events.some((event) => event.payload.type === "item.recorded" + && event.payload.item.payload.projection === "providerRuntimeEvent" + && event.payload.item.payload.runtimeEvent.event.type === AgentEventType.TextDelta))).toBe(true); + expect(batches.every((batch) => batch.deliveryAttempt === 2 && batch.turnId === "test-turn")).toBe(true); + expect(legacy.some((event) => event.event.type === AgentEventType.TextDelta)).toBe(false); + }); + + it("keeps Codex on the existing event channel until canonical delivery is enabled", async () => { + const submit = vi.fn<(batch: ProviderEventBatch) => Promise>().mockResolvedValue(acceptedEventReceipt); + const provider = makeProvider(undefined, new BrowserAutomationSessionLease(), undefined, { submit }); + const legacy: ProviderRuntimeEvent[] = []; + provider.on("event", (event: ProviderRuntimeEvent) => legacy.push(event)); + + await provider.sendTurn({ + turnId: "test-turn", + turnExecutionId: schemaValidExecutionId, + deliveryAttempt: 2, + sessionId, + workspaceId: "workspace-test", + threadId, + message: "hey", + cwd: process.cwd(), + model: "gpt-5.4", + interactionMode: "build", + providerOptions: {}, + permissionMode: "auto", + }); + await new Promise((resolve) => setImmediate(resolve)); + appServers[0]!.emit("notification", { + method: "item/agentMessage/delta", + params: { threadId: "sdk-thread-1", turnId: "turn-test-id", delta: "Hello" }, + }); + + expect(legacy.some((event) => event.event.type === AgentEventType.TextDelta)).toBe(true); + expect(submit).not.toHaveBeenCalled(); + }); + + it("fences duplicate parent Ended callbacks for an owned attempt", async () => { + const submit = vi.fn<(batch: ProviderEventBatch) => Promise>() + .mockResolvedValue(acceptedEventReceipt); + const provider = makeProvider(undefined, new BrowserAutomationSessionLease(), undefined, { submit }); + const legacy: ProviderRuntimeEvent[] = []; + provider.on("event", (event: ProviderRuntimeEvent) => legacy.push(event)); + provider.setCanonicalTurnEventDeliveryEnabled(true); + await provider.sendTurn({ + turnId: "test-turn", turnExecutionId: schemaValidExecutionId, deliveryAttempt: 2, + sessionId, workspaceId: "workspace-test", threadId, message: "hey", cwd: process.cwd(), + model: "gpt-5.4", interactionMode: "build", providerOptions: {}, permissionMode: "auto", + }); + const emit = (provider as unknown as { emitRuntimeEvent(event: ProviderRuntimeEvent): void }).emitRuntimeEvent.bind(provider); + const ended: ProviderRuntimeEvent = { event: { type: AgentEventType.Ended, + threadId, turnExecutionId: schemaValidExecutionId, outcome: "completed" } }; + emit(ended); + emit(ended); + await provider.waitForCanonicalTurnEvents({ + threadId, turnId: "test-turn", executionId: schemaValidExecutionId, deliveryAttempt: 2, + }); + + expect(submit.mock.calls.flatMap(([batch]) => batch.events).filter((event) => + event.payload.type === "item.recorded" + && event.payload.item.payload.projection === "providerRuntimeEvent" + && event.payload.item.payload.runtimeEvent.event.type === AgentEventType.Ended)).toHaveLength(1); + expect(legacy.some((event) => event.event.type === AgentEventType.Ended)).toBe(false); + }); + + it("stamps the admitted attempt on an early parent event before native turn binding", async () => { + let resolveNativeTurn!: (turnId: string) => void; + sendTurnMock.mockImplementationOnce(() => new Promise((resolve) => { resolveNativeTurn = resolve; })); + const submit = vi.fn<(batch: ProviderEventBatch) => Promise>().mockResolvedValue(acceptedEventReceipt); + const provider = makeProvider(undefined, new BrowserAutomationSessionLease(), undefined, { submit }); + const legacy: ProviderRuntimeEvent[] = []; + provider.on("event", (event: ProviderRuntimeEvent) => legacy.push(event)); + provider.setCanonicalTurnEventDeliveryEnabled(true); + + await provider.sendTurn({ + turnId: "test-turn", + turnExecutionId: schemaValidExecutionId, + deliveryAttempt: 3, + sessionId, + workspaceId: "workspace-test", + threadId, + message: "hey", + cwd: process.cwd(), + model: "gpt-5.4", + interactionMode: "build", + providerOptions: {}, + permissionMode: "auto", + }); + await vi.waitFor(() => expect(sendTurnMock).toHaveBeenCalled()); + const server = appServers[0]!; + server.emit("notification", { + method: "item/agentMessage/delta", + params: { threadId: "sdk-thread-1", turnId: "early-native-turn", delta: "Early" }, + }); + await vi.waitFor(() => expect(submit).toHaveBeenCalled()); + expect(submit.mock.calls[0]?.[0]).toMatchObject({ + deliveryAttempt: 3, + events: [{ payload: { item: { payload: { runtimeEvent: { + deliveryAttempt: 3, + event: { type: AgentEventType.TextDelta, turnExecutionId: schemaValidExecutionId }, + } } } } }], + }); + expect(legacy.some((event) => event.event.type === AgentEventType.TextDelta)).toBe(false); + + resolveNativeTurn("early-native-turn"); + server.emit("notification", { + method: "turn/completed", + params: { threadId: "sdk-thread-1", turn: { id: "early-native-turn", status: "completed" } }, + }); + await provider.waitForCanonicalTurnEvents({ + threadId, turnId: "test-turn", executionId: schemaValidExecutionId, deliveryAttempt: 3, + }); + }); + + it("fences new parent events synchronously while draining already queued batches", async () => { + const submit = vi.fn<(batch: ProviderEventBatch) => Promise>().mockResolvedValue(acceptedEventReceipt); + const provider = makeProvider(undefined, new BrowserAutomationSessionLease(), undefined, { submit }); + const legacy: ProviderRuntimeEvent[] = []; + provider.on("event", (event: ProviderRuntimeEvent) => legacy.push(event)); + provider.setCanonicalTurnEventDeliveryEnabled(true); + await provider.sendTurn({ + turnId: "test-turn", turnExecutionId: schemaValidExecutionId, deliveryAttempt: 2, + sessionId, workspaceId: "workspace-test", threadId, message: "hey", cwd: process.cwd(), + model: "gpt-5.4", interactionMode: "build", providerOptions: {}, permissionMode: "auto", + }); + await new Promise((resolve) => setImmediate(resolve)); + const emitText = (delta: string) => appServers[0]!.emit("notification", { + method: "item/agentMessage/delta", + params: { threadId: "sdk-thread-1", turnId: "turn-test-id", delta }, + }); + emitText("Before stop"); + const drained = provider.fenceCanonicalTurnEvents({ + threadId, turnId: "test-turn", executionId: schemaValidExecutionId, deliveryAttempt: 2, + }); + emitText("After stop"); + await drained; + + expect(submit).toHaveBeenCalledTimes(1); + expect(legacy.some((event) => event.event.type === AgentEventType.TextDelta)).toBe(false); + }); + it("cancels the main turn and reuses its app-server for the next turn", async () => { const provider = makeProvider(); const events: ProviderRuntimeEvent[] = []; diff --git a/packages/providers/src/__tests__/codex/codex-provider-test-fixture.ts b/packages/providers/src/__tests__/codex/codex-provider-test-fixture.ts index 6fbd27220..0d3564489 100644 --- a/packages/providers/src/__tests__/codex/codex-provider-test-fixture.ts +++ b/packages/providers/src/__tests__/codex/codex-provider-test-fixture.ts @@ -5,6 +5,7 @@ import type { ProviderBrowserLeaseGrant, ProviderBrowserLeaseHandle, ProviderBrowserLeaseRequest, + ProviderEventSinkPort, ProviderHostPorts, } from "../../host-ports.js"; import type { TurnRequest } from "@mcode/contracts"; @@ -167,6 +168,7 @@ export class CodexProvider extends PackageCodexProvider { createCodexConfiguration?: (sessionId: string) => Promise; close?: (sessionId: string) => Promise; }, + eventSink?: ProviderEventSinkPort, ) { const delegatedSettingsService = { get: () => settings.get() }; const host: ProviderHostPorts = { @@ -186,7 +188,7 @@ export class CodexProvider extends PackageCodexProvider { close: (sessionId) => threadControl?.close?.(sessionId) ?? Promise.resolve(), }, grants: { consume: () => false }, - events: { submit: async () => undefined }, + events: eventSink ?? { submit: async () => undefined }, }; const codexPorts: CodexProviderPorts = { settings: { diff --git a/packages/providers/src/factory-types.ts b/packages/providers/src/factory-types.ts index 3f17e105c..53dedd5f8 100644 --- a/packages/providers/src/factory-types.ts +++ b/packages/providers/src/factory-types.ts @@ -6,6 +6,7 @@ import type { StoredAttachment, } from "@mcode/contracts"; import type { ProviderHostPorts } from "./host-ports.js"; +import type { CodexCanonicalEventRouting } from "./private/codex/codex-canonical-event-publisher.js"; /** Configuration validated by every inert Provider factory. */ export interface ProviderFactoryConfiguration { @@ -63,7 +64,13 @@ export interface ProviderBoundary { } /** Usable Codex Provider returned by its public factory. */ -export type CodexProviderBoundary = IAgentProvider & ProviderBoundary; +export type CodexProviderBoundary = IAgentProvider & ProviderBoundary & { + setCanonicalTurnEventDeliveryEnabled(enabled: boolean): void; + setCanonicalTurnDeliveryFailureHandler(handler: (routing: CodexCanonicalEventRouting, error: Error) => void | Promise): void; + fenceCanonicalTurnEvents(routing: CodexCanonicalEventRouting, options?: { discardQueued?: boolean }): Promise; + waitForCanonicalTurnEvents(routing: CodexCanonicalEventRouting): Promise; + retireCanonicalTurnEvents(routing: CodexCanonicalEventRouting): Promise; +}; /** Usable Cursor Provider returned by its public factory. */ export type CursorProviderBoundary = IAgentProvider & ProviderBoundary & { diff --git a/packages/providers/src/index.ts b/packages/providers/src/index.ts index 8f47a6442..75c035dbb 100644 --- a/packages/providers/src/index.ts +++ b/packages/providers/src/index.ts @@ -16,6 +16,7 @@ export type { ProviderFactoryConfiguration, ProviderFactoryInput, } from "./factory-types.js"; +export type { CodexCanonicalEventRouting } from "./private/codex/codex-canonical-event-publisher.js"; export type { ProviderBrowserLeaseHandle, ProviderBrowserLeaseGrant, diff --git a/packages/providers/src/private/codex/codex-canonical-event-publisher.ts b/packages/providers/src/private/codex/codex-canonical-event-publisher.ts new file mode 100644 index 000000000..273636666 --- /dev/null +++ b/packages/providers/src/private/codex/codex-canonical-event-publisher.ts @@ -0,0 +1,166 @@ +import { AgentEventType } from "@mcode/contracts"; +import type { ProviderRuntimeEvent } from "@mcode/contracts"; +import { logger } from "@mcode/shared"; +import type { ProviderEventDraft, ProviderEventSinkPort } from "../../host-ports.js"; + +const MAX_PENDING_EVENTS_PER_EXECUTION = 1_024; + +/** Exact Mcode turn and delivery attempt that produced a Codex event. */ +export interface CodexCanonicalEventRouting { + threadId: string; + turnId: string; + executionId: string; + deliveryAttempt: number; +} + +interface ExecutionQueue { + nextSourceSequence: number; + pendingEventCount: number; + tail: Promise; + failure: Error | undefined; + discardQueued: boolean; +} + +/** Serializes Codex parent events through the server-owned canonical sink. */ +export class CodexCanonicalEventPublisher { + private readonly queues = new Map(); + private failureHandler: + | ((routing: CodexCanonicalEventRouting, error: Error) => void | Promise) + | undefined; + + constructor(private readonly sink: ProviderEventSinkPort) {} + + /** Reports the first failed delivery for an exact execution without waiting for its next turn. */ + setFailureHandler( + handler: (routing: CodexCanonicalEventRouting, error: Error) => void | Promise, + ): void { + this.failureHandler = handler; + } + + /** Queues one event; the caller must have verified its exact execution and attempt. */ + publish(routing: CodexCanonicalEventRouting, runtimeEvent: ProviderRuntimeEvent): void { + const queue = this.queueFor(routing); + if (queue.failure || queue.discardQueued) return; + if (queue.pendingEventCount >= MAX_PENDING_EVENTS_PER_EXECUTION) { + this.reportFailure(routing, queue, new Error(`Codex canonical event queue overflowed for execution ${routing.executionId}`)); + return; + } + + const sourceSequence = queue.nextSourceSequence++; + queue.pendingEventCount++; + const draft = this.createDraft(routing, runtimeEvent, sourceSequence); + queue.tail = queue.tail + .then(async () => { + if (queue.failure || queue.discardQueued) return; + const receipt = await this.sink.submit({ + threadId: routing.threadId, + turnId: routing.turnId, + executionId: routing.executionId, + batchId: draft.eventId, + deliveryAttempt: routing.deliveryAttempt, + phase: "running", + events: [draft], + }); + if (receipt.commit.outcome === "conflict" || receipt.commit.outcome === "ingest-overflow") { + throw new Error(`Codex canonical event ${draft.eventId} was ${receipt.commit.outcome}`); + } + }) + .catch((error: unknown) => { this.reportFailure(routing, queue, toError(error)); }) + .finally(() => { queue.pendingEventCount--; }); + } + + /** Waits for acknowledged delivery and exposes a failed or overflowing queue. */ + async waitForExecution(routing: CodexCanonicalEventRouting): Promise { + const key = this.queueKey(routing); + const queue = this.queues.get(key); + if (!queue) return; + await queue.tail; + if (queue.failure) throw queue.failure; + } + + /** Stop queued deliveries at a cancellation cut, then await only the current sink call. */ + async discardQueuedForExecution(routing: CodexCanonicalEventRouting): Promise { + const queue = this.queues.get(this.queueKey(routing)); + if (queue) queue.discardQueued = true; + await this.waitForExecution(routing); + } + + /** Releases a completed route after its final acknowledged event. */ + async retireExecution(routing: CodexCanonicalEventRouting): Promise { + await this.waitForExecution(routing); + this.queues.delete(this.queueKey(routing)); + } + + private queueFor(routing: CodexCanonicalEventRouting): ExecutionQueue { + const key = this.queueKey(routing); + const existing = this.queues.get(key); + if (existing) return existing; + const queue: ExecutionQueue = { + nextSourceSequence: 1, + pendingEventCount: 0, + tail: Promise.resolve(), + failure: undefined, + discardQueued: false, + }; + this.queues.set(key, queue); + return queue; + } + + private queueKey(routing: CodexCanonicalEventRouting): string { + return `${routing.executionId}:attempt:${routing.deliveryAttempt}`; + } + + private reportFailure(routing: CodexCanonicalEventRouting, queue: ExecutionQueue, error: Error): void { + if (queue.failure) return; + queue.failure = error; + try { + void Promise.resolve(this.failureHandler?.(routing, error)).catch((handlerError: unknown) => { + logger.error("Codex canonical failure handler failed", { + executionId: routing.executionId, + error: toError(handlerError).message, + }); + }); + } catch (handlerError: unknown) { + logger.error("Codex canonical failure handler failed", { + executionId: routing.executionId, + error: toError(handlerError).message, + }); + } + } + + private createDraft( + routing: CodexCanonicalEventRouting, + runtimeEvent: ProviderRuntimeEvent, + sourceSequence: number, + ): ProviderEventDraft { + const eventId = `codex:${routing.executionId}:attempt:${routing.deliveryAttempt}:event:${sourceSequence}`; + const itemId = `codex:${routing.executionId}:attempt:${routing.deliveryAttempt}:item:${sourceSequence}`; + const timestamp = new Date().toISOString(); + return { + eventId, + routing: { threadId: routing.threadId, turnId: routing.turnId, executionId: routing.executionId, itemId }, + sourceProviderId: "codex", + sourceIdentities: [], + sourceSequence, + providerTimestamp: timestamp, + ...(runtimeEvent.event.type === AgentEventType.TextDelta ? { ingestClass: "volatile" as const } : {}), + payload: { + type: "item.recorded", + item: { + id: itemId, + threadId: routing.threadId, + turnId: routing.turnId, + kind: "system", + providerIdentities: [], + payload: { projection: "providerRuntimeEvent", runtimeEvent }, + createdAt: timestamp, + updatedAt: timestamp, + }, + }, + }; + } +} + +function toError(error: unknown): Error { + return error instanceof Error ? error : new Error(String(error)); +} diff --git a/packages/providers/src/private/codex/codex-provider.ts b/packages/providers/src/private/codex/codex-provider.ts index b51fd0eb1..dd1121903 100644 --- a/packages/providers/src/private/codex/codex-provider.ts +++ b/packages/providers/src/private/codex/codex-provider.ts @@ -55,6 +55,10 @@ import { checkCodexVersion, meetsMinVersion } from "./codex-version.js"; import { CodexAppServer, warmCodexAppServer } from "./codex-app-server.js"; import type { CodexApprovalRequest } from "./codex-app-server.js"; import { CodexEventMapper } from "./codex-event-mapper.js"; +import { + CodexCanonicalEventPublisher, + type CodexCanonicalEventRouting, +} from "./codex-canonical-event-publisher.js"; import { buildCodexInput, hasCodexInternalThreadControlMcp, @@ -631,9 +635,68 @@ export class CodexProvider extends NodeEvents.EventEmitter implements IAgentProv } private emitRuntimeEvent(runtimeEvent: ProviderRuntimeEvent): void { + const state = this.canonicalTurnRoutingsByThread.get(runtimeEvent.event.threadId); + const routing = state?.routing; + if (routing && runtimeEvent.extension?.child === undefined + && runtimeEvent.event.turnExecutionId === routing.executionId) { + if (state.kind === "fenced") return; + if (runtimeEvent.deliveryAttempt !== undefined && runtimeEvent.deliveryAttempt !== routing.deliveryAttempt) return; + this.canonicalEventPublisher.publish(routing, { + ...runtimeEvent, + deliveryAttempt: routing.deliveryAttempt, + }); + // Ended closes this exact parent attempt; late SDK callbacks must stay fenced. + if (runtimeEvent.event.type === AgentEventType.Ended) state.kind = "fenced"; + return; + } super.emit("event", runtimeEvent); } + private readonly canonicalEventPublisher: CodexCanonicalEventPublisher; + private readonly canonicalTurnRoutingsByThread = new Map(); + private canonicalTurnEventDeliveryEnabled = false; + + /** Enables the server-owned event route for newly dispatched Codex parent turns. */ + setCanonicalTurnEventDeliveryEnabled(enabled: boolean): void { + this.canonicalTurnEventDeliveryEnabled = enabled; + } + + /** Report a failed canonical sink promptly for the exact turn attempt. */ + setCanonicalTurnDeliveryFailureHandler( + handler: (routing: CodexCanonicalEventRouting, error: Error) => void | Promise, + ): void { + this.canonicalEventPublisher.setFailureHandler(handler); + } + + /** Reports acknowledged canonical delivery or its failure for one exact attempt. */ + waitForCanonicalTurnEvents(routing: CodexCanonicalEventRouting): Promise { + return this.canonicalEventPublisher.waitForExecution(routing); + } + + /** Stops admission for one exact parent attempt before draining its queued batches. */ + fenceCanonicalTurnEvents(routing: CodexCanonicalEventRouting, options?: { discardQueued?: boolean }): Promise { + const state = this.canonicalTurnRoutingsByThread.get(routing.threadId); + if (!state || state.routing.turnId !== routing.turnId + || state.routing.executionId !== routing.executionId + || state.routing.deliveryAttempt !== routing.deliveryAttempt) return Promise.resolve(); + this.canonicalTurnRoutingsByThread.set(routing.threadId, { kind: "fenced", routing: state.routing }); + return options?.discardQueued + ? this.canonicalEventPublisher.discardQueuedForExecution(routing) + : this.canonicalEventPublisher.waitForExecution(routing); + } + + /** Drop the drained publisher queue while retaining a fence against late SDK events. */ + retireCanonicalTurnEvents(routing: CodexCanonicalEventRouting): Promise { + const state = this.canonicalTurnRoutingsByThread.get(routing.threadId); + if (state?.kind !== "fenced" || state.routing.turnId !== routing.turnId + || state.routing.executionId !== routing.executionId + || state.routing.deliveryAttempt !== routing.deliveryAttempt) return Promise.resolve(); + return this.canonicalEventPublisher.retireExecution(routing); + } + /** Owns the session pool, idle eviction (with busy guard), and JobObject/kill. */ private readonly runtime: SessionRuntime; private sdkSessionIds = new Map(); @@ -687,6 +750,7 @@ export class CodexProvider extends NodeEvents.EventEmitter implements IAgentProv private readonly stopDrainReuseTimeoutMs: number = STOP_DRAIN_REUSE_TIMEOUT_MS, ) { super(); + this.canonicalEventPublisher = new CodexCanonicalEventPublisher(host.events); this.runtime = new SessionRuntime(this, { jobObject: { isWindowsJob: false, @@ -1048,6 +1112,16 @@ export class CodexProvider extends NodeEvents.EventEmitter implements IAgentProv async sendTurn(req: TurnRequest<"codex">): Promise { const preparingSession = this.runtime.get(req.sessionId); if (preparingSession) preparingSession.nextTurnExecutionId = req.turnExecutionId; + if (this.canonicalTurnEventDeliveryEnabled) { + const threadId = this.threadIdForSession(req.sessionId); + await this.retirePreviousCanonicalTurn(threadId); + this.canonicalTurnRoutingsByThread.set(threadId, { kind: "active", routing: { + threadId, + turnId: req.turnId, + executionId: req.turnExecutionId, + deliveryAttempt: req.deliveryAttempt ?? 1, + } }); + } const turn = await this.prepareCodexTurn(req); if (!turn) return; if (this.consumePendingCodexStop(turn)) return; @@ -1175,6 +1249,16 @@ export class CodexProvider extends NodeEvents.EventEmitter implements IAgentProv this.pendingBrowserAccess.set(sessionId, { stage, workspaceId: request.workspaceId, permissionCapability: turn.browserPermissionCapability }); } + private async retirePreviousCanonicalTurn(threadId: string): Promise { + const state = this.canonicalTurnRoutingsByThread.get(threadId); + if (!state) return; + this.canonicalTurnRoutingsByThread.set(threadId, { kind: "fenced", routing: state.routing }); + await this.canonicalEventPublisher.retireExecution(state.routing); + if (this.canonicalTurnRoutingsByThread.get(threadId)?.routing === state.routing) { + this.canonicalTurnRoutingsByThread.delete(threadId); + } + } + private async acquireCodexTurn(turn: PreparedCodexTurn): Promise { const { request, threadId } = turn; try { @@ -2881,6 +2965,7 @@ export class CodexProvider extends NodeEvents.EventEmitter implements IAgentProv this.pendingBrowserAccess.clear(); this.liveSessionIds.clear(); this.activeCodexServers.clear(); + this.canonicalTurnRoutingsByThread.clear(); logger.info("CodexProvider shutdown complete"); } } diff --git a/packages/providers/src/private/cursor/__tests__/cursor-session-continuity.test.ts b/packages/providers/src/private/cursor/__tests__/cursor-session-continuity.test.ts index ffafeadb1..ed6afc771 100644 --- a/packages/providers/src/private/cursor/__tests__/cursor-session-continuity.test.ts +++ b/packages/providers/src/private/cursor/__tests__/cursor-session-continuity.test.ts @@ -1,7 +1,7 @@ import * as NodeEvents from "node:events"; import type * as NodeChildProcess from "node:child_process"; import type { ClientSideConnection, SessionNotification } from "@agentclientprotocol/sdk"; -import { getDefaultSettings, type TurnRequest } from "@mcode/contracts"; +import { AgentEventType, getDefaultSettings, type TurnRequest } from "@mcode/contracts"; import { describe, expect, it, vi } from "vitest"; import type { ProviderHostPorts } from "../../../host-ports.js"; import { AcpSessionRuntime, SessionRecoveryFailedError } from "../../protocols/acp/acp-session-runtime.js"; @@ -125,6 +125,54 @@ function submittedRuntimeEvents(host: ProviderHostPorts): unknown[] { } describe("CursorProvider session continuity", () => { + it("delivers the turn through the canonical host without EventEmitter duplicates", async () => { + const host = createHost(); + const fake = createFakeRuntime("cursor-session-1", 101); + const start = vi.spyOn(AcpSessionRuntime, "start").mockResolvedValue(fake.runtime); + const provider = new CursorProvider(host, { + settings: { get: () => getDefaultSettings() }, + skills: { list: () => [] }, + }, 60_000); + const directEvents = vi.fn(); + provider.on("event", directEvents); + + try { + await provider.sendTurn(turn("prompt", "execution-1")); + + expect(submittedRuntimeEvents(host)).toEqual([ + expect.objectContaining({ type: AgentEventType.System, turnExecutionId: "execution-1" }), + expect.objectContaining({ type: AgentEventType.TurnComplete, turnExecutionId: "execution-1" }), + expect.objectContaining({ type: AgentEventType.Ended, turnExecutionId: "execution-1" }), + ]); + expect(directEvents).not.toHaveBeenCalled(); + } finally { + start.mockRestore(); + await (provider as unknown as { runtime: { shutdown(): Promise } }).runtime.shutdown(); + } + }); + + it("rejects the turn when canonical host delivery fails", async () => { + const host = createHost(); + host.events.submit = vi.fn(async () => { throw new Error("writer unavailable"); }); + const fake = createFakeRuntime("cursor-session-1", 101); + const start = vi.spyOn(AcpSessionRuntime, "start").mockResolvedValue(fake.runtime); + const provider = new CursorProvider(host, { + settings: { get: () => getDefaultSettings() }, + skills: { list: () => [] }, + }, 60_000); + const directEvents = vi.fn(); + provider.on("event", directEvents); + + try { + await expect(provider.sendTurn(turn("prompt", "execution-1"))) + .rejects.toThrow("writer unavailable"); + expect(directEvents).not.toHaveBeenCalled(); + } finally { + start.mockRestore(); + await (provider as unknown as { runtime: { shutdown(): Promise } }).runtime.shutdown(); + } + }); + it("reuses a healthy ACP child for normal turns and replaces it only for an explicit fresh session", async () => { const host = createHost(); const first = createFakeRuntime("cursor-session-1", 101); diff --git a/scripts/perf/seven-thread-live-harness.mjs b/scripts/perf/seven-thread-live-harness.mjs index fe2e14ff3..11a884758 100644 --- a/scripts/perf/seven-thread-live-harness.mjs +++ b/scripts/perf/seven-thread-live-harness.mjs @@ -31,6 +31,8 @@ const TERMINAL_CONTROL_COUNT = 1; export const WORKLOAD_MODEL = "gpt-5.6-luna"; const PROVIDER_ID = "codex"; const HEALTH_SAMPLE_INTERVAL_MS = 250; +const MAX_WORKER_QUEUE_SAMPLES = 512; +const MAX_THROUGHPUT_INTERVALS = 600; const EVENT_LOOP_INTERVAL_MS = 50; const TURN_TIMEOUT_MS = 120_000; const CONTROL_RPC_TIMEOUT_MS = 90_000; @@ -325,9 +327,9 @@ function createLiveHarnessContext(repoRoot, label, stopOne) { const receipt = createReceipt(run, label, stopOne); const eventLoop = new EventLoopStallSampler(EVENT_LOOP_INTERVAL_MS); const serverStalls = new ServerLogStallReader(paths.logsDir); - const healthSampler = new HealthSampler(ports.healthUrl, receipt.metrics.health); + const healthSampler = new HealthSampler(ports.healthUrl, receipt.metrics.health, receipt.metrics.workerQueue); const eventState = new Map(); - const terminalEvents = new TerminalEventWaiter(eventState, stopOne); + const terminalEvents = new TerminalEventWaiter(eventState, stopOne, () => receipt.state.stop); const turnStarts = new TurnStartWaiter(eventState); return { repoRoot, @@ -378,6 +380,7 @@ async function prepareLiveHarness(context) { const { ports, receipt } = context; const initialHealth = await sampleHealth(ports.healthUrl); receipt.metrics.health.samples.push(initialHealth.durationMs); + recordWorkerQueueSample(receipt.metrics.workerQueue, initialHealth.workerQueue); assertHealthyRuntime(initialHealth); context.socket = await openRuntimeVerificationSocket(context.repoRoot, (push) => capturePush(push, context.eventState, context.terminalEvents, context.turnStarts)); await assertRuntimeIdle(context); @@ -529,6 +532,7 @@ async function stopOneActiveTurn(context) { if (!isCancelledStopResult(result, target.id)) { throw new Error("Stop did not return the targeted execution's cancelled outcome"); } + context.terminalEvents.notify(); writeReceipt(context.run.receiptPath, context.receipt, context.paths.devDir); } @@ -677,7 +681,8 @@ async function recordFinalReceiptMetrics(context) { receipt.metrics.turnCompletion = summarizeLatency(receipt.state.threads.map((thread) => elapsedSinceSend(thread, "completedAtMs"))); receipt.metrics.turnDurability = summarizeLatency(receipt.state.threads.map((thread) => elapsedSinceSend(thread, "persistedAtMs"))); receipt.state.activeControlLaunch = attributeControlLaunchToTurnCompletion(receipt.state.activeControlLaunch, receipt.state.threads); - receipt.metrics.events = summarizeEventAudits(receipt.state.threads, receipt.workload.scenario); + receipt.metrics.events = summarizeEventAudits(receipt); + receipt.metrics.eventThroughput = summarizeEventThroughput(receipt.state.threads); receipt.metrics.harnessEventLoopStalls = { scope: "harness-process", intervalMs: EVENT_LOOP_INTERVAL_MS, @@ -775,10 +780,18 @@ function createReceipt(run, label, stopOne) { }, metrics: { health: { samples: [], summary: null }, + workerQueue: { + intervalMs: HEALTH_SAMPLE_INTERVAL_MS, + samples: [], + omittedSamples: 0, + unavailableSamples: 0, + summary: { sampleCount: 0, maxPending: 0, maxPendingBytes: 0, maxActiveExecutions: 0, maxQueuedPerWorker: 0, maxInFlightWorkers: 0 }, + }, rpc: {}, turnCompletion: null, turnDurability: null, events: null, + eventThroughput: null, harnessEventLoopStalls: null, serverEventLoopStalls: null, memory: [], @@ -973,9 +986,10 @@ export function attributeControlLaunchToTurnCompletion(controlLaunch, threads) { } class TerminalEventWaiter { - constructor(state, stopOne = false) { + constructor(state, stopOne = false, getStop = () => null) { this.state = state; this.stopOne = stopOne; + this.getStop = getStop; this.resolve = null; } @@ -986,10 +1000,7 @@ class TerminalEventWaiter { reject(new Error("Not every controlled thread produced both turn completion and durable persistence events before the deadline")); }, timeoutMs); this.resolve = () => { - if (![...this.state.values()].every((thread) => - this.stopOne && thread.ordinal === STOP_ONE_ORDINAL - ? thread.status === "paused" || thread.status === "cancelled" - : thread.persistedAtMs !== null && thread.completedAtMs !== null)) return; + if (![...this.state.values()].every((thread) => this.isComplete(thread))) return; this.resolve = null; clearTimeout(timer); resolve(); @@ -1001,6 +1012,18 @@ class TerminalEventWaiter { notify() { this.resolve?.(); } + + isComplete(thread) { + if (!this.stopOne || thread.ordinal !== STOP_ONE_ORDINAL) { + return thread.persistedAtMs !== null && thread.completedAtMs !== null; + } + if (thread.persistedAtMs === null) return false; + if (thread.status === "paused" || thread.status === "cancelled") return true; + const stop = this.getStop(); + return thread.status === "interrupted" && stop?.threadId === thread.id + && stop.status === "cancelled" && stop.snapshotPhase === "cancelled" + && typeof stop.turnExecutionId === "string"; + } } class TurnStartWaiter { @@ -1092,12 +1115,19 @@ function auditRun(receipt) { function isExpectedTerminal(receipt, thread) { if (receipt.workload.scenario !== "stop-one" || thread.ordinal !== STOP_ONE_ORDINAL) return thread.eventAudit.completed; - return receipt.state.stop?.status === "cancelled" - && receipt.state.stop?.reconnectSnapshot === true - && (thread.status === "paused" || thread.status === "cancelled"); + return hasConfirmedStoppedTerminal(receipt, thread); +} + +function hasConfirmedStoppedTerminal(receipt, thread) { + const stop = receipt.state.stop; + return stop?.threadId === thread.id && stop.status === "cancelled" + && stop.snapshotPhase === "cancelled" && stop.reconnectSnapshot === true + && typeof stop.turnExecutionId === "string" && thread.persistedAtMs !== null + && (thread.status === "paused" || thread.status === "cancelled" || thread.status === "interrupted"); } -function summarizeEventAudits(threads, scenario) { +function summarizeEventAudits(receipt) { + const threads = receipt.state.threads; const audits = threads.map((thread) => thread.eventAudit ?? auditAgentEvents(thread.events ?? [])); for (let index = 0; index < threads.length; index += 1) threads[index].eventAudit = audits[index]; const flatten = (field) => audits.flatMap((audit) => audit[field] ?? []); @@ -1110,13 +1140,27 @@ function summarizeEventAudits(threads, scenario) { arrivalOrderViolations: flatten("arrivalOrderViolations"), ok: audits.length === THREAD_COUNT && audits.every((audit) => audit.sequenceValid) && threads.every((thread) => thread.durable?.ok === true) - && threads.every((thread) => hasExpectedEventTerminal(thread, scenario)), + && threads.every((thread) => isExpectedTerminal(receipt, thread)), }; } -function hasExpectedEventTerminal(thread, scenario) { - if (scenario !== "stop-one" || thread.ordinal !== STOP_ONE_ORDINAL) return thread.eventAudit.completed; - return thread.status === "paused" || thread.status === "cancelled"; +/** Counts public event arrivals in bounded time buckets across the workload. */ +export function summarizeEventThroughput(threads) { + const arrivals = threads.flatMap((thread) => (thread.events ?? []).map((event) => event.atMs)).filter(Number.isFinite); + if (arrivals.length === 0) return { scope: "public-agent-event-push", intervalMs: 1_000, totalEvents: 0, eventsPerInterval: [], peakEventsPerSecond: 0 }; + const sentAt = threads.map((thread) => thread.sentAtMs).filter(Number.isFinite); + const first = Math.min(...arrivals, ...sentAt); + const last = Math.max(...arrivals); + const intervalMs = Math.max(1_000, Math.ceil((last - first + 1) / MAX_THROUGHPUT_INTERVALS / 1_000) * 1_000); + const counts = Array.from({ length: Math.floor((last - first) / intervalMs) + 1 }, () => 0); + for (const atMs of arrivals) counts[Math.floor((atMs - first) / intervalMs)] += 1; + return { + scope: "public-agent-event-push", + intervalMs, + totalEvents: arrivals.length, + eventsPerInterval: counts, + peakEventsPerSecond: roundMs(Math.max(...counts) * 1_000 / intervalMs), + }; } async function cleanupOwnedResources(socket, receipt, paths, repoRoot) { @@ -1360,13 +1404,51 @@ async function sampleHealth(healthUrl) { const started = NodePerfHooks.performance.now(); const response = await fetch(healthUrl, { signal: AbortSignal.timeout(15_000) }); const payload = await response.json(); - return { durationMs: NodePerfHooks.performance.now() - started, status: response.ok ? payload?.status : `http-${response.status}` }; + return { + durationMs: NodePerfHooks.performance.now() - started, + status: response.ok ? payload?.status : `http-${response.status}`, + workerQueue: response.ok ? payload?.workerQueue : null, + }; +} + +/** Copies only queue counts from health, with a fixed receipt sample limit. */ +export function recordWorkerQueueSample(target, raw) { + if (!validWorkerQueueDepth(raw)) { + target.unavailableSamples += 1; + return; + } + const workers = raw.workers.map((worker) => ({ index: worker.index, queued: worker.queued, inFlight: worker.inFlight })); + const sample = { + atMs: roundMs(NodePerfHooks.performance.now()), + pending: raw.pending, + pendingBytes: raw.pendingBytes, + activeExecutions: raw.activeExecutions, + workers, + }; + target.summary.sampleCount += 1; + target.summary.maxPending = Math.max(target.summary.maxPending, sample.pending); + target.summary.maxPendingBytes = Math.max(target.summary.maxPendingBytes, sample.pendingBytes); + target.summary.maxActiveExecutions = Math.max(target.summary.maxActiveExecutions, sample.activeExecutions); + target.summary.maxQueuedPerWorker = Math.max(target.summary.maxQueuedPerWorker, ...workers.map((worker) => worker.queued)); + target.summary.maxInFlightWorkers = Math.max(target.summary.maxInFlightWorkers, workers.filter((worker) => worker.inFlight).length); + if (target.samples.length < MAX_WORKER_QUEUE_SAMPLES) target.samples.push(sample); + else target.omittedSamples += 1; +} + +function validWorkerQueueDepth(value) { + const count = (number) => Number.isSafeInteger(number) && number >= 0; + return value !== null && typeof value === "object" + && count(value.pending) && count(value.pendingBytes) && count(value.activeExecutions) + && Array.isArray(value.workers) && value.workers.length <= 16 + && value.workers.every((worker) => worker !== null && typeof worker === "object" + && count(worker.index) && count(worker.queued) && typeof worker.inFlight === "boolean"); } class HealthSampler { - constructor(healthUrl, target) { + constructor(healthUrl, target, queueTarget) { this.healthUrl = healthUrl; this.target = target; + this.queueTarget = queueTarget; this.timer = null; this.inFlight = null; } @@ -1375,7 +1457,10 @@ class HealthSampler { const sample = async () => { if (this.inFlight) return; this.inFlight = sampleHealth(this.healthUrl) - .then((result) => this.target.samples.push(result.durationMs)) + .then((result) => { + this.target.samples.push(result.durationMs); + recordWorkerQueueSample(this.queueTarget, result.workerQueue); + }) .catch(() => this.target.failures = (this.target.failures ?? 0) + 1) .finally(() => { this.inFlight = null; }); }; diff --git a/scripts/perf/seven-thread-live-harness.test.mjs b/scripts/perf/seven-thread-live-harness.test.mjs index 2db017f2f..c67495110 100644 --- a/scripts/perf/seven-thread-live-harness.test.mjs +++ b/scripts/perf/seven-thread-live-harness.test.mjs @@ -11,7 +11,9 @@ import { normalizeTerminalSessions, parseArguments, parseServerStallEntries, + recordWorkerQueueSample, selectTerminalTransport, + summarizeEventThroughput, summarizeLatency, } from "./seven-thread-live-harness.mjs"; @@ -190,3 +192,40 @@ NodeTest.test("reads structured server work stalls and prefers them over duplica { timestamp: "2026-09-24T19:07:37.927Z", stalledMs: 3639.19 }, ]); }); + +NodeTest.test("records only bounded worker queue counts and retains the peak after the sample limit", () => { + const target = { + samples: [], omittedSamples: 0, unavailableSamples: 0, + summary: { sampleCount: 0, maxPending: 0, maxPendingBytes: 0, maxActiveExecutions: 0, maxQueuedPerWorker: 0, maxInFlightWorkers: 0 }, + }; + for (let index = 0; index < 513; index += 1) { + recordWorkerQueueSample(target, { + pending: index, pendingBytes: index * 2, activeExecutions: 7, + workers: [{ index: 0, queued: index, inFlight: true, command: "private prompt" }], + token: "private token", + }); + } + recordWorkerQueueSample(target, { pending: "wrong shape", workers: [] }); + NodeAssertStrict.equal(target.samples.length, 512); + NodeAssertStrict.equal(target.omittedSamples, 1); + NodeAssertStrict.equal(target.unavailableSamples, 1); + NodeAssertStrict.deepEqual(target.summary, { + sampleCount: 513, maxPending: 512, maxPendingBytes: 1024, + maxActiveExecutions: 7, maxQueuedPerWorker: 512, maxInFlightWorkers: 1, + }); + NodeAssertStrict.doesNotMatch(JSON.stringify(target), /private|prompt|token/); +}); + +NodeTest.test("summarizes public event arrival throughput in bounded one-second buckets", () => { + const result = summarizeEventThroughput([ + { sentAtMs: 0, events: [{ atMs: 100 }, { atMs: 900 }, { atMs: 1_100 }] }, + { sentAtMs: 0, events: [{ atMs: 2_500 }] }, + ]); + NodeAssertStrict.deepEqual(result, { + scope: "public-agent-event-push", intervalMs: 1_000, + totalEvents: 4, eventsPerInterval: [2, 1, 1], peakEventsPerSecond: 2, + }); + const longRun = summarizeEventThroughput([{ sentAtMs: 0, events: [{ atMs: 0 }, { atMs: 3_600_000 }] }]); + NodeAssertStrict.ok(longRun.eventsPerInterval.length <= 600); + NodeAssertStrict.equal(longRun.totalEvents, 2); +}); From fe98cec36e08e58426c4e268c271f4dede79d238 Mon Sep 17 00:00:00 2001 From: "(CJ) Chukwudi Nwobodo" <142016413+chuks-qua@users.noreply.github.com> Date: Fri, 25 Sep 2026 16:01:08 +0100 Subject: [PATCH 111/136] fix(server): trace only the main event loop --- .../diagnostics/__tests__/server-work-trace.test.ts | 13 ++++++++----- .../agents/diagnostics/server-work-trace.ts | 5 +++-- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/apps/server/src/features/agents/diagnostics/__tests__/server-work-trace.test.ts b/apps/server/src/features/agents/diagnostics/__tests__/server-work-trace.test.ts index 77467d1d4..0c9fbfeaf 100644 --- a/apps/server/src/features/agents/diagnostics/__tests__/server-work-trace.test.ts +++ b/apps/server/src/features/agents/diagnostics/__tests__/server-work-trace.test.ts @@ -1,3 +1,4 @@ +import * as NodePerfHooks from "node:perf_hooks"; import { describe, expect, it } from "vitest"; import { eventApplyType, ServerWorkTrace, type ServerWorkTraceReport } from "../server-work-trace.js"; @@ -5,7 +6,8 @@ describe("ServerWorkTrace", () => { it("attributes aggregate work in a stalled tick and bounds content-free output", () => { const reports: ServerWorkTraceReport[] = []; const trace = new ServerWorkTrace((report) => reports.push(report)); - trace.tick(1_000); + const started = NodePerfHooks.performance.now(); + trace.tick(started + 20); for (let index = 0; index < 40; index += 1) { trace.record("event-apply", "thread-a", "execution-a", 12, "textDelta"); } @@ -19,7 +21,7 @@ describe("ServerWorkTrace", () => { trace.record("narrative-prepare", "thread-overflow", "execution-d", 4); trace.record("narrative-persist", "thread-overflow", "execution-d", 2); trace.record("narrative-confirm", "thread-overflow", "execution-d", 1); - trace.tick(1_240); + trace.tick(started + 260); expect(reports).toHaveLength(1); const report = reports[0]; @@ -48,14 +50,15 @@ describe("ServerWorkTrace", () => { expect(report.samples[2]).toMatchObject({ threadId: null, executionId: null }); expect(JSON.stringify(report)).not.toMatch(/prompt|toolInput|output|secret/); - trace.tick(1_260); + trace.tick(started + 280); expect(reports).toHaveLength(1); }); it("classifies only the fixed event type across threads", () => { const reports: ServerWorkTraceReport[] = []; const trace = new ServerWorkTrace((report) => reports.push(report)); - trace.tick(1_000); + const started = NodePerfHooks.performance.now(); + trace.tick(started + 20); const types = ["textDelta", "toolUse", "toolResult", "assistantMessageBoundary", "unrecognized"]; types.forEach((type, index) => { trace.record("event-apply", `thread-${index}`, "execution-a", index + 1, eventApplyType(type)); @@ -64,7 +67,7 @@ describe("ServerWorkTrace", () => { trace.record("narrative-persist", `thread-${index}`, "execution-a", 1); trace.record("narrative-confirm", `thread-${index}`, "execution-a", 1); }); - trace.tick(1_200); + trace.tick(started + 220); const report = reports[0]; expect(report?.kind).toBe("server-work-stall"); if (report?.kind !== "server-work-stall") return; diff --git a/apps/server/src/features/agents/diagnostics/server-work-trace.ts b/apps/server/src/features/agents/diagnostics/server-work-trace.ts index f0cde056b..d6c25f9cf 100644 --- a/apps/server/src/features/agents/diagnostics/server-work-trace.ts +++ b/apps/server/src/features/agents/diagnostics/server-work-trace.ts @@ -1,4 +1,5 @@ import * as NodePerfHooks from "node:perf_hooks"; +import * as NodeWorkerThreads from "node:worker_threads"; import { logger } from "@mcode/shared"; /** Fixed names keep trace output free of event bodies and provider text. */ @@ -194,8 +195,8 @@ export class ServerWorkTrace { } } -/** Null unless explicitly enabled before server startup. */ -export const serverWorkTrace = process.env.MCODE_SERVER_WORK_TRACE === "1" +/** Trace only the server loop; writer-worker delays are not server stalls. */ +export const serverWorkTrace = NodeWorkerThreads.isMainThread && process.env.MCODE_SERVER_WORK_TRACE === "1" ? new ServerWorkTrace((report) => logger.warn("Server work trace", report)) : null; From 4d3e90aa81b303c6f4d971cdd0cf81bebd757015 Mon Sep 17 00:00:00 2001 From: "(CJ) Chukwudi Nwobodo" <142016413+chuks-qua@users.noreply.github.com> Date: Fri, 25 Sep 2026 16:03:29 +0100 Subject: [PATCH 112/136] docs(performance): record issue 1760 matched runs --- .../issue-1760-seven-task-evidence.md | 83 +++++++++++++++++++ 1 file changed, 83 insertions(+) create mode 100644 docs/performance/issue-1760-seven-task-evidence.md diff --git a/docs/performance/issue-1760-seven-task-evidence.md b/docs/performance/issue-1760-seven-task-evidence.md new file mode 100644 index 000000000..9de4da6f7 --- /dev/null +++ b/docs/performance/issue-1760-seven-task-evidence.md @@ -0,0 +1,83 @@ +# Issue 1760: seven-task evidence + +Measured on Windows with Bun 1.4.1 and the public seven-task fixture harness. The +baseline was `15fa7030ad09b5c2d083efeb2e0670b68277220c`; the candidate +runtime was merge commit `0a73175d`. Each row is one sequential run with seven +Codex tasks and one terminal. All 20 runs passed the persisted reload audit, +had no missing, duplicate, or reordered events, and cleaned up their own +resources. The candidate uses four execution workers. The diagnostic-only fix +in `fe98cec3` does not change the default runtime path. + +`Stall` is the largest logged main-server event-loop delay during the run; zero +means none was logged. `Rate` is the largest one-second bucket of public event +arrivals. The baseline rate is derived from its saved arrival timestamps using +the candidate harness's bucket formula. `Memory` is peak server working set in +MiB. `Finish` is the p95 completion time among that run's seven tasks. Terminal +and models are the single RPC observation in each run. Times are milliseconds. + +| Run | Events | Stall | Rate/s | Memory | Finish | Terminal | Models | Queued/worker | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | +| Before 1 | 3458 | 7775 | 177 | 378.6 | 47756 | 24197 | 133 | unavailable | +| Before 2 | 3458 | 8079 | 245 | 623.9 | 29606 | 2605 | 40 | unavailable | +| Before 3 | 3458 | 6956 | 250 | 445.4 | 26953 | 25863 | 14 | unavailable | +| Before 4 | 3458 | 6483 | 270 | 447.3 | 25621 | 809 | 35 | unavailable | +| Before 5 | 3458 | 5871 | 259 | 460.8 | 24854 | 684 | 119 | unavailable | +| Before 6 | 3458 | 6322 | 200 | 367.5 | 36525 | 7115 | 128 | unavailable | +| Before 7 | 3458 | 7096 | 207 | 538.0 | 36256 | 133 | 24 | unavailable | +| Before 8 | 3459 | 6309 | 208 | 533.1 | 37795 | 146 | 11 | unavailable | +| Before 9 | 3459 | 6499 | 217 | 394.1 | 38664 | 117 | 16 | unavailable | +| Before 10 | 3459 | 4893 | 258 | 417.6 | 22705 | 103 | 24 | unavailable | +| After 1 | 3465 | 0 | 147 | 555.3 | 73301 | 200 | 18 | 1 | +| After 2 | 3466 | 0 | 138 | 538.2 | 75846 | 217 | 52 | 1 | +| After 3 | 3466 | 0 | 148 | 574.3 | 73599 | 176 | 9 | 1 | +| After 4 | 3466 | 0 | 149 | 576.4 | 75128 | 284 | 56 | 1 | +| After 5 | 3466 | 0 | 148 | 592.4 | 72390 | 205 | 36 | 1 | +| After 6 | 3466 | 0 | 142 | 590.1 | 72521 | 193 | 7 | 1 | +| After 7 | 3466 | 0 | 156 | 602.7 | 70955 | 166 | 49 | 1 | +| After 8 | 3466 | 0 | 141 | 627.6 | 75179 | 181 | 17 | 1 | +| After 9 | 3466 | 0 | 156 | 608.3 | 70481 | 175 | 34 | 1 | +| After 10 | 3466 | 1096 | 142 | 587.2 | 71338 | 1008 | 476 | 1 | + +| Metric | Before median (range) | After median (range) | +| --- | ---: | ---: | +| Largest server pause, ms | 6491 (4893–8079) | 0 (0–1096) | +| Peak public events/s | 231 (177–270) | 147.5 (138–156) | +| Peak server memory, MiB | 446.4 (367.5–623.9) | 588.7 (538.2–627.6) | +| Per-run finish p95, ms | 32931 (22705–47756) | 72911 (70481–75846) | +| Terminal creation, ms | 746 (103–25863) | 196 (166–1008) | +| Model list, ms | 29 (11–133) | 35 (7–476) | + +The candidate queue had at most seven pending commands and one queued command +per worker. The baseline harness did not record a comparable queue depth. +The change sharply reduced the observed long server pauses and kept the tested +controls within their budgets. It also reduced peak event throughput by about +36%, more than doubled completion p95, and raised median peak working set by +about 142 MiB. These are material costs, not an overall latency win. An +exploratory eight-worker run did not improve completion and used more memory; +the fixed pool remains at four. + +The stopped-task run durably cancelled only its selected execution, while the +six peers completed in order. Stop returned in 584 ms and terminal creation in +914 ms; cleanup passed. It recorded a 770 ms server pause 1.25 seconds after +startup. After run 10 recorded 1096 ms at 1.73 seconds, while provider sends +and terminal startup were active. Work tracing was off for these default runs, +so the exact operation causing either pause is unknown. They must not be +credited to, or ruled out from, a specific operation solely from timestamps. + +During follow-up, enabling the old `MCODE_SERVER_WORK_TRACE` also enabled it +inside the SQLite writer worker. That worker's pauses were incorrectly counted +as server-loop pauses. `fe98cec3` limits this trace to the main thread. Ten +additional seven-task runs with the corrected trace all passed, with largest +main-server delays of 312, 242, 272, 325, 471, 306, 244, 228, 316, and 430 ms. +Their traces recorded no main-thread event application or finalization work for +these worker-owned Codex turns. Those runs did not reproduce either default-run +pause above 500 ms; the intermittent startup pause is tracked in +[issue #1767](https://github.com/Mzeey-Empire/mcode/issues/1767). + +On the owned Electron app, a real Codex reply remained visible after reload; +the model picker opened in 58 ms, task switch in 1004 ms, terminal prompt in +2338 ms, and a PowerShell command printed in 269 ms. Stop cleared its button +in 119 ms. After reload the prompt remained, the Stop button was absent, and +SQLite showed a `Cancelled` turn with a `cancelled` ingest checkpoint. These +are individual desktop observations, not latency distributions. Claude and +Cursor adapter behavior was checked by focused tests, not live provider runs. From cf9dca227b1b18abc50b31379d48d484b3bfc7dc Mon Sep 17 00:00:00 2001 From: "(CJ) Chukwudi Nwobodo" <142016413+chuks-qua@users.noreply.github.com> Date: Fri, 25 Sep 2026 16:07:51 +0100 Subject: [PATCH 113/136] refactor(perf): simplify stopped-run completion check --- scripts/perf/seven-thread-live-harness.mjs | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/scripts/perf/seven-thread-live-harness.mjs b/scripts/perf/seven-thread-live-harness.mjs index 11a884758..04e813240 100644 --- a/scripts/perf/seven-thread-live-harness.mjs +++ b/scripts/perf/seven-thread-live-harness.mjs @@ -1014,18 +1014,19 @@ class TerminalEventWaiter { } isComplete(thread) { - if (!this.stopOne || thread.ordinal !== STOP_ONE_ORDINAL) { - return thread.persistedAtMs !== null && thread.completedAtMs !== null; - } if (thread.persistedAtMs === null) return false; + if (!this.stopOne || thread.ordinal !== STOP_ONE_ORDINAL) return thread.completedAtMs !== null; if (thread.status === "paused" || thread.status === "cancelled") return true; - const stop = this.getStop(); - return thread.status === "interrupted" && stop?.threadId === thread.id - && stop.status === "cancelled" && stop.snapshotPhase === "cancelled" - && typeof stop.turnExecutionId === "string"; + return isInterruptedAfterStop(thread, this.getStop()); } } +function isInterruptedAfterStop(thread, stop) { + return thread.status === "interrupted" && stop?.threadId === thread.id + && stop.status === "cancelled" && stop.snapshotPhase === "cancelled" + && typeof stop.turnExecutionId === "string"; +} + class TurnStartWaiter { constructor(state) { this.state = state; From 990dee6a496998be6996a4c8f47488a259e89f88 Mon Sep 17 00:00:00 2001 From: "(CJ) Chukwudi Nwobodo" <142016413+chuks-qua@users.noreply.github.com> Date: Fri, 25 Sep 2026 16:07:53 +0100 Subject: [PATCH 114/136] test(providers): expect Codex delivery attempts --- .../src/__tests__/codex/codex-provider-lifecycle.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/providers/src/__tests__/codex/codex-provider-lifecycle.test.ts b/packages/providers/src/__tests__/codex/codex-provider-lifecycle.test.ts index fc2cf8053..b29c34ba5 100644 --- a/packages/providers/src/__tests__/codex/codex-provider-lifecycle.test.ts +++ b/packages/providers/src/__tests__/codex/codex-provider-lifecycle.test.ts @@ -88,7 +88,7 @@ function createProvider() { const complete = async (number: number, executionId: string) => { await vi.waitFor(() => expect(starts()).toHaveLength(number)); child.complete(`native-turn-${number}`); - await vi.waitFor(() => expect(events).toContainEqual({ event: { + await vi.waitFor(() => expect(events).toContainEqual({ deliveryAttempt: 1, event: { type: AgentEventType.Ended, threadId: request.threadId, turnExecutionId: executionId, outcome: "completed", } })); }; From 25ac8186eeae315cb813f684a4292348eb89d97c Mon Sep 17 00:00:00 2001 From: "(CJ) Chukwudi Nwobodo" <142016413+chuks-qua@users.noreply.github.com> Date: Fri, 25 Sep 2026 16:18:09 +0100 Subject: [PATCH 115/136] test(server): align conversation paging and worker teardown --- .../read-model/__tests__/conversation-page.test.ts | 12 ++++++++---- .../__tests__/agent-service-gate.test.ts | 2 ++ .../__tests__/provider-composition-container.test.ts | 2 ++ 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/apps/server/src/features/agents/conversation/read-model/__tests__/conversation-page.test.ts b/apps/server/src/features/agents/conversation/read-model/__tests__/conversation-page.test.ts index d03209c9f..6c077eaab 100644 --- a/apps/server/src/features/agents/conversation/read-model/__tests__/conversation-page.test.ts +++ b/apps/server/src/features/agents/conversation/read-model/__tests__/conversation-page.test.ts @@ -55,7 +55,7 @@ function createDeps(db: Database) { } describe("loadConversationPage", () => { - it("replays current-session notices beyond the bounded tail and retains old turn notices in history", () => { + it("replays current-session notices beyond the bounded tail without spending message slots on hidden notices", () => { const db = openMemoryDatabase(); seedThread(db); const deps = createDeps(db); @@ -95,7 +95,7 @@ describe("loadConversationPage", () => { const tail = loadConversationTail(deps, { threadId: "thread-1", limit: 2 }); expect(page.messages.map((message) => message.content)).toEqual([ - "Old security notice", "Late old-session warning", "Security notice", "Warning notice", "Model rerouted", "Tail message one", "Tail message two", + "Old security notice", "Security notice", "Model rerouted", "Tail message one", "Tail message two", ]); expect(tail.messages.map((message) => message.content)).toEqual(["Tail message one", "Tail message two"]); expect(page.sessionNotices.map((message) => message.content)).toEqual([ @@ -110,8 +110,10 @@ describe("loadConversationPage", () => { "SELECT COUNT(*) AS count FROM messages WHERE content = 'Fix config again'", ).get()).toEqual({ count: 0 }); expect(deps.messageRepo.listByThread("thread-1", 20).messages.map((message) => message.content)).toEqual([ - "Old security notice", "Late old-session warning", "Security notice", "Warning notice", "Model rerouted", "Tail message one", "Tail message two", + "Old security notice", "Security notice", "Model rerouted", "Tail message one", "Tail message two", ]); + expect(db.prepare("SELECT content FROM messages WHERE content IN ('Late old-session warning', 'Warning notice') ORDER BY sequence").all()) + .toEqual([{ content: "Late old-session warning" }, { content: "Warning notice" }]); db.close(); }); @@ -191,7 +193,9 @@ describe("loadConversationPage", () => { repo.beginNoticeSession("thread-1", "session-1"); for (let i = 0; i < 23; i++) repo.createSystemNotice("thread-1", `Warning ${i}`, i, { kind: "warning", presentation: "timeline", scope: "turn", sessionId: "session-1", noticeKey: `warning-${i}` }); expect(repo.listSessionNotices("thread-1").map((message) => message.content)).toEqual(Array.from({ length: 20 }, (_, i) => `Warning ${i + 3}`)); - expect(repo.listByThread("thread-1", 23).messages).toHaveLength(23); + expect(repo.listByThread("thread-1", 23).messages).toEqual([]); + expect(db.prepare("SELECT COUNT(*) AS count FROM messages WHERE thread_id = ?").get("thread-1")) + .toEqual({ count: 23 }); db.close(); }); diff --git a/apps/server/src/features/agents/orchestration/__tests__/agent-service-gate.test.ts b/apps/server/src/features/agents/orchestration/__tests__/agent-service-gate.test.ts index 27e5ca42c..3446fd76d 100644 --- a/apps/server/src/features/agents/orchestration/__tests__/agent-service-gate.test.ts +++ b/apps/server/src/features/agents/orchestration/__tests__/agent-service-gate.test.ts @@ -340,6 +340,8 @@ describe("AgentService.sendMessage — admission gates", () => { sourceTurnId: "source-turn", sourceProviderId: "claude", }, + expect.any(String), + undefined, ); expect(threadRepo.updateStatus).toHaveBeenCalledWith(THREAD_ID, "active"); expect(providerStub.sendTurn).toHaveBeenCalledTimes(1); diff --git a/apps/server/src/features/providers/composition/__tests__/provider-composition-container.test.ts b/apps/server/src/features/providers/composition/__tests__/provider-composition-container.test.ts index 05f96508c..4efa269f2 100644 --- a/apps/server/src/features/providers/composition/__tests__/provider-composition-container.test.ts +++ b/apps/server/src/features/providers/composition/__tests__/provider-composition-container.test.ts @@ -13,6 +13,7 @@ import type { ProviderHostPorts } from "@mcode/providers"; import { setupContainer } from "../../../../application/composition/container.js"; import { CanonicalAgentBoundary } from "../../../agents/canonical/canonical-agent-boundary.js"; +import { WorkerOwnedTurnRuntime } from "../../../agents/execution/worker-owned-turn-runtime.js"; import { MessageRepo } from "../../../agents/conversation/persistence/message-repo.js"; import { ProviderRegistry } from "../provider-registry.js"; import { SettingsService } from "../../../settings/settings-service.js"; @@ -90,6 +91,7 @@ describe("provider composition container", () => { afterEach(async () => { vi.restoreAllMocks(); await container.resolve(ProviderRegistry).shutdown(); + await container.resolve(WorkerOwnedTurnRuntime).close(); container.resolve(ProviderEventIngress).shutdown(); container.resolve(SettingsService).dispose(); database?.close(true); From 4e77fc331dcc111c21b349688049bd69582d1b2c Mon Sep 17 00:00:00 2001 From: "(CJ) Chukwudi Nwobodo" <142016413+chuks-qua@users.noreply.github.com> Date: Fri, 25 Sep 2026 20:36:33 +0100 Subject: [PATCH 116/136] fix(server): restrict agent cleanup to fixture workspace --- .../application/bootstrap/server-bootstrap.ts | 4 +- .../cleanup/__tests__/cleanup-worker.test.ts | 82 +++++++++++++++++++ .../thread-control/cleanup/cleanup-worker.ts | 22 +++-- .../cleanup/persistence/cleanup-job-repo.ts | 29 ++++--- 4 files changed, 117 insertions(+), 20 deletions(-) diff --git a/apps/server/src/application/bootstrap/server-bootstrap.ts b/apps/server/src/application/bootstrap/server-bootstrap.ts index 6b4742fdb..f9855b707 100644 --- a/apps/server/src/application/bootstrap/server-bootstrap.ts +++ b/apps/server/src/application/bootstrap/server-bootstrap.ts @@ -852,7 +852,9 @@ async function bootstrapServer(): Promise { interruptThreadStartupsAtStartup(); projectActionService.recoverStaleRuns(); - cleanupWorker.start(); + cleanupWorker.start(process.env.MCODE_AGENT_RUNTIME === "1" + ? process.env.MCODE_AGENT_FIXTURE_REPO?.trim() + : undefined); recordStartupCheckpoint("stale startup work recovery completed"); await workerOwnedTurnRuntime.whenReady(); diff --git a/apps/server/src/features/thread-control/cleanup/__tests__/cleanup-worker.test.ts b/apps/server/src/features/thread-control/cleanup/__tests__/cleanup-worker.test.ts index c6d15f9db..57ec42d4e 100644 --- a/apps/server/src/features/thread-control/cleanup/__tests__/cleanup-worker.test.ts +++ b/apps/server/src/features/thread-control/cleanup/__tests__/cleanup-worker.test.ts @@ -295,6 +295,88 @@ describe("CleanupWorker sandbox worktrees", { timeout: 20_000 }, () => { expect(gitWorktrees.removeWorktree).toHaveBeenCalledOnce(); }); + it("restricts startup recovery and cleanup admission to the fixture workspace", async () => { + const fixture = workspaces.create("Fixture", "/fixture"); + const user = workspaces.create("Copied user project", "/fixture-other"); + const deleting = workspaces.create("Deleting user project", "/user-deleting"); + const empty = workspaces.create("Empty deleting user project", "/user-empty"); + workspaces.softDelete(deleting.id); + workspaces.softDelete(empty.id); + addThread(deleting.id, "user-missing-job", "/user-worktree", "user", null); + + for (const workspace of [fixture, user]) { + const expiredId = `${workspace.id}-expired`; + const explicitId = `${workspace.id}-explicit`; + addThread(workspace.id, expiredId, `${workspace.path}/expired`, "expired", new Date(0).toISOString()); + addThread(workspace.id, explicitId, `${workspace.path}/explicit`, "explicit", null); + threads.softDelete(explicitId); + const job = cleanupJobs.insert({ + thread_id: explicitId, + workspace_path: workspace.path, + worktree_path: `${workspace.path}/explicit`, + branch: "explicit", + }); + database.prepare("UPDATE cleanup_jobs SET attempts = 5, last_error = 'original error' WHERE id = ?") + .run(job.id); + } + for (const kind of ["explicit", "retention"] as const) { + cleanupJobs.insert({ + thread_id: `user-orphan-${kind}`, + workspace_path: user.path, + worktree_path: null, + branch: null, + kind, + }); + } + const userRows = () => ({ + workspaces: database.prepare("SELECT * FROM workspaces WHERE id != ? ORDER BY id").all(fixture.id), + threads: database.prepare("SELECT * FROM threads WHERE workspace_id != ? ORDER BY id").all(fixture.id), + jobs: database.prepare("SELECT * FROM cleanup_jobs WHERE workspace_path != ? ORDER BY id").all(fixture.path), + }); + const before = userRows(); + + worker.start(fixture.path); + try { + await worker.reconcileOnStartup(); + await worker.poll(); + + expect(userRows()).toEqual(before); + expect(threads.findById(`${fixture.id}-expired`)).toBeNull(); + expect(threads.findById(`${fixture.id}-explicit`)).toBeNull(); + expect(cleanupJobs.countByWorkspacePath(fixture.path)).toBe(0); + expect(gitWorktrees.removeWorktree).toHaveBeenCalledTimes(2); + expect(cleanupJobs.getDueCounts(Date.now(), fixture.path)).toEqual({ explicit: 0, retention: 0 }); + expect(await worker.processOneJob()).toBe(false); + expect(userRows()).toEqual(before); + } finally { + await worker.shutdown(); + } + }); + + it("leaves all copied rows untouched when the fixture path has no matching workspace", async () => { + const user = workspaces.create("Copied user project", "/user"); + addThread(user.id, "user-expired", "/user/expired", "expired", new Date(0).toISOString()); + const job = cleanupJobs.insert({ + thread_id: "user-orphan", workspace_path: user.path, worktree_path: null, branch: null, + }); + database.prepare("UPDATE cleanup_jobs SET attempts = 5 WHERE id = ?").run(job.id); + const threadBefore = threads.findById("user-expired"); + const jobBefore = cleanupJobs.findById(job.id); + + worker.start("/missing-fixture"); + try { + await worker.reconcileOnStartup(); + await worker.poll(); + expect(await worker.processOneJob()).toBe(false); + expect(threads.findById("user-expired")).toEqual(threadBefore); + expect(cleanupJobs.findById(job.id)).toEqual(jobBefore); + expect(cleanupJobs.count()).toBe(1); + expect(gitWorktrees.removeWorktree).not.toHaveBeenCalled(); + } finally { + await worker.shutdown(); + } + }); + it("deletes exhausted orphan job rows on the next poll after startup requeue", async () => { const job = cleanupJobs.insert({ thread_id: "thread-already-gone", diff --git a/apps/server/src/features/thread-control/cleanup/cleanup-worker.ts b/apps/server/src/features/thread-control/cleanup/cleanup-worker.ts index 1f918e76a..a3b18d9bc 100644 --- a/apps/server/src/features/thread-control/cleanup/cleanup-worker.ts +++ b/apps/server/src/features/thread-control/cleanup/cleanup-worker.ts @@ -65,6 +65,7 @@ export class CleanupWorker { private running = false; private stopped = false; private pollPromise: Promise | null = null; + private workspacePath: string | undefined; private readonly mutationReservations: ThreadControlMutationReservationService; constructor( @@ -89,10 +90,12 @@ export class CleanupWorker { } /** - * Start the worker and begin polling for due jobs. + * Start polling, optionally restricted to the agent runtime's fixture workspace. */ - start(): void { + start(workspacePath?: string): void { if (this.pollTimer !== null) return; + // Agent setup copies user projects into its database; cleanup must leave them untouched. + this.workspacePath = workspacePath; const removedArtifacts = pruneStaleToolOutputArtifacts(); if (removedArtifacts > 0) { logger.info("Pruned stale tool-output artifacts", { removed: removedArtifacts }); @@ -156,11 +159,13 @@ export class CleanupWorker { // that arrives during the async job execution sees running=true. this.running = true; try { - const retentionJobsEnqueued = this.cleanupJobRepo.enqueueExpiredCompleted(new Date().toISOString()); + const retentionJobsEnqueued = this.cleanupJobRepo.enqueueExpiredCompleted( + new Date().toISOString(), undefined, this.workspacePath, + ); const nowMs = Date.now(); - const jobs = this.cleanupJobRepo.findDue(nowMs); + const jobs = this.cleanupJobRepo.findDue(nowMs, undefined, this.workspacePath); if (jobs.length > 0 || retentionJobsEnqueued > 0) { - const dueCounts = this.cleanupJobRepo.getDueCounts(nowMs); + const dueCounts = this.cleanupJobRepo.getDueCounts(nowMs, this.workspacePath); const selectedExplicitJobs = jobs.filter((job) => job.kind === "explicit").length; const selectedRetentionJobs = jobs.length - selectedExplicitJobs; logger.info("CleanupWorker batch selected", { @@ -429,12 +434,13 @@ export class CleanupWorker { * hard-deletes it immediately. */ async reconcileOnStartup(): Promise { - const requeued = this.cleanupJobRepo.requeueExhaustedJobs(); + const requeued = this.cleanupJobRepo.requeueExhaustedJobs(this.workspacePath); if (requeued > 0) { logger.info("Requeued exhausted cleanup jobs", { requeued }); } - const deletingWorkspaces = this.workspaceRepo.findDeleting(); + const deletingWorkspaces = this.workspaceRepo.findDeleting() + .filter((workspace) => this.workspacePath === undefined || workspace.path === this.workspacePath); for (const ws of deletingWorkspaces) { const threads = this.threadRepo.listAllByWorkspace(ws.id); @@ -534,7 +540,7 @@ export class CleanupWorker { /** Process a single due cleanup job. Returns true if a job was processed. Exported for testing. */ async processOneJob(): Promise { - const jobs = this.cleanupJobRepo.findDue(Date.now()); + const jobs = this.cleanupJobRepo.findDue(Date.now(), undefined, this.workspacePath); if (jobs.length === 0) return false; await this.executeJob(jobs[0]); return true; diff --git a/apps/server/src/features/thread-control/cleanup/persistence/cleanup-job-repo.ts b/apps/server/src/features/thread-control/cleanup/persistence/cleanup-job-repo.ts index 07b8d38b8..90b4ac2d3 100644 --- a/apps/server/src/features/thread-control/cleanup/persistence/cleanup-job-repo.ts +++ b/apps/server/src/features/thread-control/cleanup/persistence/cleanup-job-repo.ts @@ -167,16 +167,16 @@ export class CleanupJobRepo { * Return jobs that are due to run: next_retry_at <= now and attempts < max. * Ordered by created_at ascending so oldest jobs are processed first. */ - findDue(nowMs: number, limit = CLEANUP_BATCH_LIMIT): CleanupJob[] { + findDue(nowMs: number, limit = CLEANUP_BATCH_LIMIT, workspacePath?: string): CleanupJob[] { const boundedLimit = Math.max(1, Math.min(100, Math.trunc(limit))); - const counts = this.getDueCounts(nowMs); + const counts = this.getDueCounts(nowMs, workspacePath); if (counts.explicit === 0 || counts.retention === 0) { - return this.findDueFromAvailableKind(nowMs, boundedLimit, counts); + return this.findDueFromAvailableKind(nowMs, boundedLimit, counts, workspacePath); } const limits = calculateDueLimits(counts, boundedLimit); - const explicitJobs = this.findDueByKind(nowMs, "explicit", limits.explicit); - const retentionJobs = this.findDueByKind(nowMs, "retention", limits.retention); + const explicitJobs = this.findDueByKind(nowMs, "explicit", limits.explicit, workspacePath); + const retentionJobs = this.findDueByKind(nowMs, "retention", limits.retention, workspacePath); return interleaveDueJobs(retentionJobs, explicitJobs, boundedLimit); } @@ -184,12 +184,13 @@ export class CleanupJobRepo { nowMs: number, limit: number, counts: CleanupJobDueCounts, + workspacePath?: string, ): CleanupJob[] { const kind = counts.retention > 0 ? "retention" : "explicit"; - return this.findDueByKind(nowMs, kind, limit); + return this.findDueByKind(nowMs, kind, limit, workspacePath); } - private findDueByKind(nowMs: number, kind: CleanupJob["kind"], limit: number): CleanupJob[] { + private findDueByKind(nowMs: number, kind: CleanupJob["kind"], limit: number, workspacePath?: string): CleanupJob[] { const rows = this.orm .select() .from(cleanupJobs) @@ -197,6 +198,7 @@ export class CleanupJobRepo { lte(cleanupJobs.nextRetryAt, nowMs), lt(cleanupJobs.attempts, MAX_CLEANUP_ATTEMPTS), eq(cleanupJobs.kind, kind), + workspacePath === undefined ? undefined : eq(cleanupJobs.workspacePath, workspacePath), )) .orderBy(asc(cleanupJobs.createdAt)) .limit(limit) @@ -205,7 +207,7 @@ export class CleanupJobRepo { } /** Return due cleanup job counts grouped by processing kind. */ - getDueCounts(nowMs: number): CleanupJobDueCounts { + getDueCounts(nowMs: number, workspacePath?: string): CleanupJobDueCounts { const counts: CleanupJobDueCounts = { explicit: 0, retention: 0 }; const rows = this.orm .select({ kind: cleanupJobs.kind, count: sql`COUNT(*)` }) @@ -214,6 +216,7 @@ export class CleanupJobRepo { lte(cleanupJobs.nextRetryAt, nowMs), lt(cleanupJobs.attempts, MAX_CLEANUP_ATTEMPTS), inArray(cleanupJobs.kind, ["explicit", "retention"]), + workspacePath === undefined ? undefined : eq(cleanupJobs.workspacePath, workspacePath), )) .groupBy(cleanupJobs.kind) .all(); @@ -222,7 +225,7 @@ export class CleanupJobRepo { } /** Queue a bounded set of expired completed threads in one database transaction. */ - enqueueExpiredCompleted(nowIso: string, limit = CLEANUP_BATCH_LIMIT): number { + enqueueExpiredCompleted(nowIso: string, limit = CLEANUP_BATCH_LIMIT, workspacePath?: string): number { const boundedLimit = Math.max(1, Math.min(100, Math.trunc(limit))); const due = this.orm .select({ @@ -240,6 +243,7 @@ export class CleanupJobRepo { isNotNull(threads.scheduledDeletionAt), lte(threads.scheduledDeletionAt, nowIso), isNull(threads.cleanupState), + workspacePath === undefined ? undefined : eq(workspaces.path, workspacePath), )) .orderBy(asc(threads.scheduledDeletionAt), asc(threads.id)) .limit(boundedLimit) @@ -316,12 +320,15 @@ export class CleanupJobRepo { * deleting workspaces. Orphaned rows self-clean on the next poll because * the worker deletes explicit jobs whose thread is gone. */ - requeueExhaustedJobs(): number { + requeueExhaustedJobs(workspacePath?: string): number { const result = runChanges( this.orm .update(cleanupJobs) .set({ attempts: 0, nextRetryAt: 0 }) - .where(gte(cleanupJobs.attempts, MAX_CLEANUP_ATTEMPTS)), + .where(and( + gte(cleanupJobs.attempts, MAX_CLEANUP_ATTEMPTS), + workspacePath === undefined ? undefined : eq(cleanupJobs.workspacePath, workspacePath), + )), ); return result.changes; } From 904e8dadd781cd530b647f014d4f2a166c9d8454 Mon Sep 17 00:00:00 2001 From: "(CJ) Chukwudi Nwobodo" <142016413+chuks-qua@users.noreply.github.com> Date: Fri, 25 Sep 2026 21:36:42 +0100 Subject: [PATCH 117/136] perf(server): load bounded state for terminal batches --- .../canonical-agent-event-sink.test.ts | 141 +++++++++++++++++- .../canonical/canonical-agent-boundary.ts | 55 +++++-- 2 files changed, 176 insertions(+), 20 deletions(-) diff --git a/apps/server/src/features/agents/canonical/__tests__/canonical-agent-event-sink.test.ts b/apps/server/src/features/agents/canonical/__tests__/canonical-agent-event-sink.test.ts index 2d86a68a6..2478066ba 100644 --- a/apps/server/src/features/agents/canonical/__tests__/canonical-agent-event-sink.test.ts +++ b/apps/server/src/features/agents/canonical/__tests__/canonical-agent-event-sink.test.ts @@ -1,5 +1,8 @@ import "reflect-metadata"; import type { Database } from "bun:sqlite"; +import * as NodeFS from "node:fs"; +import * as NodeOS from "node:os"; +import * as NodePath from "node:path"; import { beforeEach, describe, expect, it, vi } from "vitest"; import { CANONICAL_AGENT_EVENT_BATCH_MAX, @@ -19,7 +22,7 @@ import { type ProviderIdentity, type ProviderRuntimeExtension, } from "@mcode/contracts"; -import { openMemoryDatabase } from "../../../../runtime/persistence/sqlite/database.js"; +import { openDatabase, openMemoryDatabase } from "../../../../runtime/persistence/sqlite/database.js"; import { MessageRepo } from "../../conversation/persistence/message-repo.js"; import { ThreadRepo } from "../../../thread-control/persistence/thread-repo.js"; import { ACTIVE_TURN_WRITE_BATCH_LIMITS } from "../../../../runtime/persistence/sqlite/bounded-write-batches.js"; @@ -271,7 +274,7 @@ function appendItemDraft(eventId: string, itemId: string): CanonicalAgentEventDr }; } -function parentNarrativeToolCall(index: number): ParentNarrativeRecoveryItem { +function parentNarrativeToolCall(index: number): Extract { return { kind: "toolCall", record: { @@ -309,6 +312,27 @@ function executionIdForTurn(db: Database, turnId: string): string { return row.execution_id; } +function parentTerminalInput(db: Database, toolCount = 0): Parameters[0] { + const messageRepo = new MessageRepo(db); + const message = messageRepo.create(THREAD_ID, "assistant", "answer", 2, undefined, undefined, undefined, undefined, true); + return { + threadId: THREAD_ID, + turnId: TURN_ID, + executionId: EXECUTION_ID, + providerId: "codex", + providerIdentities: [], + outcome: "completed", + projectTurn: () => ({ + message: { ...message, is_internal: false }, + narrative: Array.from({ length: toolCount }, (_, index) => { + const item = parentNarrativeToolCall(index); + return { ...item, record: { ...item.record, message_id: message.id }, sequence: 2, sortOrder: index }; + }), + }), + finalizeCompatibility: () => messageRepo.publishAssistant(message.id), + }; +} + describe("CanonicalAgentEventSink", () => { let db: Database; let published: ReturnType void>>; @@ -858,7 +882,7 @@ describe("CanonicalAgentEventSink", () => { expect(messageRepo.listByThread(THREAD_ID, 10).messages).toHaveLength(2); }); - it("resumes terminal batches when a recovered item changes", async () => { + it("resumes terminal batches over large history when a recovered item changes", async () => { const messageRepo = new MessageRepo(db); let interruptPublication = false; let interrupted = false; @@ -888,6 +912,8 @@ describe("CanonicalAgentEventSink", () => { providerIdentities: [], projectUserMessage: () => messageRepo.create(THREAD_ID, "user", "question", 1), }); + const historyCount = 2_048; + seedCanonicalItemHistory(db, historyCount); interruptPublication = true; const message = messageRepo.create(THREAD_ID, "assistant", "answer", 2, undefined, undefined, undefined, undefined, true); const narrative = Array.from({ length: 100 }, (_, index) => ({ @@ -944,10 +970,10 @@ describe("CanonicalAgentEventSink", () => { expect(result.outcome).toBe("committed"); expect(sink.loadCheckpoint(EXECUTION_ID)).toMatchObject({ terminalOutcome: "completed", - lastAcceptedSequence: 107, - lastDurableSequence: 107, + lastAcceptedSequence: historyCount + 107, + lastDurableSequence: historyCount + 107, }); - expect(db.prepare("SELECT COUNT(*) AS count FROM canonical_agent_events").get()).toEqual({ count: 107 }); + expect(db.prepare("SELECT COUNT(*) AS count FROM canonical_agent_events").get()).toEqual({ count: historyCount + 107 }); expect(messageRepo.listByThread(THREAD_ID, 10).messages).toHaveLength(2); expect(sink.loadConversationProjection(THREAD_ID, 10).messages).toHaveLength(2); expect(sink.loadConversationProjection(THREAD_ID, 10).narrativeByMessage[message.id]?.tools) @@ -965,11 +991,110 @@ describe("CanonicalAgentEventSink", () => { expect(db.prepare(` SELECT DISTINCT durable_revision FROM canonical_agent_events - WHERE accepted_sequence > 4 - `).all()).toEqual([{ durable_revision: 2 }]); + WHERE accepted_sequence > ? + `).all(historyCount + 4)).toEqual([{ durable_revision: 2 }]); expect(sink.loadThread(THREAD_ID)?.conversationRevision).toBe(2); }); + it("rolls back a failed terminal batch and resumes without publishing its writes", async () => { + startCanonicalParent(sink, db); + published.mockClear(); + const input = parentTerminalInput(db, ACTIVE_TURN_WRITE_BATCH_LIMITS.maxRows + 1); + let batches = 0; + await expect(sink.finishParentTurnBatched(input, () => { + batches += 1; + if (batches === 2) throw new Error("terminal batch write failed"); + })).rejects.toThrow("terminal batch write failed"); + + const committed = published.mock.calls.flatMap(([events]) => events); + expect(committed.length).toBeGreaterThan(0); + expect(sink.loadCheckpoint(EXECUTION_ID)).toMatchObject({ + lastAcceptedSequence: committed.length + 4, + lastDurableSequence: committed.length + 4, + terminalOutcome: null, + }); + expect(db.prepare("SELECT COUNT(*) AS count FROM canonical_agent_events").get()) + .toEqual({ count: committed.length + 4 }); + expect(sink.loadTurn(TURN_ID)?.status).toBe("Running"); + expect(new MessageRepo(db).listByThread(THREAD_ID, 10).messages).toHaveLength(1); + + await sink.finishParentTurnBatched(input); + const events = published.mock.calls.flatMap(([batch]) => batch); + expect(events.map((event) => event.acceptedSequence)) + .toEqual(Array.from({ length: 67 }, (_, index) => index + 5)); + expect(new Set(events.map((event) => event.eventId)).size).toBe(67); + expect(sink.loadCheckpoint(EXECUTION_ID)?.terminalOutcome).toBe("completed"); + expect(new MessageRepo(db).listByThread(THREAD_ID, 10).messages).toHaveLength(2); + }); + + it("rejects terminal sequence collisions from a stale checkpoint before publication", async () => { + startCanonicalParent(sink, db); + seedCanonicalItemHistory(db, 1); + db.prepare("UPDATE canonical_agent_ingest_checkpoints SET last_accepted_sequence = 4, last_durable_sequence = 4").run(); + published.mockClear(); + + await expect(sink.finishParentTurnBatched(parentTerminalInput(db))).rejects.toThrow("sequence-conflict"); + + expect(published).not.toHaveBeenCalled(); + expect(db.prepare("SELECT COUNT(*) AS count FROM canonical_agent_events").get()).toEqual({ count: 5 }); + expect(sink.loadTurn(TURN_ID)?.status).toBe("Running"); + expect(sink.loadCheckpoint(EXECUTION_ID)?.lastAcceptedSequence).toBe(4); + expect(new MessageRepo(db).listByThread(THREAD_ID, 10).messages).toHaveLength(1); + }); + + it("continues terminal sequences from durable events when the checkpoint is absent", async () => { + startCanonicalParent(sink, db); + seedCanonicalItemHistory(db, 256); + db.prepare("DELETE FROM canonical_agent_ingest_checkpoints WHERE execution_id = ?").run(EXECUTION_ID); + published.mockClear(); + + const result = await sink.finishParentTurnBatched(parentTerminalInput(db)); + + expect(result.events.map((event) => event.acceptedSequence)).toEqual([261, 262]); + expect(sink.loadCheckpoint(EXECUTION_ID)).toMatchObject({ + lastAcceptedSequence: 262, + lastDurableSequence: 262, + terminalOutcome: "completed", + }); + }); + + it("reloads thread and turn fields changed by another connection between terminal batches", async () => { + const directory = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "mcode-terminal-freshness-")); + const path = NodePath.join(directory, "app.sqlite"); + const writerDb = openDatabase({ dbPath: path }); + const otherDb = openDatabase({ dbPath: path }); + try { + seedThread(writerDb); + let finishing = false; + let mutated = false; + const writer = new CanonicalAgentEventSink(writerDb, () => { + if (!finishing || mutated) return; + mutated = true; + otherDb.prepare("UPDATE canonical_agent_threads SET roster_revision = 17 WHERE id = ?").run(THREAD_ID); + otherDb.prepare("UPDATE canonical_agent_turns SET approval_review_reason = 'changed-between-batches' WHERE id = ?").run(TURN_ID); + otherDb.prepare("UPDATE canonical_agent_ingest_checkpoints SET native_cursor_json = ? WHERE execution_id = ?") + .run(JSON.stringify({ position: "changed-between-batches" }), EXECUTION_ID); + }); + startCanonicalParent(writer, writerDb); + finishing = true; + + const result = await writer.finishParentTurnBatched(parentTerminalInput(writerDb, ACTIVE_TURN_WRITE_BATCH_LIMITS.maxRows + 1)); + + expect(mutated).toBe(true); + expect(result.writeBatches.batches).toBeGreaterThan(1); + expect(writer.loadThread(THREAD_ID)?.rosterRevision).toBe(17); + expect(writer.loadTurn(TURN_ID)).toMatchObject({ + status: "Completed", + approvalReviewReason: "changed-between-batches", + }); + expect(writer.loadCheckpoint(EXECUTION_ID)?.nativeCursor).toEqual({ position: "changed-between-batches" }); + } finally { + otherDb.close(true); + writerDb.close(true); + NodeFS.rmSync(directory, { recursive: true, force: true }); + } + }); + it("commits terminal projections above the semantic event limit in bounded batches", async () => { const messageRepo = new MessageRepo(db); sink.startParentTurn({ diff --git a/apps/server/src/features/agents/canonical/canonical-agent-boundary.ts b/apps/server/src/features/agents/canonical/canonical-agent-boundary.ts index 3e94eedfd..1edf36c69 100644 --- a/apps/server/src/features/agents/canonical/canonical-agent-boundary.ts +++ b/apps/server/src/features/agents/canonical/canonical-agent-boundary.ts @@ -2142,16 +2142,42 @@ export class CanonicalAgentBoundary implements ParentTurnDurability, CodexCollab state: ParentTerminalBatchState, input: CanonicalParentTurnFinishInput, ): void { - state.modelState = this.loadState(input.threadId, input.executionId); - state.checkpoint = this.loadCheckpoint(input.executionId); + this.loadParentTerminalBatchState(state, input); state.acceptedAt = new Date().toISOString(); - state.acceptedSequence = state.checkpoint?.lastAcceptedSequence ?? 0; state.changed = false; state.wrote = false; state.terminal = false; state.ignoredTerminal = false; } + private loadParentTerminalBatchState( + state: ParentTerminalBatchState, + input: CanonicalParentTurnFinishInput, + ): void { + const modelState = createAgentModelState(); + const thread = this.loadThread(input.threadId); + if (thread) modelState.threads[thread.id] = thread; + const turnRow = this.loadCommitTurnStatement.get({ id: input.turnId, threadId: input.threadId }); + if (turnRow) { + const turn = this.turnFromRow(turnRow); + modelState.turns[turn.id] = turn; + } + state.checkpoint = this.loadCheckpoint(input.executionId); + state.acceptedSequence = state.checkpoint?.lastAcceptedSequence + ?? this.lastAcceptedSequence(input.threadId, input.executionId); + if (state.acceptedSequence > 0) { + modelState.lastAcceptedSequenceByExecution[input.executionId] = state.acceptedSequence; + } + // Every draft costs at least one row, bounding the sequences this transaction can accept. + this.addSequenceCollisions( + modelState, + input.executionId, + state.acceptedSequence + 1, + state.acceptedSequence + ACTIVE_TURN_WRITE_BATCH_LIMITS.maxRows, + ); + state.modelState = modelState; + } + private writeParentTerminalBatchDraft( state: ParentTerminalBatchState, input: CanonicalParentTurnFinishInput, @@ -3011,15 +3037,7 @@ export class CanonicalAgentBoundary implements ParentTurnDurability, CodexCollab const acceptedSequences = events.map((event) => event.acceptedSequence); const firstAcceptedSequence = Math.min(...acceptedSequences); const lastAcceptedSequence = Math.max(...acceptedSequences); - const existing = this.loadCommitSequenceCollisionStatement.all({ - executionId: first.routing.executionId, - minSequence: firstAcceptedSequence, - maxSequence: lastAcceptedSequence, - }); - for (const event of existing) { - state.appliedEventIds[event.eventId] = true; - state.acceptedInputEventIds[`${first.routing.executionId}:${event.acceptedSequence}`] = event.eventId; - } + this.addSequenceCollisions(state, first.routing.executionId, firstAcceptedSequence, lastAcceptedSequence); const acceptedSequence = checkpoint?.lastAcceptedSequence ?? this.lastAcceptedSequence( first.routing.threadId, @@ -3030,6 +3048,19 @@ export class CanonicalAgentBoundary implements ParentTurnDurability, CodexCollab } } + private addSequenceCollisions( + state: AgentModelState, + executionId: string, + minSequence: number, + maxSequence: number, + ): void { + const existing = this.loadCommitSequenceCollisionStatement.all({ executionId, minSequence, maxSequence }); + for (const event of existing) { + state.appliedEventIds[event.eventId] = true; + state.acceptedInputEventIds[`${executionId}:${event.acceptedSequence}`] = event.eventId; + } + } + private lastAcceptedSequence(threadId: string, executionId: string): number { const row = this.loadLastAcceptedSequenceStatement.get({ threadId, executionId }); return row?.acceptedSequence ?? 0; From fc892ceccd216837c5f3f4f314aeb9cff3b9e3c2 Mon Sep 17 00:00:00 2001 From: "(CJ) Chukwudi Nwobodo" <142016413+chuks-qua@users.noreply.github.com> Date: Fri, 25 Sep 2026 21:36:43 +0100 Subject: [PATCH 118/136] perf(server): reuse narrative message resolutions and queries --- .../conversation-display-materializer.test.ts | 192 ++++++++++++++++++ .../conversation-display-materializer.ts | 164 ++++++++++----- 2 files changed, 308 insertions(+), 48 deletions(-) diff --git a/apps/server/src/features/agents/conversation/migrations/__tests__/conversation-display-materializer.test.ts b/apps/server/src/features/agents/conversation/migrations/__tests__/conversation-display-materializer.test.ts index f89a4abb8..5dafa480f 100644 --- a/apps/server/src/features/agents/conversation/migrations/__tests__/conversation-display-materializer.test.ts +++ b/apps/server/src/features/agents/conversation/migrations/__tests__/conversation-display-materializer.test.ts @@ -69,6 +69,29 @@ function insertItem( `).run(id, THREAD_ID, TURN_ID, JSON.stringify(payload), createdAt, createdAt); } +function recoveryThought(messageId?: string, id = "thought-1"): Record { + return { + projection: "narrativeRecovery", + narrative: { + kind: "narrationSegment", + record: { + id, + message_id: messageId, + text: "Working", + started_at: NOW, + sort_order: 0, + }, + }, + }; +} + +function directThought(id: string, messageId: string): Record { + return { + projection: "narrationSegment", + record: { id, message_id: messageId, text: id, started_at: NOW, sort_order: 0 }, + }; +} + describe("ConversationDisplayMaterializer", () => { let db: Database; @@ -81,6 +104,175 @@ describe("ConversationDisplayMaterializer", () => { db.close(); }); + it("keeps explicit anchors while choosing the earliest visible assistant for mixed child rows", () => { + insertItem(db, "hidden", { + projection: "message", message: { ...message("hidden-1", "assistant", 0), is_internal: true }, + }); + insertItem(db, "prompt", { projection: "message", message: message("prompt-1", "user", 1) }); + insertItem(db, "first", { projection: "message", message: message("assistant-a", "assistant", 2) }); + insertItem(db, "tied", { projection: "message", message: message("assistant-z", "assistant", 2) }); + insertItem(db, "last", { projection: "message", message: message("assistant-last", "assistant", 3) }); + insertItem(db, "direct-1", directThought("explicit-1", "assistant-last")); + insertItem(db, "direct-2", directThought("explicit-2", "assistant-last")); + insertItem(db, "recovery", recoveryThought(undefined, "implicit-1")); + insertItem(db, "child-reasoning", { projection: "codexChildReasoning", nativeItemId: "reasoning", content: "Thinking" }); + insertItem(db, "child-call", { projection: "codexChildToolCall", nativeItemId: "tool", toolName: "Read", toolInput: {} }); + insertItem(db, "child-result", { projection: "codexChildToolResult", nativeItemId: "tool", output: "contents" }); + insertItem(db, "hook", { + projection: "hook", + record: { id: "hook-1", message_id: "assistant-last", hook_name: "Stop", phase: "stop", started_at: NOW }, + }); + + new ConversationDisplayMaterializer(db).materializeItems(["last"]); + + expect(db.prepare("SELECT id, message_id FROM thought_segments WHERE id NOT LIKE 'codex-child-%' ORDER BY id").all()).toEqual([ + { id: "explicit-1", message_id: "assistant-last" }, + { id: "explicit-2", message_id: "assistant-last" }, + { id: "implicit-1", message_id: "assistant-a" }, + ]); + expect(db.prepare("SELECT message_id, text FROM thought_segments WHERE id LIKE 'codex-child-%'").all()) + .toEqual([{ message_id: "assistant-a", text: "Thinking" }]); + expect(db.prepare("SELECT message_id, output_summary, status FROM tool_call_records").all()) + .toEqual([{ message_id: "assistant-a", output_summary: "contents", status: "completed" }]); + expect(db.prepare("SELECT id, message_id FROM hook_executions").all()) + .toEqual([{ id: "hook-1", message_id: "assistant-last" }]); + expect(db.prepare("SELECT id FROM messages ORDER BY id").all()) + .toEqual([{ id: "assistant-a" }, { id: "assistant-last" }]); + }); + + it("preserves message updates between different resolutions of a shared canonical message ID", () => { + insertItem(db, "a-explicit-source", { + projection: "message", message: { ...message("shared", "assistant", 3), content: "Explicit source" }, + }); + insertItem(db, "b-anchor-source", { + projection: "message", message: { ...message("shared", "assistant", 2), content: "Anchor source" }, + }); + insertItem(db, "child-a", directThought("explicit-a", "shared")); + insertItem(db, "child-b", { projection: "codexChildReasoning", nativeItemId: "reasoning", content: "Thinking" }); + insertItem(db, "child-c", directThought("explicit-c", "shared")); + const materializer = new ConversationDisplayMaterializer(db); + materializer.materializeItems(["a-explicit-source"]); + expect(db.prepare("SELECT content FROM messages WHERE id = 'shared'").get()).toEqual({ content: "Explicit source" }); + + insertItem(db, "child-d", recoveryThought(undefined, "implicit")); + materializer.materializeItems(["a-explicit-source"]); + expect(db.prepare("SELECT content FROM messages WHERE id = 'shared'").get()).toEqual({ content: "Explicit source" }); + }); + + it("rolls back a failed child traversal and resolves current messages when the same materializer retries", () => { + insertItem(db, "assistant", { projection: "message", message: message("assistant-1", "assistant", 2) }); + insertItem(db, "a-recovery", recoveryThought()); + insertItem(db, "b-direct", directThought("direct-1", "assistant-1")); + insertItem(db, "c-invalid", { + projection: "narrationSegment", + record: { id: "invalid-1", message_id: "assistant-1", text: "Retried", started_at: "" }, + }); + const materializer = new ConversationDisplayMaterializer(db); + const materialize = db.transaction(() => materializer.materializeItems(["assistant"])); + + expect(materialize).toThrow("Canonical thought started_at is required"); + expect(db.prepare("SELECT COUNT(*) AS count FROM messages").get()).toEqual({ count: 0 }); + expect(db.prepare("SELECT COUNT(*) AS count FROM thought_segments").get()).toEqual({ count: 0 }); + expect(db.prepare("SELECT COUNT(*) AS count FROM canonical_conversation_display_mappings").get()).toEqual({ count: 0 }); + + db.prepare("UPDATE canonical_agent_items SET payload_json = json_set(payload_json, '$.record.started_at', ?) WHERE id = 'c-invalid'").run(NOW); + db.prepare("UPDATE canonical_agent_items SET payload_json = json_set(payload_json, '$.message.content', 'New answer') WHERE id = 'assistant'").run(); + materialize(); + + expect(db.prepare("SELECT content FROM messages WHERE id = 'assistant-1'").get()).toEqual({ content: "New answer" }); + expect(db.prepare("SELECT id, message_id FROM thought_segments ORDER BY id").all()).toEqual([ + { id: "direct-1", message_id: "assistant-1" }, + { id: "invalid-1", message_id: "assistant-1" }, + { id: "thought-1", message_id: "assistant-1" }, + ]); + expect(db.prepare("SELECT COUNT(*) AS count FROM canonical_conversation_display_mappings").get()).toEqual({ count: 4 }); + }); + + it("resolves later canonical visibility and content changes without reusing a prior traversal", () => { + insertItem(db, "first", { projection: "message", message: message("assistant-first", "assistant", 2) }); + insertItem(db, "last", { projection: "message", message: message("assistant-last", "assistant", 3) }); + insertItem(db, "recovery", recoveryThought()); + insertItem(db, "direct", directThought("direct-1", "assistant-last")); + const materializer = new ConversationDisplayMaterializer(db); + materializer.materializeItems(["last"]); + expect(db.prepare("SELECT message_id FROM thought_segments WHERE id = 'thought-1'").get()) + .toEqual({ message_id: "assistant-first" }); + + db.prepare("UPDATE canonical_agent_items SET payload_json = json_set(payload_json, '$.message.is_internal', 1) WHERE id = 'first'").run(); + db.prepare("UPDATE canonical_agent_items SET payload_json = json_set(payload_json, '$.message.content', 'Updated answer') WHERE id = 'last'").run(); + materializer.materializeItems(["last"]); + + expect(db.prepare("SELECT id, message_id FROM thought_segments ORDER BY id").all()).toEqual([ + { id: "direct-1", message_id: "assistant-last" }, + { id: "thought-1", message_id: "assistant-last" }, + ]); + expect(db.prepare("SELECT content FROM messages WHERE id = 'assistant-last'").get()).toEqual({ content: "Updated answer" }); + + db.prepare("UPDATE canonical_agent_ingest_checkpoints SET terminal_outcome = NULL, phase = 'running'").run(); + insertItem(db, "prompt", { projection: "message", message: message("prompt-1", "user", 1) }); + materializer.materializeItems(["recovery", "direct"]); + expect(db.prepare("SELECT DISTINCT message_id FROM thought_segments").all()).toEqual([{ message_id: "prompt-1" }]); + }); + + it.each(["completed", "cancelled"])("keeps recovered narrative on the prompt until the turn is %s", (outcome) => { + db.prepare("UPDATE canonical_agent_ingest_checkpoints SET terminal_outcome = NULL, phase = 'running'").run(); + insertItem(db, "prompt-item", { projection: "message", message: message("prompt-1", "user", 1) }); + insertItem(db, "assistant-item", { projection: "message", message: message("assistant-1", "assistant", 2) }); + insertItem(db, "recovery-item", recoveryThought()); + + new ConversationDisplayMaterializer(db).materializeItems(["recovery-item"]); + + expect(db.prepare("SELECT id, content FROM messages").all()).toEqual([{ id: "prompt-1", content: "Question" }]); + expect(db.prepare("SELECT id, message_id, text FROM thought_segments").all()).toEqual([ + { id: "thought-1", message_id: "prompt-1", text: "Working" }, + ]); + + const recoveredMaterializer = new ConversationDisplayMaterializer(db); + recoveredMaterializer.materializeItems(["recovery-item"]); + expect(db.prepare("SELECT message_id FROM thought_segments").all()).toEqual([{ message_id: "prompt-1" }]); + + db.prepare("UPDATE canonical_agent_ingest_checkpoints SET terminal_outcome = ?, phase = ?").run(outcome, outcome); + recoveredMaterializer.materializeItems(["assistant-item"]); + + expect(db.prepare("SELECT id, message_id, text FROM thought_segments").all()).toEqual([ + { id: "thought-1", message_id: "assistant-1", text: "Working" }, + ]); + expect(db.prepare("SELECT target_kind, target_id FROM canonical_conversation_display_mappings WHERE source_item_id = 'recovery-item'").get()) + .toEqual({ target_kind: "narrationSegment", target_id: "thought-1" }); + }); + + it("defers recovery narrative with no visible anchor until a terminal message arrives", () => { + insertItem(db, "recovery-item", recoveryThought()); + const materializer = new ConversationDisplayMaterializer(db); + + materializer.materializeItems(["recovery-item"]); + + expect(db.prepare("SELECT COUNT(*) AS count FROM thought_segments").get()).toEqual({ count: 0 }); + expect(db.prepare("SELECT COUNT(*) AS count FROM canonical_conversation_display_mappings").get()).toEqual({ count: 0 }); + + insertItem(db, "assistant-item", { projection: "message", message: message("assistant-1", "assistant", 2) }); + materializer.materializeItems(["assistant-item"]); + + expect(db.prepare("SELECT id, message_id FROM thought_segments").all()).toEqual([ + { id: "thought-1", message_id: "assistant-1" }, + ]); + }); + + it.each(["missing", "hidden"])("rejects recovery narrative with an explicit %s message", (visibility) => { + insertItem(db, "prompt-item", { projection: "message", message: message("prompt-1", "user", 1) }); + if (visibility === "hidden") { + insertItem(db, "hidden-item", { + projection: "message", + message: { ...message("invalid-1", "user", 2), is_internal: true }, + }); + } + insertItem(db, "recovery-item", recoveryThought("invalid-1")); + + expect(() => new ConversationDisplayMaterializer(db).materializeItems(["recovery-item"])) + .toThrow(`Canonical narrative item recovery-item references ${visibility === "hidden" ? "a hidden" : "missing"} message invalid-1`); + expect(db.prepare("SELECT COUNT(*) AS count FROM thought_segments").get()).toEqual({ count: 0 }); + }); + it("defers unanchored canonical child rows until a terminal assistant can display them", async () => { insertItem(db, "child-tool-item", { projection: "codexChildToolCall", diff --git a/apps/server/src/features/agents/conversation/migrations/conversation-display-materializer.ts b/apps/server/src/features/agents/conversation/migrations/conversation-display-materializer.ts index 2b0bfd68b..6c9946632 100644 --- a/apps/server/src/features/agents/conversation/migrations/conversation-display-materializer.ts +++ b/apps/server/src/features/agents/conversation/migrations/conversation-display-materializer.ts @@ -51,6 +51,12 @@ interface DisplayToolRecord { sortOrder: number; } +interface NarrativeDisplayResolver { + childAnchor(): string | null; + recoveryAnchor(): string | null; + displayMessageId(value: unknown, row: CanonicalItemRow): string | null; +} + /** * Writes display-table projections for canonical conversation items before the * server admits provider work. The mapping stores identities only, so canonical @@ -63,6 +69,10 @@ export class ConversationDisplayMaterializer { private readonly insertThought: ReturnType; private readonly insertHook: ReturnType; private readonly mapSource: ReturnType; + private readonly findChildMessage: ReturnType; + private readonly findCanonicalMessage: ReturnType; + private readonly findDisplayMessage: ReturnType; + private readonly findTerminalCheckpoint: ReturnType; constructor(private readonly db: Database) { this.orm = drizzle(db); @@ -71,6 +81,10 @@ export class ConversationDisplayMaterializer { this.insertThought = this.buildInsertThought(); this.insertHook = this.buildInsertHook(); this.mapSource = this.buildMapSource(); + this.findChildMessage = this.buildFindChildMessage(); + this.findCanonicalMessage = this.buildFindCanonicalMessage(); + this.findDisplayMessage = this.buildFindDisplayMessage(); + this.findTerminalCheckpoint = this.buildFindTerminalCheckpoint(); } private buildInsertMessage() { @@ -377,9 +391,9 @@ export class ConversationDisplayMaterializer { return bounded; } - private materializeRow(row: CanonicalItemRow): void { + private materializeRow(row: CanonicalItemRow, resolver?: NarrativeDisplayResolver): void { const payload = parsePayload(row); - const target = this.materializePayload(row, payload); + const target = this.materializePayload(row, payload, resolver); if (!target) return; this.mapSource.run({ sourceItemId: row.id, @@ -393,16 +407,17 @@ export class ConversationDisplayMaterializer { private materializePayload( row: CanonicalItemRow, payload: NarrativePayload, + resolver?: NarrativeDisplayResolver, ): { kind: string; id: string } | null { switch (payload.projection) { case "message": return this.materializeMessagePayload(row, payload.message); - case "toolCall": return this.materializeDirectNarrative(row, payload, "toolCall"); - case "narrationSegment": return this.materializeDirectNarrative(row, payload, "narrationSegment"); - case "hook": return this.materializeDirectNarrative(row, payload, "hook"); - case "narrativeRecovery": return this.materializeRecoveryPayload(row, payload.narrative); - case "codexChildReasoning": return this.materializeChildReasoning(row, payload); + case "toolCall": return this.materializeDirectNarrative(row, payload, "toolCall", resolver); + case "narrationSegment": return this.materializeDirectNarrative(row, payload, "narrationSegment", resolver); + case "hook": return this.materializeDirectNarrative(row, payload, "hook", resolver); + case "narrativeRecovery": return this.materializeRecoveryPayload(row, payload.narrative, resolver); + case "codexChildReasoning": return this.materializeChildReasoning(row, payload, resolver); case "codexChildToolCall": - case "codexChildToolResult": return this.materializeChildTool(row, payload); + case "codexChildToolResult": return this.materializeChildTool(row, payload, resolver); default: return null; } } @@ -410,8 +425,9 @@ export class ConversationDisplayMaterializer { private materializeRecoveryPayload( row: CanonicalItemRow, narrative: NarrativePayload["narrative"], + resolver?: NarrativeDisplayResolver, ): { kind: string; id: string } | null { - return narrative?.record ? this.materializeRecoveryNarrative(row, narrative) : null; + return narrative?.record ? this.materializeRecoveryNarrative(row, narrative, resolver) : null; } private materializeMessagePayload( @@ -428,9 +444,10 @@ export class ConversationDisplayMaterializer { row: CanonicalItemRow, payload: NarrativePayload, kind: "toolCall" | "narrationSegment" | "hook", + resolver?: NarrativeDisplayResolver, ): { kind: string; id: string } | null { if (!payload.record) return null; - const record = this.withDisplayMessage(payload.record, row); + const record = this.withDisplayMessage(payload.record, row, resolver); return record ? this.materializeNarrativeRecord(kind, record) : null; } @@ -454,17 +471,21 @@ export class ConversationDisplayMaterializer { private materializeRecoveryNarrative( row: CanonicalItemRow, narrative: { kind?: string; record?: Record }, + resolver?: NarrativeDisplayResolver, ): { kind: string; id: string } | null { const record = { ...narrative.record! }; requiredString(record.id, "recovery narrative id"); + let visibleRecord: Record | null; if (typeof record.message_id !== "string" || record.message_id.length === 0) { // Recovery snapshots intentionally omit a staged assistant message. Keep // their visible rows on the prompt until a terminal assistant can own them. - const anchor = this.childAnchor(row); + const anchor = resolver ? resolver.recoveryAnchor() : this.recoveryAnchor(row); if (!anchor) return null; record.message_id = anchor; + visibleRecord = record; + } else { + visibleRecord = this.withDisplayMessage(record, row, resolver); } - const visibleRecord = this.withDisplayMessage(record, row); if (!visibleRecord) return null; return narrative.kind === "toolCall" || narrative.kind === "narrationSegment" || narrative.kind === "hook" ? this.materializeNarrativeRecord(narrative.kind, visibleRecord) @@ -569,8 +590,9 @@ export class ConversationDisplayMaterializer { private materializeChildReasoning( row: CanonicalItemRow, payload: NarrativePayload, + resolver?: NarrativeDisplayResolver, ): { kind: string; id: string } | null { - const messageId = this.childAnchor(row); + const messageId = resolver ? resolver.childAnchor() : this.childAnchor(row); if (!messageId) return null; const id = `codex-child-reasoning:${hashCodexKey(`${payload.nativeItemId ?? row.payloadJson}:${row.id}`)}`; const existing = this.orm.select({ sortOrder: thoughtSegments.sortOrder }) @@ -593,8 +615,9 @@ export class ConversationDisplayMaterializer { private materializeChildTool( row: CanonicalItemRow, payload: NarrativePayload, + resolver?: NarrativeDisplayResolver, ): { kind: string; id: string } | null { - const messageId = this.childAnchor(row); + const messageId = resolver ? resolver.childAnchor() : this.childAnchor(row); if (!messageId) return null; const nativeItemId = payload.nativeItemId ?? row.payloadJson; const id = `codex-child-tool:${hashCodexKey(nativeItemId)}`; @@ -659,7 +682,19 @@ export class ConversationDisplayMaterializer { } private childAnchor(row: CanonicalItemRow): string | null { - const childMessage = this.orm.select({ + const childMessage = this.findChildMessage.get({ threadId: row.threadId, turnId: row.turnId }); + // Historical child events may be complete while their provider never + // emitted a visible message. They have no display projection yet, but the + // canonical row remains intact and a later terminal message reanchors it. + if (!childMessage) return null; + const payload = parsePayload(childMessage); + if (!payload.message) throw new Error(`Canonical child anchor ${childMessage.id} has no message payload`); + this.materializeMessage(payload.message); + return payload.message.id; + } + + private buildFindChildMessage() { + return this.orm.select({ id: canonicalAgentItems.id, threadId: canonicalAgentItems.threadId, turnId: canonicalAgentItems.turnId, @@ -668,8 +703,8 @@ export class ConversationDisplayMaterializer { updatedAt: canonicalAgentItems.updatedAt, }).from(canonicalAgentItems) .where(and( - eq(canonicalAgentItems.threadId, row.threadId), - eq(canonicalAgentItems.turnId, row.turnId), + eq(canonicalAgentItems.threadId, placeholder("threadId")), + eq(canonicalAgentItems.turnId, placeholder("turnId")), eq(canonicalAgentItems.kind, "message"), sql`json_extract(${canonicalAgentItems.payloadJson}, '$.projection') = 'message'`, sql`COALESCE(json_extract(${canonicalAgentItems.payloadJson}, '$.message.is_internal'), 0) = 0`, @@ -688,45 +723,24 @@ export class ConversationDisplayMaterializer { sql`json_extract(${canonicalAgentItems.payloadJson}, '$.message.id') ASC`, ) .limit(1) - .get(); - // Historical child events may be complete while their provider never - // emitted a visible message. They have no display projection yet, but the - // canonical row remains intact and a later terminal message reanchors it. - if (!childMessage) return null; - const payload = parsePayload(childMessage); - if (!payload.message) throw new Error(`Canonical child anchor ${childMessage.id} has no message payload`); - this.materializeMessage(payload.message); - return payload.message.id; + .prepare(); } private withDisplayMessage( record: Record, row: CanonicalItemRow, + resolver?: NarrativeDisplayResolver, ): Record | null { - const messageId = this.displayMessageId(record.message_id, row); + const messageId = resolver + ? resolver.displayMessageId(record.message_id, row) + : this.displayMessageId(record.message_id, row); return messageId ? { ...record, message_id: messageId } : null; } private displayMessageId(value: unknown, row: CanonicalItemRow): string | null { const messageId = requiredString(value, "narrative message_id"); - const exists = this.orm.select({ id: messages.id }).from(messages) - .where(eq(messages.id, messageId)) - .get(); - const source = this.orm.select({ - id: canonicalAgentItems.id, - threadId: canonicalAgentItems.threadId, - turnId: canonicalAgentItems.turnId, - payloadJson: canonicalAgentItems.payloadJson, - createdAt: canonicalAgentItems.createdAt, - updatedAt: canonicalAgentItems.updatedAt, - }).from(canonicalAgentItems) - .where(and( - eq(canonicalAgentItems.threadId, row.threadId), - sql`json_extract(${canonicalAgentItems.payloadJson}, '$.projection') = 'message'`, - sql`json_extract(${canonicalAgentItems.payloadJson}, '$.message.id') = ${messageId}`, - )) - .limit(1) - .get(); + const exists = this.findDisplayMessage.get({ messageId }); + const source = this.findCanonicalMessage.get({ threadId: row.threadId, messageId }); if (!source) { if (exists) return messageId; throw new Error(`Canonical narrative item ${row.id} references missing message ${messageId}`); @@ -741,6 +755,30 @@ export class ConversationDisplayMaterializer { throw new Error(`Canonical narrative item ${row.id} references a hidden message ${messageId}`); } + private buildFindDisplayMessage() { + return this.orm.select({ id: messages.id }).from(messages) + .where(eq(messages.id, placeholder("messageId"))) + .prepare(); + } + + private buildFindCanonicalMessage() { + return this.orm.select({ + id: canonicalAgentItems.id, + threadId: canonicalAgentItems.threadId, + turnId: canonicalAgentItems.turnId, + payloadJson: canonicalAgentItems.payloadJson, + createdAt: canonicalAgentItems.createdAt, + updatedAt: canonicalAgentItems.updatedAt, + }).from(canonicalAgentItems) + .where(and( + eq(canonicalAgentItems.threadId, placeholder("threadId")), + sql`json_extract(${canonicalAgentItems.payloadJson}, '$.projection') = 'message'`, + sql`json_extract(${canonicalAgentItems.payloadJson}, '$.message.id') = ${placeholder("messageId")}`, + )) + .limit(1) + .prepare(); + } + private nextChildThoughtSortOrder(messageId: string): number { return this.orm.select({ count: count() }).from(thoughtSegments) .where(and( @@ -766,14 +804,18 @@ export class ConversationDisplayMaterializer { } private turnHasTerminalCheckpoint(turnId: string): boolean { + return this.findTerminalCheckpoint.get({ turnId }) != null; + } + + private buildFindTerminalCheckpoint() { return this.orm.select({ executionId: canonicalAgentIngestCheckpoints.executionId }) .from(canonicalAgentIngestCheckpoints) .where(and( - eq(canonicalAgentIngestCheckpoints.turnId, turnId), + eq(canonicalAgentIngestCheckpoints.turnId, placeholder("turnId")), isNotNull(canonicalAgentIngestCheckpoints.terminalOutcome), )) .limit(1) - .get() != null; + .prepare(); } private reanchorTurnChildren(row: CanonicalItemRow): void { @@ -796,7 +838,33 @@ export class ConversationDisplayMaterializer { )) .orderBy(asc(canonicalAgentItems.createdAt), asc(canonicalAgentItems.id)) .all(); - for (const child of children) this.materializeRow(child); + const resolver = this.reanchorMessageResolver(row); + for (const child of children) this.materializeRow(child, resolver); + } + + private recoveryAnchor(row: CanonicalItemRow): string | null { + const anchor = this.childAnchor(row); + return anchor ? this.displayMessageId(anchor, row) : null; + } + + private reanchorMessageResolver(row: CanonicalItemRow): NarrativeDisplayResolver { + // Resolution also updates the display message. Only consecutive identical + // lookups can skip that write when canonical sources share a message ID. + let previous: { key: string; messageId: string | null } | undefined; + const resolve = (key: string, read: () => string | null): string | null => { + if (previous?.key === key) return previous.messageId; + const messageId = read(); + previous = { key, messageId }; + return messageId; + }; + return { + childAnchor: () => resolve("child", () => this.childAnchor(row)), + recoveryAnchor: () => resolve("recovery", () => this.recoveryAnchor(row)), + displayMessageId: (value, child) => { + const messageId = requiredString(value, "narrative message_id"); + return resolve(`message:${messageId}`, () => this.displayMessageId(messageId, child)); + }, + }; } } From 84af532cb7f87ab06d1ef410ee39c393a9a9f793 Mon Sep 17 00:00:00 2001 From: "(CJ) Chukwudi Nwobodo" <142016413+chuks-qua@users.noreply.github.com> Date: Fri, 25 Sep 2026 21:36:44 +0100 Subject: [PATCH 119/136] perf(contracts): omit unused schema modules from bundles --- packages/contracts/package.json | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/contracts/package.json b/packages/contracts/package.json index 56d481a6a..3d6b36c8f 100644 --- a/packages/contracts/package.json +++ b/packages/contracts/package.json @@ -3,6 +3,7 @@ "version": "0.0.1", "private": true, "type": "module", + "sideEffects": false, "exports": { ".": "./src/index.ts", "./snapshot": "./src/snapshot.ts" From 2a08ac74a434f07fb2304e74ac11fc38555cfce2 Mon Sep 17 00:00:00 2001 From: "(CJ) Chukwudi Nwobodo" <142016413+chuks-qua@users.noreply.github.com> Date: Fri, 25 Sep 2026 21:57:44 +0100 Subject: [PATCH 120/136] perf(server): group bounded independent event commits --- .../__tests__/canonical-append-group.test.ts | 234 ++++++++++++++++++ .../canonical-agent-writer.worker.ts | 105 ++++++-- .../canonical/canonical-append-group.ts | 37 +++ .../canonical-execution-semantic-writer.ts | 72 +++++- 4 files changed, 416 insertions(+), 32 deletions(-) create mode 100644 apps/server/src/features/agents/canonical/__tests__/canonical-append-group.test.ts create mode 100644 apps/server/src/features/agents/canonical/canonical-append-group.ts diff --git a/apps/server/src/features/agents/canonical/__tests__/canonical-append-group.test.ts b/apps/server/src/features/agents/canonical/__tests__/canonical-append-group.test.ts new file mode 100644 index 000000000..0cc1e2a45 --- /dev/null +++ b/apps/server/src/features/agents/canonical/__tests__/canonical-append-group.test.ts @@ -0,0 +1,234 @@ +import "reflect-metadata"; +import { Database } from "bun:sqlite"; +import * as NodeFS from "node:fs"; +import * as NodeOS from "node:os"; +import * as NodePath from "node:path"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { openDatabase } from "../../../../runtime/persistence/sqlite/database.js"; +import type { ExecutionSemanticOperation } from "../../execution/execution-worker-handler.js"; +import { APPEND_GROUP_LIMITS, selectAppendGroup, type QueuedCanonicalWrite } from "../canonical-append-group.js"; +import { CanonicalExecutionSemanticWriter } from "../canonical-execution-semantic-writer.js"; +import { CanonicalAgentWriterClient } from "../canonical-agent-writer-client.js"; + +const NOW = "2026-09-25T12:00:00.000Z"; + +function append(index: number, ordinal = 2): ExecutionSemanticOperation { + const execution = { threadId: `thread-${index}`, turnId: `turn-${index}`, + executionId: `00000000-0000-4000-8000-${String(index).padStart(12, "0")}` }; + const itemId = `item-${index}-${ordinal}`; + return { + execution, lease: { ownerEpoch: 1, workerIndex: index, workerGeneration: 1, leaseId: `lease-${index}` }, + operationId: `lease-${index}:${ordinal}`, ordinal, + mutation: { kind: "append-events", phase: "running", nativeCursor: null, events: [{ + eventId: `${execution.executionId}:item-${ordinal}`, routing: { ...execution, itemId }, + sourceProviderId: "codex", sourceIdentities: [], + payload: { type: "item.recorded", item: { id: itemId, threadId: execution.threadId, + turnId: execution.turnId, kind: "message", providerIdentities: [], + payload: { projection: "message", content: "Durable event" }, createdAt: NOW, updatedAt: NOW } }, + }] }, + }; +} + +function queued(operation: ExecutionSemanticOperation, bytes = 100): QueuedCanonicalWrite { + return { bytes, request: { kind: "semantic-transact", requestId: `request-${operation.operationId}`, + operationId: operation.operationId, executionId: operation.execution.executionId, operation } }; +} + +describe("bounded append groups", () => { + it("preserves FIFO across repeated identities, controls, mismatched routing, and reclassification", () => { + const first = queued(append(1)); + const second = queued(append(2)); + const repeated = queued(append(1, 3)); + const control = queued({ ...append(3), mutation: { kind: "checkpoint", phase: "running", nativeCursor: null } }); + expect(selectAppendGroup([first, second, repeated, queued(append(4))])).toEqual([first.request, second.request]); + expect(selectAppendGroup([first, control, second])).toEqual([first.request]); + const mismatched: QueuedCanonicalWrite = { ...second, request: { ...second.request, executionId: "wrong" } }; + expect(selectAppendGroup([first, mismatched, queued(append(4))])).toEqual([first.request]); + const source = append(3); + if (source.mutation.kind !== "append-events") throw new Error("Expected append"); + const reclassify = queued({ ...source, mutation: { ...source.mutation, + parentLive: { text: { kind: "reclassify", expectedText: "" } } } }); + expect(selectAppendGroup([first, reclassify, second])).toEqual([first.request]); + }); + + it("caps count and bytes without consuming the rest of the queue", () => { + const queue = Array.from({ length: 10 }, (_, index) => queued(append(index + 1))); + expect(selectAppendGroup(queue)).toHaveLength(APPEND_GROUP_LIMITS.operations); + expect(queue).toHaveLength(10); + expect(selectAppendGroup([queued(append(1), APPEND_GROUP_LIMITS.bytes), queued(append(2))])).toHaveLength(1); + }); +}); + +describe("file-backed grouped semantic commits", () => { + let directory: string; + let db: Database; + let observer: Database; + let writer: CanonicalExecutionSemanticWriter; + let published: string[]; + + beforeEach(async () => { + directory = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "mcode-append-group-")); + const path = NodePath.join(directory, "app.sqlite"); + db = openDatabase({ dbPath: path }); + observer = new Database(path, { strict: true, readonly: true }); + published = []; + writer = new CanonicalExecutionSemanticWriter(db, (events) => published.push(...events.map((event) => event.eventId))); + db.prepare("INSERT INTO workspaces (id, name, path, created_at, updated_at) VALUES (?, ?, ?, ?, ?)") + .run("workspace-1", "Fixture", "C:/fixture", NOW, NOW); + for (const index of [1, 2]) { + const operation = append(index); + db.prepare("INSERT INTO threads (id, workspace_id, title, branch, provider, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?)") + .run(operation.execution.threadId, "workspace-1", "Fixture", "main", "codex", NOW, NOW); + expect(await writer.transact({ ...operation, ordinal: 1, operationId: `lease-${index}:1`, mutation: { + kind: "begin", providerId: "codex", input: { + thread: { id: operation.execution.threadId, workspaceId: "workspace-1", providerId: "codex", createdAt: NOW }, + turnId: operation.execution.turnId, executionId: operation.execution.executionId, + permissionMode: "supervised", providerIdentities: [], + userMessage: { kind: "create", messageId: `user-${index}`, content: "Question", sequence: 1 }, + }, + } })).toMatchObject({ kind: "committed" }); + } + published.length = 0; + // A fixed clock makes the transaction tests independent of host load; SQLite remains real. + vi.spyOn(performance, "now").mockReturnValue(0); + }); + + afterEach(() => { + vi.restoreAllMocks(); + observer.close(true); + db.close(true); + NodeFS.rmSync(directory, { recursive: true, force: true }); + }); + + function persistedAppends() { + return observer.query("SELECT operation_id FROM canonical_writer_operation_receipts WHERE kind = 'semantic:append-events' ORDER BY execution_id").all(); + } + + it("returns each request's publications only after the outer commit is visible to another connection", () => { + const results = writer.transactAppendGroup([append(1), append(2)]); + expect(results).toHaveLength(2); + expect(results.map((result) => result.receipt.kind)).toEqual(["committed", "committed"]); + expect(db.inTransaction).toBe(false); + expect(persistedAppends()).toEqual([{ operation_id: "lease-1:2" }, { operation_id: "lease-2:2" }]); + expect(published).toEqual([]); + for (const result of results) { + expect(result.publications.flat().every((event) => event.routing.executionId === result.operation.execution.executionId)).toBe(true); + expect(result.publications.flat().length).toBeGreaterThan(0); + } + }); + + it("rolls back a conflicting append's savepoint while committing its peer", () => { + const conflicting = append(1); + if (conflicting.mutation.kind !== "append-events") throw new Error("Expected append"); + const results = writer.transactAppendGroup([{ ...conflicting, mutation: { ...conflicting.mutation, + parentLive: { text: { kind: "unchanged" } } } }, append(2)]); + expect(results.map((result) => result.receipt.kind)).toEqual(["conflict", "committed"]); + expect(results[0]?.publications).toEqual([]); + expect(persistedAppends()).toEqual([{ operation_id: "lease-2:2" }]); + expect(observer.query("SELECT event_id FROM canonical_agent_events WHERE event_id = ?") + .get(`${conflicting.execution.executionId}:item-2`)).toBeNull(); + }); + + it("discards all results and publications when the physical outer commit fails", () => { + db.exec(`CREATE TABLE group_commit_parent (id INTEGER PRIMARY KEY); + CREATE TABLE group_commit_child (parent_id INTEGER REFERENCES group_commit_parent(id) DEFERRABLE INITIALLY DEFERRED); + CREATE TRIGGER fail_group_commit AFTER INSERT ON canonical_writer_operation_receipts + WHEN NEW.kind = 'semantic:append-events' + BEGIN INSERT INTO group_commit_child(parent_id) VALUES (999); END;`); + expect(() => writer.transactAppendGroup([append(1), append(2)])).toThrow(); + expect(db.inTransaction).toBe(false); + expect(persistedAppends()).toEqual([]); + expect(published).toEqual([]); + db.exec("DROP TRIGGER fail_group_commit"); + expect(writer.transactAppendGroup([append(1), append(2)]).map((result) => result.receipt.kind)) + .toEqual(["committed", "committed"]); + }); + + it("replays a lost acknowledgement with the original receipts and no duplicate rows", () => { + const initial = writer.transactAppendGroup([append(1), append(2)]); + const replay = writer.transactAppendGroup([append(1), append(2)]); + expect(replay).toEqual(initial); + expect(persistedAppends()).toHaveLength(2); + expect(published).toEqual([]); + }); + + it("returns the consumed prefix when the elapsed limit ends the transaction", () => { + vi.spyOn(performance, "now").mockReturnValueOnce(0).mockReturnValue(APPEND_GROUP_LIMITS.elapsedMs); + const operations = [append(1), append(2)]; + const first = writer.transactAppendGroup(operations); + expect(first).toHaveLength(1); + expect(writer.transactAppendGroup(operations.slice(first.length))).toHaveLength(1); + expect(persistedAppends()).toHaveLength(2); + }); + + it("correlates concurrent worker requests and publishes only rows visible from a separate connection", async () => { + vi.restoreAllMocks(); + const client = new CanonicalAgentWriterClient(NodePath.join(directory, "app.sqlite")); + const observed: string[] = []; + try { + const operations = [append(1), append(2)]; + const receipts = await Promise.all(operations.map(async (operation) => { + const receipt = await client.transactSemantic(operation, (events) => { + for (const event of events) { + expect(event.routing.executionId).toBe(operation.execution.executionId); + expect(observer.query("SELECT event_id FROM canonical_agent_events WHERE event_id = ?") + .get(event.eventId)).toEqual({ event_id: event.eventId }); + expect(observer.query("SELECT operation_id FROM canonical_writer_operation_receipts WHERE execution_id = ? AND operation_id = ?") + .get(operation.execution.executionId, operation.operationId)).toEqual({ operation_id: operation.operationId }); + } + observed.push(`publication:${operation.operationId}`); + }); + observed.push(`receipt:${operation.operationId}`); + return receipt; + })); + expect(receipts.map((receipt) => receipt.kind)).toEqual(["committed", "committed"]); + for (const operation of operations) { + expect(observed.indexOf(`publication:${operation.operationId}`)).toBeGreaterThanOrEqual(0); + expect(observed.indexOf(`publication:${operation.operationId}`)).toBeLessThan(observed.indexOf(`receipt:${operation.operationId}`)); + } + expect(persistedAppends()).toHaveLength(2); + expect(observer.query("SELECT id, thread_id FROM canonical_agent_items WHERE id IN ('item-1-2', 'item-2-2') ORDER BY id").all()) + .toEqual([{ id: "item-1-2", thread_id: "thread-1" }, { id: "item-2-2", thread_id: "thread-2" }]); + } finally { + await client.close(); + } + }); + + it("keeps a concurrent worker peer usable when one execution's database write fails", async () => { + vi.restoreAllMocks(); + db.exec(`CREATE TRIGGER reject_first_append BEFORE INSERT ON canonical_writer_operation_receipts + WHEN NEW.kind = 'semantic:append-events' AND NEW.execution_id = '${append(1).execution.executionId}' + BEGIN SELECT RAISE(ABORT, 'execution write failed'); END;`); + const client = new CanonicalAgentWriterClient(NodePath.join(directory, "app.sqlite")); + const seen: string[] = []; + try { + const results = await Promise.allSettled([1, 2].map((index) => client.transactSemantic(append(index), + (events) => seen.push(...events.map((event) => event.routing.executionId))))); + expect(results[0]).toMatchObject({ status: "rejected" }); + expect(results[1]).toMatchObject({ status: "fulfilled", value: { kind: "committed" } }); + expect(seen).not.toContain(append(1).execution.executionId); + expect(seen).toContain(append(2).execution.executionId); + expect(persistedAppends()).toEqual([{ operation_id: "lease-2:2" }]); + } finally { + await client.close(); + } + }); + + it("rejects an unserializable queued append without losing a healthy worker peer", async () => { + vi.restoreAllMocks(); + const invalid = append(1); + if (invalid.mutation.kind !== "append-events") throw new Error("Expected append"); + const operation = { ...invalid, mutation: { ...invalid.mutation, nativeCursor: 1n } }; + const client = new CanonicalAgentWriterClient(NodePath.join(directory, "app.sqlite")); + try { + const results = await Promise.allSettled([ + client.transactSemantic(operation, () => {}), client.transactSemantic(append(2), () => {}), + ]); + expect(results[0]).toMatchObject({ status: "rejected" }); + expect(results[1]).toMatchObject({ status: "fulfilled", value: { kind: "committed" } }); + expect(persistedAppends()).toEqual([{ operation_id: "lease-2:2" }]); + } finally { + await client.close(); + } + }); +}); diff --git a/apps/server/src/features/agents/canonical/canonical-agent-writer.worker.ts b/apps/server/src/features/agents/canonical/canonical-agent-writer.worker.ts index eb1393da0..321e7b61a 100644 --- a/apps/server/src/features/agents/canonical/canonical-agent-writer.worker.ts +++ b/apps/server/src/features/agents/canonical/canonical-agent-writer.worker.ts @@ -5,6 +5,8 @@ import * as NodePath from "node:path"; import { applySQLiteConnectionPolicy } from "../../../runtime/persistence/sqlite/sqlite-connection-policy.js"; import { CanonicalAgentBoundary } from "./canonical-agent-boundary.js"; import { CanonicalExecutionSemanticWriter } from "./canonical-execution-semantic-writer.js"; +import { isGroupableAppend, selectAppendGroup, type QueuedCanonicalWrite } from "./canonical-append-group.js"; +import type { CanonicalAgentEventPublisher } from "./canonical-agent-boundary.js"; import { ParentAssistantTextCheckpointService } from "../turns/parent-assistant-text-checkpoint-service.js"; import { CanonicalAgentWriterReceipts, CanonicalWriterOperationConflict, CanonicalWriterReceiptCapacity } from "./canonical-agent-writer-receipts.js"; import type { @@ -36,13 +38,7 @@ function openDatabase(dbPath: string): void { semanticWriter = new CanonicalExecutionSemanticWriter(connection, (events) => { const correlation = semanticPublication; if (!correlation) throw new Error("Semantic publication has no active request"); - for (let offset = 0; offset < events.length; offset += SEMANTIC_PUBLICATION_PAGE_SIZE) { - globalThis.postMessage({ - ...correlation, - kind: "semantic-publication", - events: events.slice(offset, offset + SEMANTIC_PUBLICATION_PAGE_SIZE), - } satisfies CanonicalWriterResponse); - } + publishSemantic(correlation, events); }); db = connection; } catch (error) { @@ -155,6 +151,16 @@ async function handleSemanticWrite( } } +function publishSemantic( + correlation: Pick, + events: Parameters[0], +): void { + for (let offset = 0; offset < events.length; offset += SEMANTIC_PUBLICATION_PAGE_SIZE) { + globalThis.postMessage({ ...correlation, kind: "semantic-publication", + events: events.slice(offset, offset + SEMANTIC_PUBLICATION_PAGE_SIZE) } satisfies CanonicalWriterResponse); + } +} + async function withSQLiteBusyRetry(work: () => Promise | T): Promise { const delaysMs = [10, 30, 90, 270, 500]; for (const delayMs of delaysMs) { @@ -223,23 +229,78 @@ function applyWrite( }; } -let requestTail = Promise.resolve(); +const requestQueue: QueuedCanonicalWrite[] = []; +let draining = false; + +function failRequest(request: CanonicalWriterRequest): void { + globalThis.postMessage({ requestId: request.requestId, operationId: request.operationId, + executionId: request.executionId, kind: "failed", reason: "write-failed" } satisfies CanonicalWriterResponse); +} + +async function handleAppendGroup(requests: Extract[]): Promise { + const first = requests[0]; + if (!first) return 0; + let results: ReturnType; + try { + const writer = semanticWriter; + if (!writer || semanticPublication) throw new Error("Semantic writer is not ready"); + results = await withSQLiteBusyRetry( + () => writer.transactAppendGroup(requests.map((request) => request.operation))); + } catch { + // A rolled-back peer must not inherit another execution's write failure. + // Replaying semantic operations is safe because their durable receipts fence duplicates. + for (const request of requests) await handleSingleRequest(request); + return requests.length; + } + for (const [index, result] of results.entries()) { + const request = requests[index]; + if (!request) throw new Error("Semantic append group lost its request"); + const correlation = { requestId: request.requestId, operationId: request.operationId, executionId: request.executionId }; + for (const events of result.publications) publishSemantic(correlation, events); + globalThis.postMessage({ ...correlation, kind: "semantic-transacted", receipt: result.receipt } satisfies CanonicalWriterResponse); + } + return results.length; +} + +async function handleSingleRequest(request: CanonicalWriterRequest): Promise { + const response = await handle(request); + globalThis.postMessage(response); +} + +async function drainRequests(): Promise { + const queued = requestQueue[0]; + if (!queued) { draining = false; return; } + const request = queued.request; + let consumed = 1; + try { + const group = selectAppendGroup(requestQueue); + if (group.length > 1) consumed = await handleAppendGroup(group); + else await handleSingleRequest(request); + } catch (error) { + console.error("Canonical writer request failed unexpectedly", error); + failRequest(request); + } finally { + requestQueue.splice(0, consumed); + if (requestQueue.length > 0) setImmediate(() => { void drainRequests(); }); + else draining = false; + } +} + globalThis.onmessage = (message: MessageEvent): void => { const request = message.data; - requestTail = requestTail.then(async () => { - try { - globalThis.postMessage(await handle(request)); - } catch (error) { - console.error("Canonical writer request failed unexpectedly", error); - globalThis.postMessage({ - requestId: request.requestId, - operationId: request.operationId, - executionId: request.executionId, - kind: "failed", - reason: "write-failed", - } satisfies CanonicalWriterResponse); - } - }); + try { + requestQueue.push({ request, + bytes: request.kind === "semantic-transact" && isGroupableAppend(request.operation) + ? Buffer.byteLength(JSON.stringify(request.operation), "utf8") : 0 }); + } catch { + failRequest(request); + return; + } + if (!draining) { + draining = true; + // Collect already-ready messages without waiting for a group to fill. + setImmediate(() => { void drainRequests(); }); + } }; process.on("exit", () => db?.close(true)); diff --git a/apps/server/src/features/agents/canonical/canonical-append-group.ts b/apps/server/src/features/agents/canonical/canonical-append-group.ts new file mode 100644 index 000000000..4e014b3e3 --- /dev/null +++ b/apps/server/src/features/agents/canonical/canonical-append-group.ts @@ -0,0 +1,37 @@ +import type { ExecutionSemanticOperation } from "../execution/execution-worker-handler.js"; +import type { CanonicalWriterRequest } from "./canonical-agent-writer-protocol.js"; + +/** Bounds one physical commit without waiting for more append requests. */ +export const APPEND_GROUP_LIMITS = { operations: 8, bytes: 512 * 1024, elapsedMs: 16 } as const; + +/** Reclassification removes a recovery journal after committing, so it remains a separate write. */ +export function isGroupableAppend(operation: ExecutionSemanticOperation): boolean { + return operation.mutation.kind === "append-events" + && operation.mutation.parentLive?.text.kind !== "reclassify"; +} + +/** Request bytes are measured once when the request enters the worker queue. */ +export interface QueuedCanonicalWrite { + readonly request: CanonicalWriterRequest; + readonly bytes: number; +} + +/** Select a FIFO prefix of independent appends; controls and repeated identities are barriers. */ +export function selectAppendGroup(queue: readonly QueuedCanonicalWrite[]): Extract[] { + const requests: Extract[] = []; + const executions = new Set(); + let bytes = 0; + for (const queued of queue) { + const request = queued.request; + if (request.kind !== "semantic-transact" || !isGroupableAppend(request.operation) + || request.operationId !== request.operation.operationId + || request.executionId !== request.operation.execution.executionId + || executions.has(request.executionId) + || requests.length === APPEND_GROUP_LIMITS.operations + || bytes + queued.bytes > APPEND_GROUP_LIMITS.bytes) break; + requests.push(request); + executions.add(request.executionId); + bytes += queued.bytes; + } + return requests; +} diff --git a/apps/server/src/features/agents/canonical/canonical-execution-semantic-writer.ts b/apps/server/src/features/agents/canonical/canonical-execution-semantic-writer.ts index f098e4314..8cfed789a 100644 --- a/apps/server/src/features/agents/canonical/canonical-execution-semantic-writer.ts +++ b/apps/server/src/features/agents/canonical/canonical-execution-semantic-writer.ts @@ -34,6 +34,7 @@ import type { } from "../execution/execution-worker-handler.js"; import type { CanonicalAgentEventPublisher } from "./canonical-agent-boundary.js"; import { CanonicalAgentBoundary } from "./canonical-agent-boundary.js"; +import { APPEND_GROUP_LIMITS, isGroupableAppend } from "./canonical-append-group.js"; import { CanonicalCommittedProviderProjector } from "./canonical-committed-provider-projector.js"; import type { ProviderEventProjection } from "../../providers/composition/provider-event-adapter.js"; import { CanonicalCodexSystemErrorProjection, matchesCodexSystemIntents } from "./canonical-codex-system-error-projection.js"; @@ -237,11 +238,12 @@ export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter private readonly updateCheckpoint: ReturnType; private bufferedPublication: Parameters[0][] | null = null; private bufferedPublicationCount = 0; + private groupedPublication: Parameters[0][] | null = null; constructor(private readonly db: Database, private readonly publish: CanonicalAgentEventPublisher) { this.turns = new CanonicalParentTurnWrite(db, (events) => { if (this.bufferedPublication) this.bufferPublication(events); - else this.publish(events); + else this.publishCommitted(events); }); // Compound terminal batches publish from their durable sequence log only after the final receipt commits. this.compoundTurns = new CanonicalParentTurnWrite(db, () => {}); @@ -275,16 +277,12 @@ export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter /** Commit a supported operation with a durable receipt, or reject it without changing canonical state. */ async transact(operation: ExecutionSemanticOperation): Promise { + if (operation.mutation.kind === "append-events") return this.transactAppend(operation); if (!validOperation(operation)) return conflict(operation); const hash = fingerprint(operation); const existing = this.existingReceipt(operation, hash); if (existing) { - if (existing.kind === "committed" && isFinishMutation(operation.mutation)) { - this.assistantText.retire(operation.execution.executionId); - } - if (existing.kind === "committed" && parentLiveOperation(operation)?.mutation.text.kind === "reclassify") { - this.assistantText.discardRecoveryJournal(operation.execution.executionId); - } + this.retireReplayedText(operation, existing); return existing; } try { @@ -346,7 +344,6 @@ export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter if (isFinishMutation(operation.mutation)) return await this.finish(operation, hash); switch (operation.mutation.kind) { case "begin": return this.begin(operation, hash); - case "append-events": return this.append(operation, hash); case "checkpoint": case "stop-requested": case "provider-outcome": return this.control(operation, hash); @@ -444,6 +441,56 @@ export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter return receipt; } + /** Commit a bounded prefix of independent appends, returning publications only after physical commit. */ + transactAppendGroup(operations: readonly ExecutionSemanticOperation[]) { + if (this.groupedPublication || operations.length === 0 || operations.length > APPEND_GROUP_LIMITS.operations + || operations.some((operation) => !isGroupableAppend(operation)) + || new Set(operations.map((operation) => operation.execution.executionId)).size !== operations.length) { + throw new Error("Invalid semantic append group"); + } + try { + return this.db.transaction(() => { + const results: { operation: ExecutionSemanticOperation; receipt: ExecutionWriteReceipt; + publications: Parameters[0][] }[] = []; + const startedAt = performance.now(); + for (const operation of operations) { + const publications: Parameters[0][] = []; + this.groupedPublication = publications; + const receipt = this.transactAppend(operation); + results.push({ operation, receipt, publications }); + if (performance.now() - startedAt >= APPEND_GROUP_LIMITS.elapsedMs) break; + } + return results; + }).immediate(); + } finally { + this.groupedPublication = null; + } + } + + private transactAppend(operation: ExecutionSemanticOperation): ExecutionWriteReceipt { + if (!validOperation(operation)) return conflict(operation); + const hash = fingerprint(operation); + const existing = this.existingReceipt(operation, hash); + if (existing) { + this.retireReplayedText(operation, existing); + return existing; + } + try { + return this.append(operation, hash); + } catch (error) { + if (error instanceof SemanticConflict) return conflict(operation); + throw error; + } + } + + private retireReplayedText(operation: ExecutionSemanticOperation, receipt: ExecutionWriteReceipt): void { + if (receipt.kind !== "committed") return; + if (isFinishMutation(operation.mutation)) this.assistantText.retire(operation.execution.executionId); + if (parentLiveOperation(operation)?.mutation.text.kind === "reclassify") { + this.assistantText.discardRecoveryJournal(operation.execution.executionId); + } + } + private appendParentLive( operation: ExecutionSemanticOperation, head: SemanticHead, @@ -1128,7 +1175,7 @@ export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter const events = sequences.map((sequence) => typeof sequence === "number" ? this.loadPublishedEvent(executionId, sequence) : this.loadPublishedEvent(sequence.executionId, sequence.sequence)); - this.publish(events); + this.publishCommitted(events); if (acknowledge) this.acknowledgePublication.run(ACKNOWLEDGED_PUBLICATION_KIND, executionId, chunk.operation_id, PUBLICATION_KIND, hash); } @@ -1156,13 +1203,18 @@ export class CanonicalExecutionSemanticWriter implements ExecutionSemanticWriter try { const result = write(); this.bufferedPublication = null; - if (publishAfterCommit) for (const events of pending) this.publish(events); + if (publishAfterCommit) for (const events of pending) this.publishCommitted(events); return result; } finally { this.bufferedPublication = null; this.bufferedPublicationCount = 0; } } + + private publishCommitted(events: Parameters[0]): void { + if (this.groupedPublication) this.groupedPublication.push(events); + else this.publish(events); + } } function validOperation(operation: ExecutionSemanticOperation): boolean { From 8c7bf597483be1a4d5d7090e2a70aa7fc3084402 Mon Sep 17 00:00:00 2001 From: "(CJ) Chukwudi Nwobodo" <142016413+chuks-qua@users.noreply.github.com> Date: Fri, 25 Sep 2026 21:58:00 +0100 Subject: [PATCH 121/136] perf(server): reclaim released execution memory --- apps/desktop/scripts/dev-electron.mjs | 2 +- .../canonical-agent-writer-client.ts | 3 ++- .../canonical-agent-writer.worker.ts | 4 ++++ .../execution/execution-worker-handler.ts | 3 +++ .../agents/execution/execution-worker-port.ts | 3 ++- .../agents/execution/execution.worker.ts | 6 +++++- .../__tests__/memory-pressure-service.test.ts | 19 +++++++++++++++++++ .../runtime/memory/memory-pressure-service.ts | 11 ++++++----- apps/server/tsconfig.json | 2 +- scripts/build-server-dev-bundle.mjs | 2 +- 10 files changed, 44 insertions(+), 11 deletions(-) diff --git a/apps/desktop/scripts/dev-electron.mjs b/apps/desktop/scripts/dev-electron.mjs index ed807079e..834834b69 100644 --- a/apps/desktop/scripts/dev-electron.mjs +++ b/apps/desktop/scripts/dev-electron.mjs @@ -160,7 +160,7 @@ const serverEsbuildCfg = { ...shared, entryPoints: [NodePath.resolve(serverRoot, "dist-tsc/index.js")], outfile: serverOutFile, - external: ["bun:sqlite", "node-pty", "electron", "koffi"], + external: ["bun", "bun:sqlite", "node-pty", "electron", "koffi"], banner: { js: 'var __importMetaUrl = require("url").pathToFileURL(__filename).href;', }, diff --git a/apps/server/src/features/agents/canonical/canonical-agent-writer-client.ts b/apps/server/src/features/agents/canonical/canonical-agent-writer-client.ts index b90880793..e7724e0cc 100644 --- a/apps/server/src/features/agents/canonical/canonical-agent-writer-client.ts +++ b/apps/server/src/features/agents/canonical/canonical-agent-writer-client.ts @@ -358,7 +358,8 @@ function defaultCreateWorker(): Worker { const workerFile = import.meta.url.endsWith(".cjs") ? "./canonical-agent-writer.worker.cjs" : "./canonical-agent-writer.worker.ts"; - return new Worker(new URL(workerFile, import.meta.url), { type: "module" }); + const workerOptions = { type: "module", smol: true } satisfies WorkerOptions & { smol: boolean }; + return new Worker(new URL(workerFile, import.meta.url), workerOptions); } function acknowledgementKey(executionId: string, operationId: string): string { diff --git a/apps/server/src/features/agents/canonical/canonical-agent-writer.worker.ts b/apps/server/src/features/agents/canonical/canonical-agent-writer.worker.ts index 321e7b61a..d472ff0a8 100644 --- a/apps/server/src/features/agents/canonical/canonical-agent-writer.worker.ts +++ b/apps/server/src/features/agents/canonical/canonical-agent-writer.worker.ts @@ -1,4 +1,5 @@ import "reflect-metadata"; +import { gc } from "bun"; import { Database } from "bun:sqlite"; import * as NodeFS from "node:fs"; import * as NodePath from "node:path"; @@ -265,6 +266,9 @@ async function handleAppendGroup(requests: Extract { const response = await handle(request); globalThis.postMessage(response); + if (requestQueue.length === 1 && response.kind === "semantic-transacted" && response.receipt.kind === "committed" + && request.kind === "semantic-transact" && (request.operation.mutation.kind === "finish" + || request.operation.mutation.kind === "finish-live-event")) gc(true); } async function drainRequests(): Promise { diff --git a/apps/server/src/features/agents/execution/execution-worker-handler.ts b/apps/server/src/features/agents/execution/execution-worker-handler.ts index bc1c51889..0da87b271 100644 --- a/apps/server/src/features/agents/execution/execution-worker-handler.ts +++ b/apps/server/src/features/agents/execution/execution-worker-handler.ts @@ -211,6 +211,9 @@ export class ExecutionWorkerHandler { constructor(private readonly writer: ExecutionSemanticWriter, private readonly files = new ExecutionWorkerFileEvidence()) {} + /** Whether every execution assigned to this worker has released its state. */ + get isIdle(): boolean { return this.states.size === 0; } + /** Process one mailbox command and echo its exact execution and lease. */ async handle(request: ExecutionWorkerRequest): Promise> { const result = await this.apply(request); diff --git a/apps/server/src/features/agents/execution/execution-worker-port.ts b/apps/server/src/features/agents/execution/execution-worker-port.ts index e48fa1320..f37eeb8a7 100644 --- a/apps/server/src/features/agents/execution/execution-worker-port.ts +++ b/apps/server/src/features/agents/execution/execution-worker-port.ts @@ -54,7 +54,8 @@ export class ExecutionThreadWorkerPort implements ExecutionWorkerPort { this.resolveReady = resolve; }); - this.worker = new Worker(workerUrl, { type: "module" }); + const workerOptions = { type: "module", smol: true } satisfies WorkerOptions & { smol: boolean }; + this.worker = new Worker(workerUrl, workerOptions); this.worker.onmessage = (event: MessageEvent) => this.receive(event.data); this.worker.onerror = (event) => this.fail(event); this.worker.onmessageerror = () => this.fail(new ErrorEvent("error")); diff --git a/apps/server/src/features/agents/execution/execution.worker.ts b/apps/server/src/features/agents/execution/execution.worker.ts index bda8f86d8..eb69c3dcf 100644 --- a/apps/server/src/features/agents/execution/execution.worker.ts +++ b/apps/server/src/features/agents/execution/execution.worker.ts @@ -1,3 +1,4 @@ +import { gc } from "bun"; import { ExecutionWorkerHandler, type ExecutionSemanticOperation, @@ -70,7 +71,10 @@ function handleCommand(request: Extract { - if (!closed) post({ kind: "command-reply", reply }); + if (!closed) { + post({ kind: "command-reply", reply }); + if (reply.result.kind === "released" && handler.isIdle) gc(true); + } }) .catch((error: unknown) => { if (error instanceof ExecutionWriterRpcFailure) { diff --git a/apps/server/src/runtime/memory/__tests__/memory-pressure-service.test.ts b/apps/server/src/runtime/memory/__tests__/memory-pressure-service.test.ts index a566df2db..062d5f6f1 100644 --- a/apps/server/src/runtime/memory/__tests__/memory-pressure-service.test.ts +++ b/apps/server/src/runtime/memory/__tests__/memory-pressure-service.test.ts @@ -1,9 +1,15 @@ import "reflect-metadata"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { gc } from "bun"; import { getDefaultSettings } from "@mcode/contracts"; import { MemoryPressureService } from "../memory-pressure-service.js"; import type { RuntimeMemoryMeasurement } from "../runtime-memory-sampler.js"; +vi.mock("bun", async (importOriginal) => ({ + ...await importOriginal(), + gc: vi.fn(), +})); + const db = { run: vi.fn() } as unknown as import("bun:sqlite").Database; function settingsWithHeapBudget(getHeapMb: () => number) { @@ -80,6 +86,19 @@ describe("MemoryPressureService", () => { expect(levels).toEqual(["warning"]); }); + it("reclaims elevated pressure after the final turn without forcing collection during peer work", () => { + service.markActive("thread-1"); + service.markActive("thread-2"); + service.sampleActiveMemoryForTest(v8Measurement(85, 100)); + expect(gc).toHaveBeenCalledWith(false); + vi.mocked(gc).mockClear(); + + service.markIdle("thread-1"); + expect(gc).not.toHaveBeenCalled(); + service.markIdle("thread-2"); + expect(gc).toHaveBeenCalledExactlyOnceWith(true); + }); + it("uses Bun RSS for thresholds, recovery, and updated settings", async () => { let heapMb = 256; const initialBudgetBytes = heapMb * 1024 * 1024; diff --git a/apps/server/src/runtime/memory/memory-pressure-service.ts b/apps/server/src/runtime/memory/memory-pressure-service.ts index f663a178a..5f0d5cf17 100644 --- a/apps/server/src/runtime/memory/memory-pressure-service.ts +++ b/apps/server/src/runtime/memory/memory-pressure-service.ts @@ -7,6 +7,7 @@ */ import { injectable, inject } from "tsyringe"; +import { gc } from "bun"; import type { Database } from "bun:sqlite"; import { logger } from "@mcode/shared"; import { SettingsService } from "../../features/settings/settings-service.js"; @@ -141,6 +142,7 @@ export class MemoryPressureService { } if (!threadId && this.activeTurns.size > 0) return; this.stopActiveMemoryPolling(); + if (this.pressure.level !== "normal") gc(true); this.setPressure({ level: "normal", source: this.pressure.source, usedBytes: 0, budgetBytes: 0, ratio: 0 }); this.clearIdleTimers(); if (this.isWindowBackground) { @@ -249,6 +251,7 @@ export class MemoryPressureService { budgetBytes: snapshot.budgetBytes, }); this.notifyPressureListeners(snapshot); + if (snapshot.level !== "normal") gc(false); } private notifyPressureListeners(snapshot: MemoryPressureSnapshot): void { @@ -300,9 +303,7 @@ export class MemoryPressureService { error: err instanceof Error ? err.message : String(err), }); } - if (typeof global.gc === "function") { - global.gc(); - } + gc(false); // These synchronous ops block the event loop; the duration ties stalls to // the health-probe failures they can cause. logger.info("Warm idle maintenance completed", { @@ -326,9 +327,9 @@ export class MemoryPressureService { }); } const now = Date.now(); - if (typeof global.gc === "function" && now - this.lastFullGcAt > MIN_FULL_GC_INTERVAL_MS) { + if (now - this.lastFullGcAt > MIN_FULL_GC_INTERVAL_MS) { this.lastFullGcAt = now; - global.gc(true); + gc(true); } this.state = "background-idle"; logger.info("Background idle maintenance completed", { diff --git a/apps/server/tsconfig.json b/apps/server/tsconfig.json index 21045ad40..e099d1c96 100644 --- a/apps/server/tsconfig.json +++ b/apps/server/tsconfig.json @@ -13,7 +13,7 @@ "declaration": true, "declarationMap": true, "sourceMap": true, - "types": ["bun-types/sqlite"], + "types": ["bun-types/sqlite", "bun-types/bun"], "noUnusedLocals": true, "noUnusedParameters": true, "noFallthroughCasesInSwitch": true, diff --git a/scripts/build-server-dev-bundle.mjs b/scripts/build-server-dev-bundle.mjs index 5ab323205..d2e0aa6e0 100644 --- a/scripts/build-server-dev-bundle.mjs +++ b/scripts/build-server-dev-bundle.mjs @@ -590,7 +590,7 @@ export async function buildServerRuntimeBundles({ target: "node20", sourcemap: true, format: "cjs", - external: ["bun:sqlite", "node-pty", "electron", "koffi"], + external: ["bun", "bun:sqlite", "node-pty", "electron", "koffi"], banner: { js: 'var __importMetaUrl = require("url").pathToFileURL(__filename).href;', }, From 722dd18cea7c7c521ca0737995f84c0d72d71b15 Mon Sep 17 00:00:00 2001 From: "(CJ) Chukwudi Nwobodo" <142016413+chuks-qua@users.noreply.github.com> Date: Fri, 25 Sep 2026 21:58:00 +0100 Subject: [PATCH 122/136] perf(server): start legacy event workers on demand --- .../__tests__/provider-event-ingress.test.ts | 42 +++++++++++++++++-- .../composition/provider-event-ingress.ts | 1 - 2 files changed, 39 insertions(+), 4 deletions(-) diff --git a/apps/server/src/features/providers/composition/__tests__/provider-event-ingress.test.ts b/apps/server/src/features/providers/composition/__tests__/provider-event-ingress.test.ts index c27491498..fe2a49ff7 100644 --- a/apps/server/src/features/providers/composition/__tests__/provider-event-ingress.test.ts +++ b/apps/server/src/features/providers/composition/__tests__/provider-event-ingress.test.ts @@ -18,9 +18,10 @@ import { type ProviderEventIngressEvent, } from "../provider-event-ingress.js"; import type { ProviderEventAdapter } from "../provider-event-adapter.js"; -import type { - ProviderEventWorkerCallbacks, - ProviderEventWorkerPool, +import { + ThreadEventWorkerPool, + type ProviderEventWorkerCallbacks, + type ProviderEventWorkerPool, } from "../provider-event-worker-pool.js"; import { processProviderEventWorkerTask, @@ -216,6 +217,41 @@ async function drainIngress(ingress: ProviderEventIngress): Promise { } describe("ProviderEventIngress", () => { + it("starts legacy workers only when the first legacy event needs preprocessing", async () => { + const createWorker = vi.fn(() => new Worker(new URL("../provider-event.worker.ts", import.meta.url), { type: "module" })); + const workerPool = new ThreadEventWorkerPool({ workerCount: 2, createWorker }); + const { ingress, projected, received } = createIngress(undefined, undefined, workerPool); + try { + expect(createWorker).not.toHaveBeenCalled(); + ingress.acceptProjectedCommitted([{ ...projectedEvent("owned-codex", "projected"), providerId: "codex" }]); + await ingress.waitForThread("thread-1"); + expect(projected).toEqual([expect.objectContaining({ providerId: "codex" })]); + expect(createWorker).not.toHaveBeenCalled(); + + ingress.acceptProviderRuntime("claude", runtimeEvent("legacy")); + await ingress.waitForThread("thread-1"); + expect(createWorker).toHaveBeenCalledTimes(2); + expect(received).toEqual([expect.objectContaining({ + providerId: "claude", event: expect.objectContaining({ delta: "legacy" }), + })]); + expect(ingress.queueMetrics().pendingEvents).toBe(0); + } finally { + ingress.shutdown(); + } + }); + + it("shuts down before the first submission without creating workers", async () => { + const createWorker = vi.fn(() => new Worker(new URL("../provider-event.worker.ts", import.meta.url), { type: "module" })); + const workerPool = new ThreadEventWorkerPool({ workerCount: 2, createWorker }); + const { ingress, diagnostics, received } = createIngress(undefined, undefined, workerPool); + ingress.shutdown(); + ingress.acceptProviderRuntime("claude", runtimeEvent("after shutdown")); + await ingress.waitForThread("thread-1"); + expect(createWorker).not.toHaveBeenCalled(); + expect(received).toEqual([]); + expect(diagnostics).toEqual([expect.objectContaining({ reason: "worker-shutdown" })]); + }); + it("resolves the diagnostic sink through its explicit injection token", () => { const child = container.createChildContainer(); const provider = createProvider("claude"); diff --git a/apps/server/src/features/providers/composition/provider-event-ingress.ts b/apps/server/src/features/providers/composition/provider-event-ingress.ts index 82b436031..3f7c8c5e9 100644 --- a/apps/server/src/features/providers/composition/provider-event-ingress.ts +++ b/apps/server/src/features/providers/composition/provider-event-ingress.ts @@ -196,7 +196,6 @@ export class ProviderEventIngress { if (this.started) return; this.started = true; this.consumer = consumer; - this.workerPool.start(); for (const provider of providerRegistry.resolveAll()) { provider.on("file_mutation_start", (event) => consumer.handleProviderFileMutation(event)); if (isTurnDiffSource(provider)) provider.onTurnDiff((event) => consumer.handleProviderTurnDiff(event)); From da074aacc1d5e5be6cb81ae75f889f4e81696360 Mon Sep 17 00:00:00 2001 From: "(CJ) Chukwudi Nwobodo" <142016413+chuks-qua@users.noreply.github.com> Date: Fri, 25 Sep 2026 21:58:13 +0100 Subject: [PATCH 123/136] perf(server): checkpoint only the changed tool result --- .../narrative/narrative-turn-state.ts | 8 +- .../codex-live-event-effects.test.ts | 15 ++ .../execution/codex-live-event-effects.ts | 8 +- .../execution/codex-live-event-reducer.ts | 3 +- .../__tests__/narrative-tool-recovery.test.ts | 164 ++++++++++++++++++ .../agents/turns/narrative-recovery-delta.ts | 38 +++- 6 files changed, 230 insertions(+), 6 deletions(-) create mode 100644 apps/server/src/features/agents/turns/__tests__/narrative-tool-recovery.test.ts diff --git a/apps/server/src/features/agents/conversation/narrative/narrative-turn-state.ts b/apps/server/src/features/agents/conversation/narrative/narrative-turn-state.ts index 4e02e587e..236e8fbaf 100644 --- a/apps/server/src/features/agents/conversation/narrative/narrative-turn-state.ts +++ b/apps/server/src/features/agents/conversation/narrative/narrative-turn-state.ts @@ -663,6 +663,12 @@ export class NarrativeTurnState { return this.turnToolCalls; } + /** Project only the tool changed by a result; other narrative records remain unchanged. */ + toolRecoveryItem(threadId: string, toolCallId: string): Extract | null { + const toolCall = this.latestBufferedToolCall(threadId, toolCallId); + return toolCall ? this.toolCallRecoveryItem(toolCall) : null; + } + /** * Snapshot the visible structured narrative for an unfinished turn without * retaining provider protocol traffic or private raw tool input. @@ -738,7 +744,7 @@ export class NarrativeTurnState { return retainedBytes; } - private toolCallRecoveryItem(toolCall: BufferedToolCall): ParentNarrativeRecoveryItem { + private toolCallRecoveryItem(toolCall: BufferedToolCall): Extract { return { kind: "toolCall", record: { diff --git a/apps/server/src/features/agents/execution/__tests__/codex-live-event-effects.test.ts b/apps/server/src/features/agents/execution/__tests__/codex-live-event-effects.test.ts index e06db28ed..57a87b7e9 100644 --- a/apps/server/src/features/agents/execution/__tests__/codex-live-event-effects.test.ts +++ b/apps/server/src/features/agents/execution/__tests__/codex-live-event-effects.test.ts @@ -24,6 +24,21 @@ function setup(plan: CodexPlanFeature = "none") { } describe("CodexLiveEventEffects", () => { + it("checkpoints only the result tool while keeping full recovery and terminal narrative consistent", () => { + const { prepare } = setup(); + prepare("textDelta", { delta: "Thought", isFinalResponse: false }); + prepare("toolUse", { toolCallId: "read", toolName: "Read", toolInput: { file_path: "file.txt" } }); + const result = prepare("toolResult", { toolCallId: "read", output: "contents", isError: false }); + expect(result.effects.narrative?.items).toMatchObject([{ kind: "toolCall", record: { id: "read", output_summary: "contents" } }]); + expect(result.effects.narrative?.items).toHaveLength(1); + expect(prepare("assistantMessageBoundary", { isFinalResponse: false }).effects.narrative).toBeUndefined(); + const terminal = prepare("turnComplete", { tokensIn: 0, tokensOut: 0, reason: "stop", costUsd: 0 }); + expect(terminal.terminal?.narrative).toHaveLength(2); + expect(result.effects.narrative?.items).toHaveLength(1); + expect(() => prepare("toolResult", { toolCallId: "read", output: "late", isError: false })) + .toThrow("post-terminal event needs the terminal lifecycle owner"); + }); + it("moves unknown text into narrative and restarts its checkpoint sequence", () => { const { prepare } = setup(); expect(prepare("textDelta", { delta: "Looking at the files" }).effects.text).toEqual({ diff --git a/apps/server/src/features/agents/execution/codex-live-event-effects.ts b/apps/server/src/features/agents/execution/codex-live-event-effects.ts index ecc28ae40..2bbbdfe40 100644 --- a/apps/server/src/features/agents/execution/codex-live-event-effects.ts +++ b/apps/server/src/features/agents/execution/codex-live-event-effects.ts @@ -87,7 +87,8 @@ export class CodexLiveEventEffects { ): ParentLiveEffects { switch (intent.kind) { case "assistant-body": return this.messageEffects(intent, effects); - case "narrative-recovery": return this.narrativeEffects(intent, effects); + case "narrative-recovery": + case "tool-recovery": return this.narrativeEffects(intent, effects); case "feature-event": return this.featureEffects(intent, effects, runtime); case "plan-questions": return { ...effects, planQuestions: intent.questions }; case "plan-output": return { ...effects, planOutput: intent.output }; @@ -108,8 +109,9 @@ export class CodexLiveEventEffects { content: intent.content, model: intent.model, attachments: intent.attachments } }; } - private narrativeEffects(intent: Extract, effects: ParentLiveEffects): ParentLiveEffects { - const delta = this.narrative.prepare(intent.items); + private narrativeEffects(intent: Extract, effects: ParentLiveEffects): ParentLiveEffects { + const delta = intent.kind === "tool-recovery" + ? this.narrative.prepareToolUpdate(intent.item) : this.narrative.prepare(intent.items); if (!delta) return effects; delta.acknowledge(); return { ...effects, narrative: { executionId: this.execution.executionId, diff --git a/apps/server/src/features/agents/execution/codex-live-event-reducer.ts b/apps/server/src/features/agents/execution/codex-live-event-reducer.ts index b10d0bec5..e142931c0 100644 --- a/apps/server/src/features/agents/execution/codex-live-event-reducer.ts +++ b/apps/server/src/features/agents/execution/codex-live-event-reducer.ts @@ -76,6 +76,7 @@ export type CodexLiveWriterIntent = | { readonly kind: "hook-started"; readonly hookId: string; readonly late: boolean } | { readonly kind: "hook-completed"; readonly hookId: string; readonly late: boolean; readonly exitCode: number; readonly durationMs: number; readonly didBlock: boolean } | { readonly kind: "narrative-recovery"; readonly items: ParentNarrativeRecoveryItem[] } + | { readonly kind: "tool-recovery"; readonly item: Extract | null } | { readonly kind: "narrative-effect"; readonly effect: NarrativeTurnStateEffect } | { readonly kind: "feature-event"; readonly feature: "plan-text" | "assistant-message" | "task-tool" | "goal-refresh"; readonly event: AgentEvent } | { readonly kind: "plan-questions"; readonly questions: readonly PlanQuestion[] } @@ -364,7 +365,7 @@ export class CodexLiveEventReducer { return [ { kind: "tool-result", event }, { kind: "feature-event", feature: "task-tool", event }, - this.recovery(), + { kind: "tool-recovery", item: this.narrative.toolRecoveryItem(event.threadId, event.toolCallId) }, ]; } diff --git a/apps/server/src/features/agents/turns/__tests__/narrative-tool-recovery.test.ts b/apps/server/src/features/agents/turns/__tests__/narrative-tool-recovery.test.ts new file mode 100644 index 000000000..3667d21db --- /dev/null +++ b/apps/server/src/features/agents/turns/__tests__/narrative-tool-recovery.test.ts @@ -0,0 +1,164 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import type { ParentNarrativeRecoveryItem } from "@mcode/contracts"; +import { NarrativeTurnState } from "../../conversation/narrative/narrative-turn-state.js"; +import { NarrativeRecoveryDelta, type PreparedNarrativeRecoveryDelta } from "../narrative-recovery-delta.js"; +import { ACTIVE_TURN_RECOVERY_RETAINED_LIMITS } from "../active-turn-recovery-retention-policy.js"; + +const execution = { threadId: "thread", turnId: "turn", executionId: "execution" }; +const thought = { kind: "narrationSegment", record: { + id: "thought", message_id: "", text: "", started_at: "2026-09-25T12:00:00.000Z", ended_at: null, sort_order: 0, +} } satisfies ParentNarrativeRecoveryItem; + +function setup(toolName = "Read") { + const state = new NarrativeTurnState(execution); + state.beginTurn(execution.threadId); + state.openOrExtendThought(execution.threadId, "Keep this thought"); + state.closeOpenThought(execution.threadId); + state.bufferToolCall(execution.threadId, { toolCallId: "tool", toolName, + parentToolCallId: "parent", toolInput: toolName === "Agent" ? { description: "Initial agent" } : { file_path: "first.txt" } }); + const partial = new NarrativeRecoveryDelta(); + const full = new NarrativeRecoveryDelta(); + const checkpoint = () => { + const snapshot = state.recoverySnapshot(execution.threadId); + const actual = partial.prepare(snapshot); + const expected = full.prepare(snapshot); + expect(changes(actual)).toEqual(changes(expected)); + actual?.acknowledge(); + expected?.acknowledge(); + return actual; + }; + checkpoint(); + const update = (toolCallId = "tool") => { + const actual = partial.prepareToolUpdate(state.toolRecoveryItem(execution.threadId, toolCallId)); + const expected = full.prepare(state.recoverySnapshot(execution.threadId)); + expect(changes(actual)).toEqual(changes(expected)); + actual?.acknowledge(); + expected?.acknowledge(); + return actual; + }; + return { state, partial, checkpoint, update }; +} + +function changes(delta: PreparedNarrativeRecoveryDelta | null) { + return delta && { items: delta.items, discardedItemIds: delta.discardedItemIds }; +} + +function toolRecord() { + const { state } = setup(); + const item = state.toolRecoveryItem(execution.threadId, "tool"); + if (!item) throw new Error("Expected buffered tool"); + return item; +} + +beforeEach(() => { + vi.useFakeTimers(); + vi.setSystemTime(new Date("2026-09-25T12:00:00.000Z")); +}); +afterEach(() => { vi.useRealTimers(); }); + +describe("targeted tool recovery", () => { + it("matches full recovery for output, metadata removal, duplicates and later full checkpoints", () => { + const { state, update, checkpoint } = setup(); + state.updateBufferedToolCallOutput(execution.threadId, "tool", "first output", false, + { file_path: "updated.txt" }, { exitCode: 0, outputTruncated: true, outputTotalBytes: 500 }); + const first = update(); + expect(first?.items).toMatchObject([{ kind: "toolCall", record: { + id: "tool", parent_tool_call_id: "parent", input_summary: "updated.txt", output_summary: "first output", + exit_code: 0, output_truncated: 1, output_total_bytes: 500, + } }]); + state.updateBufferedToolCallOutput(execution.threadId, "tool", "second output", true); + expect(update()?.items).toMatchObject([{ record: { status: "failed", exit_code: null, output_total_bytes: null } }]); + state.updateBufferedToolCallOutput(execution.threadId, "tool", "second output", true); + expect(update()).toBeNull(); + expect(checkpoint()).toBeNull(); + state.openOrExtendThought(execution.threadId, "New thought"); + expect(checkpoint()?.items).toHaveLength(1); + state.takeOpenThought(execution.threadId); + expect(checkpoint()?.discardedItemIds).toHaveLength(1); + }); + + it("preserves parent and Agent metadata through a targeted result after a duplicate tool use", () => { + const { state, update, checkpoint } = setup("Agent"); + state.bufferToolCall(execution.threadId, { toolCallId: "tool", toolName: "Agent", + parentToolCallId: "reconciled-parent", toolInput: { description: "Reviewer", agentId: "agent-7", prompt: "Review changes" } }); + checkpoint(); + state.updateBufferedToolCallOutput(execution.threadId, "tool", "Reviewed", false, + { description: "Updated reviewer", agentId: "agent-7", prompt: "Review final changes", durationMs: 123 }); + expect(update()?.items).toMatchObject([{ record: { parent_tool_call_id: "reconciled-parent", + subagent_agent_id: "agent-7", subagent_prompt: "Review final changes", subagent_duration_ms: 123 } }]); + expect(checkpoint()).toBeNull(); + }); + + it("does nothing for an unknown tool and returns an owned projection", () => { + const { state, update, checkpoint } = setup(); + state.updateBufferedToolCallOutput(execution.threadId, "unknown", "ignored", false); + expect(update("unknown")).toBeNull(); + const projected = state.toolRecoveryItem(execution.threadId, "tool"); + if (!projected) throw new Error("Expected buffered tool"); + projected.record.output_summary = "Mutated consumer copy"; + expect(checkpoint()).toBeNull(); + }); + + it("keeps failed updates retryable and fences obsolete acknowledgements", () => { + const { state, partial } = setup(); + state.updateBufferedToolCallOutput(execution.threadId, "tool", "done", false); + const item = state.toolRecoveryItem(execution.threadId, "tool"); + const first = partial.prepareToolUpdate(item); + const retry = partial.prepareToolUpdate(item); + expect(changes(retry)).toEqual(changes(first)); + retry?.acknowledge(); + expect(partial.prepareToolUpdate(item)).toBeNull(); + expect(() => first?.acknowledge()).toThrow("already superseded"); + expect(partial.prepare(state.recoverySnapshot(execution.threadId))).toBeNull(); + expect(() => new NarrativeRecoveryDelta().prepareToolUpdate(item)).toThrow("full checkpoint"); + }); + + it("checks total retained bytes and recovers the accounting after shrinkage and a full checkpoint", () => { + const tool = toolRecord(); + const padding = ACTIVE_TURN_RECOVERY_RETAINED_LIMITS.maxBytes + - Buffer.byteLength(JSON.stringify(tool)) - Buffer.byteLength(JSON.stringify(thought)) - 32; + const retainedThought = { ...thought, record: { ...thought.record, text: "x".repeat(padding) } }; + const delta = new NarrativeRecoveryDelta(); + delta.prepare([retainedThought, tool])?.acknowledge(); + const enlarged = { ...tool, record: { ...tool.record, output_summary: "x".repeat(100) } }; + expect(() => delta.prepareToolUpdate(enlarged)).toThrow("retained byte capacity"); + delta.prepare([thought, tool])?.acknowledge(); + delta.prepareToolUpdate(enlarged)?.acknowledge(); + delta.prepareToolUpdate(tool)?.acknowledge(); + expect(delta.prepare([thought, tool])).toBeNull(); + expect(() => delta.prepareToolUpdate({ ...tool, record: { ...tool.record, + output_summary: "x".repeat(ACTIVE_TURN_RECOVERY_RETAINED_LIMITS.maxBytes) } })).toThrow("active-turn byte limit"); + }); + + it("checks the retained record count even when only one tool changes", () => { + const tool = toolRecord(); + const delta = new NarrativeRecoveryDelta(); + const thoughts = Array.from({ length: ACTIVE_TURN_RECOVERY_RETAINED_LIMITS.maxRecords }, (_, index) => ({ + ...thought, record: { ...thought.record, id: `thought-${index}` }, + })); + delta.prepare([...thoughts, tool])?.acknowledge(); + expect(() => delta.prepareToolUpdate({ ...tool, record: { ...tool.record, output_summary: "done" } })) + .toThrow("retained record capacity"); + }); + + it("counts duplicate snapshot identities even when their full checkpoint has no changes", () => { + const tool = toolRecord(); + const delta = new NarrativeRecoveryDelta(); + const state = new NarrativeTurnState(execution); + state.openHook(execution.threadId, { hookName: "check", toolName: null, phase: "pre", payload: "", sortOrder: 0 }); + const hook = state.recoverySnapshot(execution.threadId).find((item) => item.kind === "hook"); + if (!hook) throw new Error("Expected hook recovery item"); + const repeated = { ...hook, record: { ...hook.record, payload: "x".repeat(130_900) } }; + delta.prepare([repeated, tool])?.acknowledge(); + expect(delta.prepare([repeated, repeated, tool])).toBeNull(); + expect(() => delta.prepareToolUpdate({ ...tool, record: { ...tool.record, output_summary: "done" } })) + .toThrow("retained byte capacity"); + expect(delta.prepare([repeated, tool])).toBeNull(); + expect(delta.prepareToolUpdate({ ...tool, record: { ...tool.record, output_summary: "done" } })).not.toBeNull(); + + const repeatedIds = new NarrativeRecoveryDelta(); + repeatedIds.prepare([thought, tool])?.acknowledge(); + expect(repeatedIds.prepare([...Array.from({ length: ACTIVE_TURN_RECOVERY_RETAINED_LIMITS.maxRecords }, () => thought), tool])).toBeNull(); + expect(() => repeatedIds.prepareToolUpdate(tool)).toThrow("retained record capacity"); + }); +}); diff --git a/apps/server/src/features/agents/turns/narrative-recovery-delta.ts b/apps/server/src/features/agents/turns/narrative-recovery-delta.ts index 6e9e60d86..0dd12f3bf 100644 --- a/apps/server/src/features/agents/turns/narrative-recovery-delta.ts +++ b/apps/server/src/features/agents/turns/narrative-recovery-delta.ts @@ -1,6 +1,7 @@ import type { ParentNarrativeRecoveryItem } from "@mcode/contracts"; import { ACTIVE_TURN_WRITE_BATCH_LIMITS } from "../../../runtime/persistence/sqlite/bounded-write-batches.js"; import type { ParentNarrativeRecoveryCommit } from "./parent-turn-durability.js"; +import { assertActiveTurnRecoveryRetention } from "./active-turn-recovery-retention-policy.js"; /** One full-snapshot difference awaiting confirmation of its durable write. */ export interface PreparedNarrativeRecoveryDelta { @@ -12,21 +13,30 @@ export interface PreparedNarrativeRecoveryDelta { /** Tracks one execution's committed narrative snapshot without owning persistence. */ export class NarrativeRecoveryDelta { private fingerprints = new Map(); + private retainedBytes = 0; + private retainedRecords = 0; private revision = 0; /** Compare a complete snapshot to the last acknowledged one. */ prepare(snapshot: readonly ParentNarrativeRecoveryItem[]): PreparedNarrativeRecoveryDelta | null { const next = new Map(); const items: ParentNarrativeRecoveryItem[] = []; + let retainedBytes = 0; for (const item of snapshot) { const id = `${item.kind}:${item.record.id}`; const fingerprint = JSON.stringify(item); + retainedBytes += Buffer.byteLength(fingerprint, "utf8"); next.set(id, fingerprint); if (this.fingerprints.get(id) !== fingerprint) items.push(item); } const discardedItemIds = [...this.fingerprints.keys()] .filter((id) => !next.has(id)).map(canonicalItemId); - if (items.length === 0 && discardedItemIds.length === 0) return null; + if (items.length === 0 && discardedItemIds.length === 0) { + // Recovery limits count snapshot entries, including repeated hook/thought identities. + this.retainedBytes = retainedBytes; + this.retainedRecords = snapshot.length; + return null; + } const preparedAt = this.revision; return { items, @@ -34,6 +44,32 @@ export class NarrativeRecoveryDelta { acknowledge: () => { if (this.revision !== preparedAt) throw new Error("Narrative recovery delta was already superseded"); this.fingerprints = next; + this.retainedBytes = retainedBytes; + this.retainedRecords = snapshot.length; + this.revision += 1; + }, + }; + } + + /** Update one already checkpointed tool without treating unrelated records as deleted. */ + prepareToolUpdate(item: Extract | null): PreparedNarrativeRecoveryDelta | null { + if (!item) return null; + const id = `${item.kind}:${item.record.id}`; + const previous = this.fingerprints.get(id); + if (previous === undefined) throw new Error("Tool recovery update requires an acknowledged full checkpoint"); + const fingerprint = JSON.stringify(item); + const bytes = Buffer.byteLength(fingerprint, "utf8"); + if (bytes > ACTIVE_TURN_WRITE_BATCH_LIMITS.maxBytes) throw new Error("Parent narrative recovery item exceeds the active-turn byte limit"); + const retainedBytes = this.retainedBytes + bytes - Buffer.byteLength(previous, "utf8"); + assertActiveTurnRecoveryRetention(this.retainedRecords, retainedBytes); + if (previous === fingerprint) return null; + const preparedAt = this.revision; + return { + items: [item], discardedItemIds: [], + acknowledge: () => { + if (this.revision !== preparedAt) throw new Error("Narrative recovery delta was already superseded"); + this.fingerprints.set(id, fingerprint); + this.retainedBytes = retainedBytes; this.revision += 1; }, }; From 57e4992b3459c98e39714ba1d996ce8cfb88ed92 Mon Sep 17 00:00:00 2001 From: "(CJ) Chukwudi Nwobodo" <142016413+chuks-qua@users.noreply.github.com> Date: Fri, 25 Sep 2026 21:58:13 +0100 Subject: [PATCH 124/136] perf(server): persist turn settings in one atomic write --- .../__tests__/agent-service-container.test.ts | 16 ++++++- .../turn-admission-dispatch-coordinator.ts | 8 ++-- .../__tests__/thread-repo-settings.test.ts | 47 +++++++++++++++++++ .../thread-control/persistence/thread-repo.ts | 6 ++- 4 files changed, 71 insertions(+), 6 deletions(-) diff --git a/apps/server/src/features/agents/composition/__tests__/agent-service-container.test.ts b/apps/server/src/features/agents/composition/__tests__/agent-service-container.test.ts index ed357640a..206c0e454 100644 --- a/apps/server/src/features/agents/composition/__tests__/agent-service-container.test.ts +++ b/apps/server/src/features/agents/composition/__tests__/agent-service-container.test.ts @@ -123,12 +123,24 @@ describe("AgentService container composition", () => { registry.bind((event) => published.push(event)); registry.start(); const workspace = container.resolve(WorkspaceRepo).create("worker-test", temporaryDirectory!); - const thread = container.resolve(ThreadRepo).create(workspace.id, "Worker turn", "direct", "main", true, "codex"); + const threads = container.resolve(ThreadRepo); + const thread = threads.create(workspace.id, "Worker turn", "direct", "main", true, "claude"); + threads.updateSettings(thread.id, { thinking: true }); - await container.resolve(AgentService).sendMessage({ threadId: thread.id, content: "hello", permissionMode: "default" }); + await container.resolve(AgentService).sendMessage({ + threadId: thread.id, content: "hello", provider: "codex", model: "gpt-5.6-luna", + permissionMode: "full", reasoningLevel: "high", codexFastMode: false, + }); await waitFor(() => workerRuntime?.owner.current(thread.id) === undefined); expect(providerError).toBeUndefined(); + expect(threads.findById(thread.id)).toMatchObject({ + provider: "codex", model: "gpt-5.6-luna", permission_mode: "full", + reasoning_level: "high", codex_fast_mode: false, thinking: true, + }); + expect(pushes.filter((push) => push.channel === "thread.modelUpdated")).toEqual([ + { type: "push", channel: "thread.modelUpdated", data: { threadId: thread.id, model: "gpt-5.6-luna", provider: "codex" } }, + ]); expect(published.map((event) => event.type)).toEqual(expect.arrayContaining([ "turnStarted", "textDelta", "turnComplete", "ended", ])); diff --git a/apps/server/src/features/agents/turns/turn-admission-dispatch-coordinator.ts b/apps/server/src/features/agents/turns/turn-admission-dispatch-coordinator.ts index 3f5ea663c..d695c8aec 100644 --- a/apps/server/src/features/agents/turns/turn-admission-dispatch-coordinator.ts +++ b/apps/server/src/features/agents/turns/turn-admission-dispatch-coordinator.ts @@ -837,9 +837,11 @@ export class TurnAdmissionDispatchCoordinator { private persistThreadSettings(prepared: PreparedCommand): void { const command = prepared.command; const model = command.model ?? "claude-sonnet-4-6"; - this.threads.updateModel(command.threadId, model); - if (command.provider !== undefined) this.threads.updateProvider(command.threadId, prepared.providerId); - this.threads.updateSettings(command.threadId, this.threadSettings(command, prepared.providerId)); + this.threads.updateSettings(command.threadId, { + ...this.threadSettings(command, prepared.providerId), + model, + ...(command.provider === undefined ? {} : { provider: prepared.providerId }), + }); broadcast("thread.modelUpdated", { threadId: command.threadId, model, diff --git a/apps/server/src/features/thread-control/persistence/__tests__/thread-repo-settings.test.ts b/apps/server/src/features/thread-control/persistence/__tests__/thread-repo-settings.test.ts index 785ce0751..ebb3b3ea5 100644 --- a/apps/server/src/features/thread-control/persistence/__tests__/thread-repo-settings.test.ts +++ b/apps/server/src/features/thread-control/persistence/__tests__/thread-repo-settings.test.ts @@ -42,6 +42,53 @@ describe("ThreadRepo.updateSettings", () => { expect(thread?.permission_mode).toBeNull(); }); + it("persists model, provider, and supplied settings together while retaining omitted values", () => { + repo.updateSettings(threadId, { reasoning_level: "high", thinking: true, codex_fast_mode: true }); + + expect(repo.updateSettings(threadId, { + model: "gpt-5.6-luna", + provider: "codex", + permission_mode: "full", + thinking: false, + codex_fast_mode: null, + })).toBe(true); + + expect(repo.findById(threadId)).toMatchObject({ + model: "gpt-5.6-luna", + provider: "codex", + permission_mode: "full", + reasoning_level: "high", + thinking: false, + codex_fast_mode: null, + }); + repo.updateSettings(threadId, { model: "gpt-5.6-sol" }); + expect(repo.findById(threadId)).toMatchObject({ model: "gpt-5.6-sol", provider: "codex", permission_mode: "full" }); + }); + + it("leaves all thread settings unchanged when the combined update fails", () => { + repo.updateSettings(threadId, { model: "original-model", provider: "claude", permission_mode: "supervised" }); + const before = repo.findById(threadId); + db.exec(` + CREATE TRIGGER reject_full_permission BEFORE UPDATE OF permission_mode ON threads + WHEN NEW.permission_mode = 'full' + BEGIN + SELECT RAISE(ABORT, 'permission update rejected'); + END; + `); + + expect(() => repo.updateSettings(threadId, { + model: "gpt-5.6-luna", provider: "codex", permission_mode: "full", + })).toThrow("permission update rejected"); + + expect(repo.findById(threadId)).toEqual(before); + }); + + it("retains the timestamp when no settings are supplied", () => { + const before = repo.findById(threadId); + expect(repo.updateSettings(threadId, {})).toBe(false); + expect(repo.findById(threadId)).toEqual(before); + }); + it("returns false for nonexistent thread", () => { const ok = repo.updateSettings("nonexistent", { reasoning_level: "low" }); expect(ok).toBe(false); diff --git a/apps/server/src/features/thread-control/persistence/thread-repo.ts b/apps/server/src/features/thread-control/persistence/thread-repo.ts index 5ecf53657..ec760ce3e 100644 --- a/apps/server/src/features/thread-control/persistence/thread-repo.ts +++ b/apps/server/src/features/thread-control/persistence/thread-repo.ts @@ -307,6 +307,8 @@ function serializeBooleanOverride(value: boolean | null): number | null { } type ThreadSettings = { + model?: string; + provider?: string; reasoning_level?: string; interaction_mode?: string; orchestration_mode?: string; @@ -320,6 +322,8 @@ type ThreadSettings = { }; const TEXT_SETTING_COLUMNS = [ + ["model", "model"], + ["provider", "provider"], ["reasoning_level", "reasoningLevel"], ["interaction_mode", "interactionMode"], ["orchestration_mode", "orchestrationMode"], @@ -1007,7 +1011,7 @@ export class ThreadRepo { return result.changes > 0; } - /** Persist per-thread composer settings (reasoning, mode, permission, copilot agent). */ + /** Persist model, provider, and supplied composer settings in one atomic update. */ updateSettings(id: string, settings: ThreadSettings): boolean { const set = buildSettingsSet(settings); if (Object.keys(set).length === 0) return false; From aea783f6c8b56b2cd318a0b410208d5740ad3daf Mon Sep 17 00:00:00 2001 From: "(CJ) Chukwudi Nwobodo" <142016413+chuks-qua@users.noreply.github.com> Date: Fri, 25 Sep 2026 22:10:04 +0100 Subject: [PATCH 125/136] perf(server): clone only changed recovery records --- .../codex-live-event-effects.test.ts | 2 + .../codex-live-event-reducer.test.ts | 45 +++++++++++++++++++ .../execution/codex-live-event-effects.ts | 14 ++---- .../execution/codex-live-event-reducer.ts | 17 ++++--- 4 files changed, 63 insertions(+), 15 deletions(-) diff --git a/apps/server/src/features/agents/execution/__tests__/codex-live-event-effects.test.ts b/apps/server/src/features/agents/execution/__tests__/codex-live-event-effects.test.ts index 57a87b7e9..1f7aeab0e 100644 --- a/apps/server/src/features/agents/execution/__tests__/codex-live-event-effects.test.ts +++ b/apps/server/src/features/agents/execution/__tests__/codex-live-event-effects.test.ts @@ -61,7 +61,9 @@ describe("CodexLiveEventEffects", () => { expect(id).toBeDefined(); const promoted = prepare("assistantMessageBoundary", { isFinalResponse: true }); expect(promoted.effects.text).toEqual({ kind: "promote", input: { ...execution, sequence: 1, text: "Answer" } }); + expect(promoted.effects.narrative?.items).toEqual([]); expect(promoted.effects.narrative?.discardedItemIds).toEqual([`narrationSegment:${id}`]); + expect(prepare("assistantMessageBoundary", { isFinalResponse: true }).effects.narrative).toBeUndefined(); }); it("retains TaskCreate input until its result and keeps file intents separate", () => { diff --git a/apps/server/src/features/agents/execution/__tests__/codex-live-event-reducer.test.ts b/apps/server/src/features/agents/execution/__tests__/codex-live-event-reducer.test.ts index d73f65b1f..e6811e34d 100644 --- a/apps/server/src/features/agents/execution/__tests__/codex-live-event-reducer.test.ts +++ b/apps/server/src/features/agents/execution/__tests__/codex-live-event-reducer.test.ts @@ -17,7 +17,52 @@ function boundByProvider(mapped: AgentEvent): AgentEvent { return { ...mapped, turnExecutionId: execution.executionId }; } +function reduceEvent(reducer: CodexLiveEventReducer, type: AgentEvent["type"], fields: Record = {}) { + const reduction = reducer.reduce(event(type, fields)); + if (reduction.kind !== "reduced") throw new Error(reduction.reason); + return reduction; +} + describe("CodexLiveEventReducer", () => { + it("returns only changed recovery items and retains the complete terminal snapshot", () => { + const reducer = new CodexLiveEventReducer(execution); + reduceEvent(reducer, "turnStarted"); + reduceEvent(reducer, "toolUse", { toolCallId: "first", toolName: "Read", toolInput: { file_path: "one" } }); + const second = reduceEvent(reducer, "toolUse", { toolCallId: "second", toolName: "Read", toolInput: { file_path: "two" } }); + expect(second.writer).toContainEqual({ kind: "narrative-recovery", discardedItemIds: [], + items: [expect.objectContaining({ kind: "toolCall", record: expect.objectContaining({ id: "second" }) })] }); + const result = reduceEvent(reducer, "toolResult", { toolCallId: "first", output: "contents", isError: false }); + expect(result.writer).toContainEqual({ kind: "narrative-recovery", discardedItemIds: [], + items: [expect.objectContaining({ kind: "toolCall", record: expect.objectContaining({ id: "first", output_summary: "contents" }) })] }); + const boundary = reduceEvent(reducer, "assistantMessageBoundary", { isFinalResponse: false }); + expect(boundary.writer).toContainEqual({ kind: "narrative-recovery", items: [], discardedItemIds: [] }); + const terminal = reduceEvent(reducer, "turnComplete", { reason: "stop", costUsd: 0, tokensIn: 0, tokensOut: 0 }); + const projection = terminal.writer.find((intent) => intent.kind === "terminal-projection"); + expect(projection?.narrative.map((item) => item.record.id)).toEqual(["first", "second"]); + const lateHook = reduceEvent(reducer, "hookStarted", { hookName: "stop", hookType: "stop" }); + expect(lateHook.writer).toContainEqual({ kind: "narrative-recovery", discardedItemIds: [], + items: [expect.objectContaining({ kind: "hook" })] }); + }); + + it("keeps recovery output independent of buffered state and emits discard-only deltas", () => { + const reducer = new CodexLiveEventReducer(execution); + reduceEvent(reducer, "turnStarted"); + const thought = reduceEvent(reducer, "textDelta", { delta: "Answer", isFinalResponse: false }); + const item = thought.writer.find((intent) => intent.kind === "narrative-recovery")?.items[0]; + if (item?.kind !== "narrationSegment") throw new Error("Expected narration recovery"); + const id = item.record.id; + item.record.text = "Mutated caller copy"; + const extended = reduceEvent(reducer, "textDelta", { delta: " continues", isFinalResponse: false }); + expect(extended.writer).toContainEqual({ kind: "narrative-recovery", discardedItemIds: [], + items: [expect.objectContaining({ record: expect.objectContaining({ id, text: "Answer continues" }) })] }); + expect(item.record.text).toBe("Mutated caller copy"); + const promoted = reduceEvent(reducer, "assistantMessageBoundary", { isFinalResponse: true }); + expect(promoted.writer).toContainEqual({ kind: "narrative-recovery", items: [], discardedItemIds: [`narrationSegment:${id}`] }); + expect(promoted.writer).toContainEqual({ kind: "assistant-text-promote", text: "Answer continues" }); + const repeated = reduceEvent(reducer, "assistantMessageBoundary", { isFinalResponse: true }); + expect(repeated.writer).toContainEqual({ kind: "narrative-recovery", items: [], discardedItemIds: [] }); + }); + it("carries parsed plan questions as data on the completing text event", () => { const reducer = new CodexLiveEventReducer(execution, "questions"); const question = { id: "q1", category: "AUTH", question: "Which login?", options: [ diff --git a/apps/server/src/features/agents/execution/codex-live-event-effects.ts b/apps/server/src/features/agents/execution/codex-live-event-effects.ts index 2bbbdfe40..472661f62 100644 --- a/apps/server/src/features/agents/execution/codex-live-event-effects.ts +++ b/apps/server/src/features/agents/execution/codex-live-event-effects.ts @@ -1,7 +1,6 @@ import type { DataOnlyParentTerminalProjectionInput } from "../canonical/canonical-parent-turn-write.js"; import type { CodexSystemWriterIntent } from "../canonical/canonical-codex-system-error-projection.js"; import { taskToolWriteIntents, type TaskToolCall } from "../tasks/task-tool-intent-reducer.js"; -import { NarrativeRecoveryDelta } from "../turns/narrative-recovery-delta.js"; import { deriveTurnAssistantMessageId } from "../turns/turn-assistant-message-id.js"; import type { ParentAssistantTextCheckpointInput } from "../turns/parent-assistant-text-checkpoint-service.js"; import type { CodexLiveReduction, CodexLiveWriterIntent } from "./codex-live-event-reducer.js"; @@ -32,7 +31,6 @@ export interface PreparedCodexLiveEvent { */ export class CodexLiveEventEffects { private sequence = 0; - private readonly narrative = new NarrativeRecoveryDelta(); private readonly calls = new Map(); private assignedMessageId: string | undefined; @@ -87,8 +85,7 @@ export class CodexLiveEventEffects { ): ParentLiveEffects { switch (intent.kind) { case "assistant-body": return this.messageEffects(intent, effects); - case "narrative-recovery": - case "tool-recovery": return this.narrativeEffects(intent, effects); + case "narrative-recovery": return this.narrativeEffects(intent, effects); case "feature-event": return this.featureEffects(intent, effects, runtime); case "plan-questions": return { ...effects, planQuestions: intent.questions }; case "plan-output": return { ...effects, planOutput: intent.output }; @@ -109,13 +106,10 @@ export class CodexLiveEventEffects { content: intent.content, model: intent.model, attachments: intent.attachments } }; } - private narrativeEffects(intent: Extract, effects: ParentLiveEffects): ParentLiveEffects { - const delta = intent.kind === "tool-recovery" - ? this.narrative.prepareToolUpdate(intent.item) : this.narrative.prepare(intent.items); - if (!delta) return effects; - delta.acknowledge(); + private narrativeEffects(intent: Extract, effects: ParentLiveEffects): ParentLiveEffects { + if (intent.items.length === 0 && intent.discardedItemIds.length === 0) return effects; return { ...effects, narrative: { executionId: this.execution.executionId, - items: delta.items, discardedItemIds: delta.discardedItemIds } }; + items: intent.items, discardedItemIds: intent.discardedItemIds } }; } private featureEffects( diff --git a/apps/server/src/features/agents/execution/codex-live-event-reducer.ts b/apps/server/src/features/agents/execution/codex-live-event-reducer.ts index e142931c0..d58e12b0c 100644 --- a/apps/server/src/features/agents/execution/codex-live-event-reducer.ts +++ b/apps/server/src/features/agents/execution/codex-live-event-reducer.ts @@ -1,6 +1,7 @@ import type { AgentEvent, ParentNarrativeRecoveryItem, PlanQuestion, StoredAttachment, TurnOutcome } from "@mcode/contracts"; import { NarrativeTurnState, type NarrativeTurnStateEffect } from "../conversation/narrative/narrative-turn-state.js"; import { AssistantExecutionState, type AssistantMaterializationInput } from "../turns/assistant-execution-state.js"; +import { NarrativeRecoveryDelta, type PreparedNarrativeRecoveryDelta } from "../turns/narrative-recovery-delta.js"; import { PlanExecutionState, type PlanPersistenceReady } from "../planning/plan-execution-state.js"; import type { ExecutionIdentity } from "./execution-mailbox-protocol.js"; @@ -75,8 +76,7 @@ export type CodexLiveWriterIntent = | { readonly kind: "tool-result"; readonly event: ToolResultEvent } | { readonly kind: "hook-started"; readonly hookId: string; readonly late: boolean } | { readonly kind: "hook-completed"; readonly hookId: string; readonly late: boolean; readonly exitCode: number; readonly durationMs: number; readonly didBlock: boolean } - | { readonly kind: "narrative-recovery"; readonly items: ParentNarrativeRecoveryItem[] } - | { readonly kind: "tool-recovery"; readonly item: Extract | null } + | { readonly kind: "narrative-recovery"; readonly items: readonly ParentNarrativeRecoveryItem[]; readonly discardedItemIds: readonly string[] } | { readonly kind: "narrative-effect"; readonly effect: NarrativeTurnStateEffect } | { readonly kind: "feature-event"; readonly feature: "plan-text" | "assistant-message" | "task-tool" | "goal-refresh"; readonly event: AgentEvent } | { readonly kind: "plan-questions"; readonly questions: readonly PlanQuestion[] } @@ -107,11 +107,13 @@ export type CodexLiveReduction = * Reduces one Codex execution's live AgentEvents without database or transport calls. * The caller must supply an exact turnExecutionId from provider turn binding; * session-level events without that proof belong to a separate owner. - * The caller must discard this instance if its writer operation fails, because + * Preparation advances narrative checkpoints before the writer commits. + * The caller must discard this instance if preparation or its writer operation fails, because * the next event may only observe state whose preceding intents were committed. */ export class CodexLiveEventReducer { private readonly narrative: NarrativeTurnState; + private readonly narrativeDelta = new NarrativeRecoveryDelta(); private readonly assistant = new AssistantExecutionState(); private phase: "awaiting-start" | "active" | "completed" | "ended" = "awaiting-start"; private unknownText = ""; @@ -365,7 +367,7 @@ export class CodexLiveEventReducer { return [ { kind: "tool-result", event }, { kind: "feature-event", feature: "task-tool", event }, - { kind: "tool-recovery", item: this.narrative.toolRecoveryItem(event.threadId, event.toolCallId) }, + this.recoveryIntent(this.narrativeDelta.prepareToolUpdate(this.narrative.toolRecoveryItem(event.threadId, event.toolCallId))), ]; } @@ -460,7 +462,12 @@ export class CodexLiveEventReducer { } private recovery(): CodexLiveWriterIntent { - return { kind: "narrative-recovery", items: this.narrative.recoverySnapshot(this.execution.threadId) }; + return this.recoveryIntent(this.narrativeDelta.prepare(this.narrative.recoverySnapshot(this.execution.threadId))); + } + + private recoveryIntent(delta: PreparedNarrativeRecoveryDelta | null): CodexLiveWriterIntent { + delta?.acknowledge(); + return { kind: "narrative-recovery", items: delta?.items ?? [], discardedItemIds: delta?.discardedItemIds ?? [] }; } private unsupported(event: AgentEvent, reason: string): CodexLiveReduction { From 8a498f7aa6ecbd566d8670ff0b4ba7b3569239dc Mon Sep 17 00:00:00 2001 From: "(CJ) Chukwudi Nwobodo" <142016413+chuks-qua@users.noreply.github.com> Date: Fri, 25 Sep 2026 22:10:04 +0100 Subject: [PATCH 126/136] perf(server): collect idle memory after completion callbacks --- .../__tests__/memory-pressure-service.test.ts | 26 ++++++++++++++++++- .../runtime/memory/memory-pressure-service.ts | 5 +++- 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/apps/server/src/runtime/memory/__tests__/memory-pressure-service.test.ts b/apps/server/src/runtime/memory/__tests__/memory-pressure-service.test.ts index 062d5f6f1..c047fc4d2 100644 --- a/apps/server/src/runtime/memory/__tests__/memory-pressure-service.test.ts +++ b/apps/server/src/runtime/memory/__tests__/memory-pressure-service.test.ts @@ -70,6 +70,18 @@ describe("MemoryPressureService", () => { expect(service.currentPressure.level).toBe("normal"); }); + it("requests collection once per pressure transition after notifying memory owners", () => { + service.markActive("thread-1"); + service.sampleActiveMemoryForTest(v8Measurement(85, 100)); + vi.mocked(gc).mockClear(); + const collectionsAtNotification: number[] = []; + service.onPressureChange(() => collectionsAtNotification.push(vi.mocked(gc).mock.calls.length)); + service.sampleActiveMemoryForTest(v8Measurement(95, 100)); + service.sampleActiveMemoryForTest(v8Measurement(96, 100)); + expect(collectionsAtNotification).toEqual([0]); + expect(gc).toHaveBeenCalledExactlyOnceWith(false); + }); + it("notifies listeners when a turn becomes idle under sustained pressure", () => { const levels: string[] = []; service.onPressureChange((snapshot) => { @@ -86,7 +98,7 @@ describe("MemoryPressureService", () => { expect(levels).toEqual(["warning"]); }); - it("reclaims elevated pressure after the final turn without forcing collection during peer work", () => { + it("reclaims elevated pressure after completion callbacks without forcing collection during peer work", async () => { service.markActive("thread-1"); service.markActive("thread-2"); service.sampleActiveMemoryForTest(v8Measurement(85, 100)); @@ -96,9 +108,21 @@ describe("MemoryPressureService", () => { service.markIdle("thread-1"); expect(gc).not.toHaveBeenCalled(); service.markIdle("thread-2"); + expect(gc).not.toHaveBeenCalled(); + await vi.advanceTimersByTimeAsync(0); expect(gc).toHaveBeenCalledExactlyOnceWith(true); }); + it("does not force an idle collection when another turn starts before callbacks finish", async () => { + service.markActive("thread-1"); + service.sampleActiveMemoryForTest(v8Measurement(85, 100)); + service.markIdle("thread-1"); + service.markActive("thread-2"); + vi.mocked(gc).mockClear(); + await vi.advanceTimersByTimeAsync(0); + expect(gc).not.toHaveBeenCalled(); + }); + it("uses Bun RSS for thresholds, recovery, and updated settings", async () => { let heapMb = 256; const initialBudgetBytes = heapMb * 1024 * 1024; diff --git a/apps/server/src/runtime/memory/memory-pressure-service.ts b/apps/server/src/runtime/memory/memory-pressure-service.ts index 5f0d5cf17..408248ae5 100644 --- a/apps/server/src/runtime/memory/memory-pressure-service.ts +++ b/apps/server/src/runtime/memory/memory-pressure-service.ts @@ -142,7 +142,10 @@ export class MemoryPressureService { } if (!threadId && this.activeTurns.size > 0) return; this.stopActiveMemoryPolling(); - if (this.pressure.level !== "normal") gc(true); + if (this.pressure.level !== "normal") { + // Completion callbacks still hold turn data until their stack unwinds. + setImmediate(() => { if (this.activeTurns.size === 0) gc(true); }); + } this.setPressure({ level: "normal", source: this.pressure.source, usedBytes: 0, budgetBytes: 0, ratio: 0 }); this.clearIdleTimers(); if (this.isWindowBackground) { From cc6f0ec7e4e99bd587574582aa4b40d753a99585 Mon Sep 17 00:00:00 2001 From: "(CJ) Chukwudi Nwobodo" <142016413+chuks-qua@users.noreply.github.com> Date: Fri, 25 Sep 2026 22:16:36 +0100 Subject: [PATCH 127/136] test(server): tolerate fractional trace clock arithmetic --- .../agents/diagnostics/__tests__/server-work-trace.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/server/src/features/agents/diagnostics/__tests__/server-work-trace.test.ts b/apps/server/src/features/agents/diagnostics/__tests__/server-work-trace.test.ts index 0c9fbfeaf..5c973002d 100644 --- a/apps/server/src/features/agents/diagnostics/__tests__/server-work-trace.test.ts +++ b/apps/server/src/features/agents/diagnostics/__tests__/server-work-trace.test.ts @@ -27,7 +27,7 @@ describe("ServerWorkTrace", () => { const report = reports[0]; expect(report?.kind).toBe("server-work-stall"); if (report?.kind !== "server-work-stall") return; - expect(report.delayMs).toBe(220); + expect(report.delayMs).toBeCloseTo(220, 8); expect(report.samples).toHaveLength(64); expect(report.overflowCount).toBe(24); expect(report.eventApplyByType.textDelta).toEqual({ count: 40, totalMs: 480, maxMs: 12 }); From 6841e8f55fc383914654c355ee1309e1cac374df Mon Sep 17 00:00:00 2001 From: "(CJ) Chukwudi Nwobodo" <142016413+chuks-qua@users.noreply.github.com> Date: Fri, 25 Sep 2026 22:35:01 +0100 Subject: [PATCH 128/136] docs(perf): record worker runtime performance recovery --- .../issue-1760-performance-recovery.md | 199 ++++++++++++++++++ 1 file changed, 199 insertions(+) create mode 100644 docs/performance/issue-1760-performance-recovery.md diff --git a/docs/performance/issue-1760-performance-recovery.md b/docs/performance/issue-1760-performance-recovery.md new file mode 100644 index 000000000..f9147808c --- /dev/null +++ b/docs/performance/issue-1760-performance-recovery.md @@ -0,0 +1,199 @@ +# Issue 1760: performance recovery + +Measured on September 25, 2026, on Windows with Bun 1.4.1. This record follows +the [original comparison](issue-1760-seven-task-evidence.md). It preserves that +earlier result and reports the optimized runtime at `8a498f7a` separately. +The later `cc6f0ec7` changes a test's floating-point assertion only. + +The optimized version improves all five nonzero metric medians from the +initial PR. Its median largest logged pause remains zero. Four of six medians +also improve on the original implementation. Peak public event rate and sampled +memory still miss the original targets. The request to improve every original +metric is therefore not fully met. + +## Ten-run comparison + +Each cell contains the median, followed by the minimum and maximum in brackets. +Times are milliseconds. Lower is better except for public event rate. + +| Metric | Original implementation | Initial PR | Optimized PR | +| --- | ---: | ---: | ---: | +| Largest logged server pause | 6491 [4893, 8079] | 0 [0, 1096] | 0 [0, 0] | +| Per-run task completion p95 | 32931 [22705, 47756] | 72911 [70481, 75846] | 26053 [23928, 30139] | +| Peak public events per second | 231 [177, 270] | 147.5 [138, 156] | 205.5 [170, 219] | +| Maximum sampled server working set, MiB | 446.4 [367.5, 623.9] | 588.7 [538.2, 627.6] | 458.3 [430.9, 493.1] | +| Terminal creation | 746 [103, 25863] | 196 [166, 1008] | 175.6 [126.9, 608.9] | +| Model list | 29 [11, 133] | 35 [7, 476] | 19.6 [5.0, 117.9] | + +Against the initial PR, completion takes 64.3% less time, peak event rate is +39.3% higher, and sampled memory is 22.2% lower. Terminal creation takes 10.4% +less time and model listing takes 44.0% less time. + +Against the original implementation, completion takes 20.9% less time. +Peak event rate remains 11.0% lower and sampled memory remains 2.7% higher. +Those two remaining gaps are not evidence of a proven performance ceiling. + +## Workload and measurement limits + +The unchanged public harness starts seven direct Codex tasks and one terminal +in this worktree's fixture repository. Each provider fixture emits 120 tool +pairs. The benchmark uses the checked-in provider fixture, without upstream +model calls. The separate desktop checks below use the real Codex provider. + +- Completion is the p95 completion time among each run's seven tasks, then the + median of those ten values. It is not a pooled p95 across all 70 tasks. +- Event rate is the largest one-second bucket of public event arrivals. It is + not average processing throughput. +- Memory is the maximum of three server working-set samples, taken before + dispatch, after the turns, and after cleanup. It is not a continuous peak. + This clarifies the earlier document's shorter label, "peak server memory." +- A zero pause means no delay was logged by the default diagnostic. It does not + mean that the event loop had zero latency. +- Terminal and model timings are one request per run while tasks are active. +- The original baseline is `15fa7030ad09b5c2d083efeb2e0670b68277220c`. The initial + PR runtime is `0a73175d`. These historical cohorts used the same host and + workload at different times. Host load was not held constant. +- No tests or builds ran concurrently with this final cohort. An unrelated + server process consumed roughly one CPU core during it. Its contribution to + the remaining differences was not established. + +All ten optimized runs passed event ordering, duplicate and missing-event +checks, persisted conversation audits, and fixture-only cleanup. Each delivered +3466 public events. The maximum queue observations were seven pending commands, +one queued command per worker, and four in flight. The original baseline did +not record a comparable queue depth. + +## Every optimized run + +All rows had zero logged main-server pauses. Memory is MiB and times are ms. + +| Run | Completion p95 | Peak events/s | Sampled memory | Terminal | Models | +| --- | ---: | ---: | ---: | ---: | ---: | +| 1 | 29904.329 | 179 | 430.895 | 608.941 | 117.923 | +| 2 | 30139.381 | 187 | 453.777 | 325.490 | 16.305 | +| 3 | 29335.843 | 170 | 444.551 | 230.683 | 25.523 | +| 4 | 25944.837 | 209 | 454.813 | 139.852 | 27.889 | +| 5 | 24035.207 | 208 | 437.164 | 196.337 | 55.097 | +| 6 | 26161.430 | 216 | 472.516 | 171.915 | 5.037 | +| 7 | 29806.950 | 180 | 471.328 | 178.944 | 10.793 | +| 8 | 25148.070 | 203 | 493.098 | 126.890 | 18.474 | +| 9 | 24624.156 | 213 | 474.363 | 172.181 | 18.552 | +| 10 | 23927.607 | 219 | 461.770 | 145.625 | 20.682 | + +The local cohort manifest is `.dev/verification/issue-1760/final-cohort.json`. +Each receipt is under `.dev/verification/performance/seven-thread-live/`, in +the following directory, with filename `receipt.json`. These names identify +local artifacts; the table above is the committed reviewable result. + +```text +after-2026-09-25T21-10-23-813Z-4bf144ab-f4b5-49df-88d4-8c80a0d0b9d8 +after-2026-09-25T21-10-56-894Z-a115cc20-328a-4dea-a2b3-d31faffde650 +after-2026-09-25T21-11-30-031Z-c95b34d0-9f6d-44ed-b57a-804f8ff6d0db +after-2026-09-25T21-12-03-033Z-e8d1e249-9f15-453c-9d18-afd9581fac85 +after-2026-09-25T21-12-32-933Z-9ad255f1-fd1f-4b16-8291-56ef49b7f660 +after-2026-09-25T21-12-59-911Z-4e4ab32b-7281-4bdf-9219-125ae026753b +after-2026-09-25T21-13-28-694Z-5b62691c-9ce3-4d20-b2d9-2db090e395bd +after-2026-09-25T21-14-01-558Z-d7ca1563-f27f-4bf2-9884-f9d2559a47ed +after-2026-09-25T21-14-29-420Z-c22652c7-ec2a-4424-866d-682b5f8ddd60 +after-2026-09-25T21-14-56-624Z-c24b17b0-d971-494d-b3db-36f4a028f7cf +``` + +The public workload is reproducible with: + +```powershell +bun scripts/perf/seven-thread-live-harness.mjs --run --confirm-run --label after +``` + +## Changes supported by the measurements + +The first trace found repeated whole-thread loads during terminal writes. +Loading transaction-local terminal state reduced 1701 loads taking 18.312 s +to 66 loads taking 26 ms in the scoped comparison. Reusing consecutive +narrative message resolutions and prepared queries removed more repeated work. +The resolver preserves the original lookup order when multiple canonical +sources share a display message. + +The writer now groups already-ready independent appends within bounds of eight +operations, 512 KiB, and 16 ms. It adds no fill delay. Repeated executions and +control operations remain ordering barriers. Acknowledgments and publication +wait for the physical commit. A failed group rolls back and retries semantic +writes individually, without replaying provider actions. One operation can +exceed the time bound before the next boundary check. + +The remaining changes reduce allocation and startup work. Worker bundles omit +unused schemas. Legacy event workers start on their first event. Recovery +checkpoints copy changed tool records and calculate deltas before cloning. +Admission persists turn settings in one atomic row update. Worker collection +and guarded idle collection release completed execution memory. The execution +pool remains at four workers. Legacy providers now pay worker startup on their +first event. + +These changes were measured incrementally and checked with focused behavior +tests. The final ten-run cohort measures their combined effect. A single +exploratory run does not establish the isolated benefit of each change. + +Several alternatives were rejected. Eight workers did not recover completion +speed. A later three-worker trial reduced memory to 399.9 MiB but took 33.648 s +and reached only 148 events/s. A 1 ms group fill delay also reduced throughput. +Forced full collection during active critical memory pressure reduced memory +but harmed throughput. None of those experiments is the shipped configuration. + +## Cancellation and remaining startup pause + +The separate Stop-one run returned in 1103 ms. It cancelled only the selected +third task. All six peers completed, and all seven persisted audits passed. +The cancelled reconnect snapshot and cleanup also passed. No server pause was +logged. The receipt directory is: + +```text +after-stop-one-2026-09-25T21-15-34-504Z-a763a605-5150-484e-8fcc-7b80ce3738a4 +``` + +A separate run with `MCODE_SERVER_WORK_TRACE=1` passed the same correctness +checks. It recorded an 842 ms startup pause about 1.5 seconds into the run. +The work trace measured the same incident at 835.6 ms. Its window contained +seven publication samples, each below 0.34 ms, with no main-thread event +application or finalization samples. Later trace windows also contained no +event application or finalization work. This does not identify the startup +pause's cause. It remains covered by +[issue #1767](https://github.com/Mzeey-Empire/mcode/issues/1767). + +The traced run's receipt directory is: + +```text +after-2026-09-25T21-17-05-673Z-e750de8c-5e38-46e9-9a1f-1eb3732938fe +``` + +## Desktop checks and automated checks + +The owned Electron app used the freshly built server bundle and its separate +worktree-local database. Two new direct fixture tasks used real Codex with +GPT-5.6 Luna. The reply `ISSUE1760_RECOVERY_FINAL_PASS` survived reload and task +reopening. Both stopped turns retained their partial replies and "You stopped" +state after reload. The public conversation API independently returned the +persisted messages. Only the two recorded task IDs were deleted during cleanup. + +Five warm model-picker opens took 62.8 to 100.8 ms, with a median of 76.3 ms. +Five task switches took 372.7 to 608.2 ms, with a median of 416.0 ms. A terminal +reached its fixture PowerShell prompt in 1189 ms. A command printed its marker +in 126 ms. These are current-build observations, not a matched desktop speed +comparison. + +The first Stop measurement took 4775 ms from the Playwright action call to the +"Stopping" state. It includes actionability waiting and did not measure final +idle restoration. A second check recorded the actual DOM click and restored +the idle composer after 591 ms, or 718 ms from the Playwright action call. +Both cancellations completed. These observations do not establish a Stop +latency distribution or prove that every desktop Stop meets the 2 s target. +Local captures and timings are under `.dev/verification/issue-1760/final-ui-*`. + +Focused checks cover transaction rollback, commit failure, peer isolation, +duplicate delivery, recovery bounds, cloning ownership, atomic settings, +legacy worker startup, and memory collection guards. Existing provider checks +cover Claude and Cursor adapters. There is no new live performance claim for +those providers. + +At `cc6f0ec7`, CI passed tests, lint, typecheck, build, and Linux canary +packaging. The earlier trace test failed on `220.00000000000006` versus `220`. +Its assertion now allows floating-point arithmetic error. Production timing +behavior did not change. From bb1e241d51e52a72757fd9b853e97142094af103 Mon Sep 17 00:00:00 2001 From: "(CJ) Chukwudi Nwobodo" <142016413+chuks-qua@users.noreply.github.com> Date: Fri, 25 Sep 2026 22:36:59 +0100 Subject: [PATCH 129/136] docs(perf): exclude reused terminal output timing --- docs/performance/issue-1760-performance-recovery.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/performance/issue-1760-performance-recovery.md b/docs/performance/issue-1760-performance-recovery.md index f9147808c..debb5d9b7 100644 --- a/docs/performance/issue-1760-performance-recovery.md +++ b/docs/performance/issue-1760-performance-recovery.md @@ -175,9 +175,10 @@ persisted messages. Only the two recorded task IDs were deleted during cleanup. Five warm model-picker opens took 62.8 to 100.8 ms, with a median of 76.3 ms. Five task switches took 372.7 to 608.2 ms, with a median of 416.0 ms. A terminal -reached its fixture PowerShell prompt in 1189 ms. A command printed its marker -in 126 ms. These are current-build observations, not a matched desktop speed -comparison. +reached its fixture PowerShell prompt in 1189 ms. PowerShell printed the expected +command marker. Command output timing is excluded because an earlier identical +marker was already present. These are current-build observations, not a matched +desktop speed comparison. The first Stop measurement took 4775 ms from the Playwright action call to the "Stopping" state. It includes actionability waiting and did not measure final From 626f3e841d57718d4b01bdb5cc1f942242647f83 Mon Sep 17 00:00:00 2001 From: "(CJ) Chukwudi Nwobodo" <142016413+chuks-qua@users.noreply.github.com> Date: Fri, 25 Sep 2026 23:18:09 +0100 Subject: [PATCH 130/136] fix(web): bind follow-up execution before terminal events --- .../references/features/README.md | 1 + .../features/multi-surface-journeys.md | 4 + .../provider-events-and-durability.md | 25 +++++- .../threadStore-followup-runtime.test.ts | 90 +++++++++++++++++++ apps/web/src/stores/threadStore.ts | 4 +- .../issue-1760-followup-regression.md | 54 +++++++++++ 6 files changed, 176 insertions(+), 2 deletions(-) create mode 100644 apps/web/src/__tests__/threadStore-followup-runtime.test.ts create mode 100644 docs/performance/issue-1760-followup-regression.md diff --git a/.agents/skills/verify-mcode/references/features/README.md b/.agents/skills/verify-mcode/references/features/README.md index ae99cca14..9f635abb5 100644 --- a/.agents/skills/verify-mcode/references/features/README.md +++ b/.agents/skills/verify-mcode/references/features/README.md @@ -26,6 +26,7 @@ | Last turn shows native agent changes, settles durably, and identifies Git fallback | [Last turn native diff](turn-diff-review.md) | Codex composer and Review Electron proof, plus focused Git/RPC and migration checks | | A thread starts, runs, stops, clears active state, and retains a cancelled reconnect snapshot | [Thread lifecycle](thread-lifecycle.md) | `runtime live --scenario stop` | | Provider events become durable assistant conversation data | [Provider events and durability](provider-events-and-durability.md) | `runtime live --scenario completion` | +| Consecutive replies settle while switching away and back during streaming | [Provider events and durability](provider-events-and-durability.md#consecutive-turns-and-navigation) | Real-provider Electron follow-ups, mid-turn navigation, idle observation, and reload | | Seven active tasks keep event order while model and terminal controls settle | [Multi-surface journeys](multi-surface-journeys.md#seven-active-tasks-and-controls) | Fixture-owned seven-task public RPC run, then Electron controls and reload | | Codex reroutes, warnings, diagnostics, and authentication recovery remain bounded, durable, and thread-scoped | [Provider events and durability](provider-events-and-durability.md) | Composer notice journey and `runtime health` | | Approval review offers only valid access modes, waits for a real permission request, and keeps Full Access review-free | [Provider events and durability](provider-events-and-durability.md) | Approval review journey and focused policy and mapper tests | diff --git a/.agents/skills/verify-mcode/references/features/multi-surface-journeys.md b/.agents/skills/verify-mcode/references/features/multi-surface-journeys.md index 1555b2a7b..82ac18326 100644 --- a/.agents/skills/verify-mcode/references/features/multi-surface-journeys.md +++ b/.agents/skills/verify-mcode/references/features/multi-surface-journeys.md @@ -36,6 +36,10 @@ final conversation check and [Electron live testing](../../../electorn-live-test for the desktop control and capture. Report any unavailable control or missing capture as a gap. A public RPC receipt alone does not prove the desktop path. +Also run the [consecutive-turn navigation journey](provider-events-and-durability.md#consecutive-turns-and-navigation) +with real Codex. The seven-task fixture starts one turn per task and cannot prove +successful follow-ups or renderer state while switching during streaming. + ## Provider completeness Use this journey when a change affects turn diffs, Review, provider events, automatic review, or workspace invalidation. Resolve the upstream Codex source with the OpenSrc command in the repository instructions before a Codex run. Record the resolved upstream commit in the receipt. The cache is read-only. diff --git a/.agents/skills/verify-mcode/references/features/provider-events-and-durability.md b/.agents/skills/verify-mcode/references/features/provider-events-and-durability.md index 9626b837c..ab61c4f88 100644 --- a/.agents/skills/verify-mcode/references/features/provider-events-and-durability.md +++ b/.agents/skills/verify-mcode/references/features/provider-events-and-durability.md @@ -17,7 +17,8 @@ 1. Open the project for this worktree. 2. Start a short thread with a selected provider and model. 3. Wait for the final reply. -4. Reload or reopen the conversation and confirm that the reply remains. +4. Send a follow-up in the same conversation. Require its reply, cleared activity indicator, and restored Send control. +5. Run the consecutive-turn journey below, then reload and confirm that replies remain. ## Driving it with verify-mcode @@ -32,6 +33,28 @@ This fixture checks the production Codex event boundary, not a real model run or ## Runtime checks +### Consecutive turns and navigation + +In the owned Electron app, select a real provider and model and create a direct +task under `.dev/fixture-repo`. Record its ID before proceeding. + +1. Send a short prompt, wait for completion, then send a second prompt in the + same task. Require both replies and a restored Send control after each turn. +2. Send a longer follow-up. After assistant text starts streaming, switch to + another fixture task and back before completion. Capture active runtime state + before leaving, while away, and after returning. The response must continue + and finish once, without cancellation or text appearing in the other task. +3. Send another follow-up after navigation. Require a completed public runtime, + no Stop control or thinking indicator, and no reactivation for 40 seconds. +4. Reload, reopen the same task, verify the durable replies through + `conversation.page`, and send one more follow-up to completion. +5. Save a video of the mid-turn switch, settled screenshots, terminal outcomes, + provider/model, and exact owned IDs. Delete only the tasks created for proof. + +A first-turn server receipt does not cover this journey. Compare desktop state +with the public runtime from that desktop's server; its isolated database can +differ from the worktree server. Report which providers were actually exercised. + Run `runtime health`, then run: ```sh diff --git a/apps/web/src/__tests__/threadStore-followup-runtime.test.ts b/apps/web/src/__tests__/threadStore-followup-runtime.test.ts new file mode 100644 index 000000000..f537e48e4 --- /dev/null +++ b/apps/web/src/__tests__/threadStore-followup-runtime.test.ts @@ -0,0 +1,90 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { createEmptyThreadRecord } from "@/stores/thread-record"; +import { resetThreadStoreForTests } from "@/stores/thread-store-test-utils"; +import { useThreadStore } from "@/stores/threadStore"; +import { useWorkspaceStore } from "@/features/projects/state/workspaceStore"; +import { createMockThread, mockTransport } from "./mocks/transport"; + +vi.mock("@/transport", async () => ({ + ...(await vi.importActual("@/transport")), + getTransport: () => mockTransport, +})); + +const THREAD_ID = "followup-runtime"; +const OTHER_THREAD_ID = "other-runtime"; + +function record() { + return useThreadStore.getState().records.get(THREAD_ID); +} + +function completeFirstTurn(): void { + const handle = useThreadStore.getState().handleAgentEvent; + handle({ type: "turnStarted", threadId: THREAD_ID, turnExecutionId: "first" }); + handle({ type: "textDelta", threadId: THREAD_ID, turnExecutionId: "first", delta: "First reply" }); + handle({ type: "ended", threadId: THREAD_ID, turnExecutionId: "first", outcome: "completed" }); + expect(record()?.runtimePhase).toBe("completed"); +} + +describe("follow-up runtime ownership", () => { + beforeEach(() => { + vi.useFakeTimers(); + vi.mocked(mockTransport.sendMessage).mockResolvedValue(undefined); + useWorkspaceStore.setState({ + activeThreadId: THREAD_ID, + threads: [createMockThread({ id: THREAD_ID }), createMockThread({ id: OTHER_THREAD_ID })], + }); + resetThreadStoreForTests({ + currentThreadId: THREAD_ID, + runningThreadIds: new Set(), + records: new Map([ + [THREAD_ID, createEmptyThreadRecord()], + [OTHER_THREAD_ID, createEmptyThreadRecord()], + ]), + }); + }); + + afterEach(() => { + vi.useRealTimers(); + }); + + it.each([false, true])("completes a sent follow-up without file-effect metadata, background=%s", async (background) => { + completeFirstTurn(); + expect(await useThreadStore.getState().sendMessage(THREAD_ID, "Second prompt")).toBe(true); + expect(record()?.turnExecutionId).toBeNull(); + if (background) { + useThreadStore.setState({ currentThreadId: OTHER_THREAD_ID }); + useWorkspaceStore.setState({ activeThreadId: OTHER_THREAD_ID }); + } + + const handle = useThreadStore.getState().handleAgentEvent; + handle({ type: "turnStarted", threadId: THREAD_ID, turnExecutionId: "second" }); + expect(record()?.turnExecutionId).toBe("second"); + handle({ type: "textDelta", threadId: THREAD_ID, turnExecutionId: "second", delta: "Second reply" }); + handle({ type: "ended", threadId: THREAD_ID, turnExecutionId: "second", outcome: "completed" }); + + expect(record()?.runtimePhase).toBe("completed"); + expect(useThreadStore.getState().runningThreadIds.has(THREAD_ID)).toBe(false); + expect(record()?.messages.map((message) => message.content)).toContain("Second reply"); + expect(record()?.messages.map((message) => message.content)).toContain("First reply"); + expect(useThreadStore.getState().records.get(OTHER_THREAD_ID)?.runtimePhase).toBe("idle"); + }); + + it("retains the follow-up response on a duplicate start and ignores the previous turn's completion", async () => { + completeFirstTurn(); + await useThreadStore.getState().sendMessage(THREAD_ID, "Second prompt"); + const handle = useThreadStore.getState().handleAgentEvent; + handle({ type: "turnStarted", threadId: THREAD_ID, turnExecutionId: "second" }); + handle({ type: "textDelta", threadId: THREAD_ID, turnExecutionId: "second", delta: "Kept reply" }); + handle({ type: "contextEstimate", threadId: THREAD_ID, turnExecutionId: "second", tokensIn: 20 }); + const responseKey = record()?.currentTurnResponseKey; + handle({ type: "turnStarted", threadId: THREAD_ID, turnExecutionId: "second" }); + handle({ type: "ended", threadId: THREAD_ID, turnExecutionId: "first", outcome: "completed" }); + + expect(record()?.turnExecutionId).toBe("second"); + expect(record()?.runtimePhase).toBe("running"); + expect(record()?.currentTurnResponseKey).toBe(responseKey); + expect(record()?.streaming).toBe("Kept reply"); + handle({ type: "ended", threadId: THREAD_ID, turnExecutionId: "second", outcome: "completed" }); + expect(record()?.runtimePhase).toBe("completed"); + }); +}); diff --git a/apps/web/src/stores/threadStore.ts b/apps/web/src/stores/threadStore.ts index f73bf6d33..e4616371e 100644 --- a/apps/web/src/stores/threadStore.ts +++ b/apps/web/src/stores/threadStore.ts @@ -1385,7 +1385,9 @@ export const useThreadStore = create((zustandSet, get) => { ): void => { const fileEffectTurnId = typeof event.fileEffectTurnId === "string" ? event.fileEffectTurnId : ""; const record = getRec(event.threadId); - if (get().runningThreadIds.has(event.threadId) && record.fileEffectTurnId === fileEffectTurnId) return; + const sameExecution = runtime.incomingExecutionId === undefined + || record.turnExecutionId === runtime.incomingExecutionId; + if (get().runningThreadIds.has(event.threadId) && sameExecution && record.fileEffectTurnId === fileEffectTurnId) return; clearStreamingTextUsage(event.threadId); useTaskStore.getState().prepareTaskBubbleForNewTurn(event.threadId); patchRec(event.threadId, (current) => ({ diff --git a/docs/performance/issue-1760-followup-regression.md b/docs/performance/issue-1760-followup-regression.md new file mode 100644 index 000000000..d8fb45d7a --- /dev/null +++ b/docs/performance/issue-1760-followup-regression.md @@ -0,0 +1,54 @@ +# Follow-up completion regression, 2026-09-25 + +## Reproduction + +The owned Electron dev app at `bb1e241d` reproduced the reported failure with +real Codex `gpt-5.6-luna`. The first message completed. The second reply arrived +and the server persisted a completed turn in about three seconds, but the UI +still showed Thinking and Stop after 50 seconds. Public `agent.activeCount` was +zero while the renderer retained `runtimePhase: running` and a null execution ID. + +The earlier completion, Stop, reload, and seven-task benchmark checks did not +cover consecutive successful follow-ups. Those results did not establish that +this interaction worked. + +## Cause and correction + +Optimistic send clears the execution ID and marks the task running. Worker-owned +`turnStarted` supplies a new execution ID without optional file-effect metadata. +The renderer compared two empty file-effect IDs, incorrectly classified the +start as a duplicate, and skipped binding the execution. Terminal events then +failed the execution ownership check. + +The duplicate guard now also checks execution identity. It still preserves the +existing file-generation comparison, duplicate-start behavior, and stale-event +fence. No backend scheduling or persistence behavior changed in this correction. + +## Verification + +The same owned Electron app was cleanly relaunched with the renderer fix. +All task mutations used this worktree's `.dev/fixture-repo`. + +| Real Codex desktop action | Observed result | +| --- | --- | +| First prompt and short follow-up | Replies appeared and Send returned; follow-up reached idle in 5.063 seconds | +| Longer follow-up; switch away after text starts, then return before completion | Public active count stayed 1 before, during, and after navigation; streaming and Stop remained visible on return | +| Completion of the switched turn | Final marker appeared, active count became 0, and Stop disappeared | +| Another follow-up after navigation | Reply completed and remained idle for 46.552 seconds | +| Reload, reopen, and another follow-up | Saved reply restored; new reply completed normally | +| Durable public conversation | Five assistant messages and five distinct completed execution receipts, with every expected marker | +| Renderer errors and cleanup | No page errors; both owned reproduction tasks deleted and owned Electron stopped | + +The new regression tests failed before the fix. Afterward, all 109 tests across +follow-up ownership, event branches, task isolation, and turn persistence passed. +Web TypeScript checking and targeted Oxlint passed. Independent read-only review +found no actionable issue in the guard or tests. + +Evidence is retained under `.dev/verification/issue-1760-followup/` and uploaded +to PR #1768. The verification feature map now requires consecutive turns, +navigation during streaming, delayed-event observation, and a post-reload turn. + +This proves the reported Codex text-response workflow in Electron. It is not a +new performance benchmark, a Claude/Cursor live proof, or a file-change Review +proof. Separate source-review concerns about late MCP attribution and worker +file-summary publication remain outside this correction. From 81e6f25b7c942f47fd2c54499c87039584a040b1 Mon Sep 17 00:00:00 2001 From: "(CJ) Chukwudi Nwobodo" <142016413+chuks-qua@users.noreply.github.com> Date: Sat, 26 Sep 2026 16:10:48 +0100 Subject: [PATCH 131/136] fix(server): renew event generations after journal eviction --- .../transport/__tests__/push.test.ts | 48 ++++++++++++++++++- apps/server/src/application/transport/push.ts | 30 +++++++----- 2 files changed, 65 insertions(+), 13 deletions(-) diff --git a/apps/server/src/application/transport/__tests__/push.test.ts b/apps/server/src/application/transport/__tests__/push.test.ts index 01e229e63..ee37655eb 100644 --- a/apps/server/src/application/transport/__tests__/push.test.ts +++ b/apps/server/src/application/transport/__tests__/push.test.ts @@ -11,9 +11,10 @@ import { setClientThreadSubscriptions, subscribeClientToThread, unsubscribeClientFromThread, + MAX_AGENT_EVENT_JOURNAL_THREADS, _resetForTest, } from "../push.js"; -import { decodeTerminalDataFrame } from "@mcode/contracts"; +import { AgentEventSchema, decodeTerminalDataFrame } from "@mcode/contracts"; import { createPassThroughTransportPayloadValidator, createValidatingTransportPayloadValidator, @@ -272,6 +273,51 @@ describe("broadcast", () => { expect(received.map((entry) => JSON.parse(entry.buf.toString("utf-8")).data.sequence)).toEqual([1, 2, 3]); }); + it("keeps an evicted thread's next event distinguishable from its old cursor", () => { + const first = AgentEventSchema().parse(broadcast("agent.event", { type: "turnStarted", threadId: "evicted-thread" })); + if (!first.epoch) throw new Error("Agent event has no journal epoch"); + for (let index = 0; index < MAX_AGENT_EVENT_JOURNAL_THREADS; index++) { + broadcast("agent.event", { type: "turnStarted", threadId: `other-${index}` }); + } + const next = broadcast("agent.event", { type: "turnStarted", threadId: "evicted-thread" }); + expect(next).not.toEqual(first); + const received: Array<{ buf: Buffer; binary: boolean }> = []; + const ws = fakeOpenSocket(received); + addClient(ws); + const result = setClientThreadSubscriptions(ws, ["evicted-thread"], { + "evicted-thread": { epoch: first.epoch, sequence: 1 }, + }); + expect(result.hydrationRequiredThreadIds).toEqual(["evicted-thread"]); + expect(received).toHaveLength(0); + }); + + it("requires hydration for an ambiguous legacy cursor or a cursor ahead of its journal", () => { + const first = AgentEventSchema().parse(broadcast("agent.event", { type: "turnStarted", threadId: "cursor-thread" })); + if (!first.epoch) throw new Error("Agent event has no journal epoch"); + const received: Array<{ buf: Buffer; binary: boolean }> = []; + const ws = fakeOpenSocket(received); + addClient(ws); + for (const cursor of [1, { epoch: first.epoch, sequence: 2 }]) { + const result = setClientThreadSubscriptions(ws, ["cursor-thread"], { "cursor-thread": cursor }); + expect(result.hydrationRequiredThreadIds).toEqual(["cursor-thread"]); + } + expect(received).toHaveLength(0); + }); + + it("replays only newer events for a cursor in the current journal generation", () => { + const first = AgentEventSchema().parse(broadcast("agent.event", { type: "turnStarted", threadId: "current-thread" })); + if (!first.epoch) throw new Error("Agent event has no journal epoch"); + broadcast("agent.event", { type: "textDelta", threadId: "current-thread", delta: "new text" }); + const received: Array<{ buf: Buffer; binary: boolean }> = []; + const ws = fakeOpenSocket(received); + addClient(ws); + const result = setClientThreadSubscriptions(ws, ["current-thread"], { + "current-thread": { epoch: first.epoch, sequence: 1 }, + }); + expect(result).toEqual({ hydrationRequiredThreadIds: [], replayedThrough: { "current-thread": 2 } }); + expect(received).toHaveLength(1); + }); + it("does not replay history for legacy subscription calls without cursors", () => { const received: Array<{ buf: Buffer; binary: boolean }> = []; broadcast("agent.event", { type: "textDelta", threadId: "thread-legacy", delta: "old" }); diff --git a/apps/server/src/application/transport/push.ts b/apps/server/src/application/transport/push.ts index 57203ef1c..a3b187c25 100644 --- a/apps/server/src/application/transport/push.ts +++ b/apps/server/src/application/transport/push.ts @@ -19,9 +19,13 @@ export const MAX_QUEUED_PUSH_BYTES = 16 * 1_024 * 1_024; const clients = new Set(); const queuedPushBytes = new Map(); const threadSubscriptions = new Map>(); -const threadJournals = new Map(); -const nextSequenceByThread = new Map(); -const eventEpoch = NodeCrypto.randomUUID(); +interface AgentEventJournal { + epoch: string; + sequence: number; + events: unknown[]; +} + +const threadJournals = new Map(); const SUBSCRIPTION_SCOPED_CHANNELS = new Set([ "agent.event", "agent.canonical", @@ -134,10 +138,12 @@ function replayThreadSubscription( function requiresThreadHydration( rawCursor: NonNullable[string], cursor: number, - journal: { events: unknown[] } | undefined, + journal: AgentEventJournal | undefined, ): boolean { - if (typeof rawCursor !== "number" && rawCursor.epoch !== eventEpoch) return true; + if (typeof rawCursor === "number" && cursor > 0) return true; if (!journal) return cursor > 0; + if (typeof rawCursor !== "number" && rawCursor.epoch !== journal.epoch) return true; + if (cursor > journal.sequence) return true; return journal.events.length > 0 && cursor < (journal.events[0] as { sequence: number }).sequence - 1; } @@ -206,21 +212,22 @@ export function broadcast( function decorateAgentEvent(channel: WsChannelName, data: unknown, threadId: string | undefined): unknown { if (channel !== "agent.event" || !threadId || !data || typeof data !== "object") return data; - const sequence = (nextSequenceByThread.get(threadId) ?? 0) + 1; - return { ...(data as Record), sequence, epoch: eventEpoch }; + const journal = threadJournals.get(threadId); + const sequence = (journal?.sequence ?? 0) + 1; + const epoch = journal?.epoch ?? NodeCrypto.randomUUID(); + return { ...(data as Record), sequence, epoch }; } function retainAgentEvent(channel: WsChannelName, threadId: string | undefined, event: unknown): void { if (channel !== "agent.event" || !threadId) return; - nextSequenceByThread.delete(threadId); - nextSequenceByThread.set(threadId, (event as { sequence: number }).sequence); + if (!event || typeof event !== "object" || !("sequence" in event) || !("epoch" in event)) return; + if (typeof event.sequence !== "number" || typeof event.epoch !== "string") return; const existing = threadJournals.get(threadId); const events = existing ? [...existing.events, event] : [event]; events.splice(0, Math.max(0, events.length - MAX_AGENT_EVENT_JOURNAL_EVENTS_PER_THREAD)); threadJournals.delete(threadId); - threadJournals.set(threadId, { events }); + threadJournals.set(threadId, { epoch: event.epoch, sequence: event.sequence, events }); trimJournalMap(threadJournals); - trimJournalMap(nextSequenceByThread); } function trimJournalMap(map: Map): void { @@ -336,5 +343,4 @@ export function _resetForTest(): void { queuedPushBytes.clear(); threadSubscriptions.clear(); threadJournals.clear(); - nextSequenceByThread.clear(); } From 419b53743149ac311a2649d15caa2329d7dfe51a Mon Sep 17 00:00:00 2001 From: "(CJ) Chukwudi Nwobodo" <142016413+chuks-qua@users.noreply.github.com> Date: Sat, 26 Sep 2026 16:26:28 +0100 Subject: [PATCH 132/136] fix(web): retain live publications when cursor writes fail --- .../threadStore-followup-runtime.test.ts | 18 ++++++++++ .../stable-agent-event-publications.test.ts | 27 +++++++++++++- .../stable-agent-event-publications.ts | 36 ++++++++++++++----- 3 files changed, 72 insertions(+), 9 deletions(-) diff --git a/apps/web/src/__tests__/threadStore-followup-runtime.test.ts b/apps/web/src/__tests__/threadStore-followup-runtime.test.ts index f537e48e4..2e5640d02 100644 --- a/apps/web/src/__tests__/threadStore-followup-runtime.test.ts +++ b/apps/web/src/__tests__/threadStore-followup-runtime.test.ts @@ -44,9 +44,27 @@ describe("follow-up runtime ownership", () => { }); afterEach(() => { + vi.restoreAllMocks(); vi.useRealTimers(); }); + it("completes a follow-up when the publication cursor write fails", async () => { + completeFirstTurn(); + await useThreadStore.getState().sendMessage(THREAD_ID, "Second prompt"); + const original = Storage.prototype.setItem; + vi.spyOn(Storage.prototype, "setItem").mockImplementation(function (this: Storage, key, value) { + if (key.startsWith("mcode-agent-publication-v2:")) throw new DOMException("quota", "QuotaExceededError"); + original.call(this, key, value); + }); + const handle = useThreadStore.getState().handleAgentEvent; + handle({ type: "turnStarted", threadId: THREAD_ID, turnExecutionId: "second", sequence: 1, publicationId: "1" }); + handle({ type: "textDelta", threadId: THREAD_ID, turnExecutionId: "second", sequence: 2, publicationId: "2", delta: "Visible reply" }); + handle({ type: "ended", threadId: THREAD_ID, turnExecutionId: "second", sequence: 3, publicationId: "3", outcome: "completed" }); + expect(record()?.runtimePhase).toBe("completed"); + expect(record()?.messages.map((message) => message.content)).toContain("Visible reply"); + expect(useThreadStore.getState().runningThreadIds.has(THREAD_ID)).toBe(false); + }); + it.each([false, true])("completes a sent follow-up without file-effect metadata, background=%s", async (background) => { completeFirstTurn(); expect(await useThreadStore.getState().sendMessage(THREAD_ID, "Second prompt")).toBe(true); diff --git a/apps/web/src/stores/thread-store/__tests__/stable-agent-event-publications.test.ts b/apps/web/src/stores/thread-store/__tests__/stable-agent-event-publications.test.ts index f8282b488..01efb4692 100644 --- a/apps/web/src/stores/thread-store/__tests__/stable-agent-event-publications.test.ts +++ b/apps/web/src/stores/thread-store/__tests__/stable-agent-event-publications.test.ts @@ -55,12 +55,37 @@ describe("stable AgentEvent publications", () => { expect(secondTab.accept(event(1))).toBe(false); }); - it("fails closed if its durable cursor cannot be stored", () => { + it("applies each publication once when its cursor write fails", () => { const cursor = new StableAgentEventPublications(() => ({ getItem: () => null, setItem: () => { throw new Error("quota"); }, })); + expect(cursor.accept(event(1))).toBe(true); expect(cursor.accept(event(1))).toBe(false); + expect(cursor.accept(event(2))).toBe(true); + expect(cursor.accept(event(1))).toBe(false); + }); + + it("retains a failed write over the older stored cursor and persists again after recovery", () => { + const shared = storage(); + let failWrites = false; + const cursor = new StableAgentEventPublications(() => ({ + getItem: shared.getItem, + setItem: (key, value) => { + if (failWrites) throw new Error("quota"); + shared.setItem(key, value); + }, + })); + expect(cursor.accept(event(6))).toBe(true); + failWrites = true; + expect(cursor.accept(event(7))).toBe(true); + expect(cursor.accept(event(7))).toBe(false); + failWrites = false; + expect(cursor.accept(event(8))).toBe(true); + const reloaded = new StableAgentEventPublications(() => shared); + expect(reloaded.accept(event(7))).toBe(false); + expect(reloaded.accept(event(8))).toBe(false); + expect(reloaded.accept(event(9))).toBe(true); }); it("fails closed on an oversized stored cursor", () => { diff --git a/apps/web/src/stores/thread-store/stable-agent-event-publications.ts b/apps/web/src/stores/thread-store/stable-agent-event-publications.ts index 2a68e8386..16b518063 100644 --- a/apps/web/src/stores/thread-store/stable-agent-event-publications.ts +++ b/apps/web/src/stores/thread-store/stable-agent-event-publications.ts @@ -11,11 +11,14 @@ function publicationSequence(value: string): number | null { return Number.isSafeInteger(sequence) ? sequence : null; } -/** Retains one per-thread cursor across server epochs, executions, and browser reloads. */ +/** Retains a per-thread cursor, surviving reloads when storage writes succeed. */ export class StableAgentEventPublications { + private readonly pendingWrites = new Map(); + private warnedAboutWriteFailure = false; + constructor(private readonly storage: () => PublicationStorage | undefined) {} - /** Reserve a publication before applying its UI effects. A storage failure fails closed. */ + /** Reserve a publication before applying its UI effects, retaining failed writes in memory. */ accept(event: AgentEvent): boolean { if (!event.publicationId) return true; if (!event.turnExecutionId || publicationSequence(event.publicationId) === null) return false; @@ -27,15 +30,32 @@ export class StableAgentEventPublications { } private reserve(threadId: string, publicationId: string): boolean { - const store = this.storage(); - if (!store) return false; - const key = `${STORAGE_PREFIX}${threadId}`; - const stored = store.getItem(key); - const previous = stored === null ? 0 : publicationSequence(stored); + const previous = this.pendingWrites.get(threadId) ?? this.readCursor(threadId); if (previous === null || Number(publicationId) <= previous) return false; - store.setItem(key, publicationId); + this.pendingWrites.set(threadId, Number(publicationId)); + this.persistCursor(threadId, publicationId); return true; } + + private readCursor(threadId: string): number | null { + const store = this.storage(); + if (!store) return null; + const stored = store.getItem(`${STORAGE_PREFIX}${threadId}`); + return stored === null ? 0 : publicationSequence(stored); + } + + private persistCursor(threadId: string, publicationId: string): void { + try { + const store = this.storage(); + if (!store) throw new Error("Publication cursor storage is unavailable"); + store.setItem(`${STORAGE_PREFIX}${threadId}`, publicationId); + this.pendingWrites.delete(threadId); + } catch { + if (this.warnedAboutWriteFailure) return; + this.warnedAboutWriteFailure = true; + console.warn("Live event cursor could not be saved. Duplicate protection remains active in this renderer; reload recovery may replay older events."); + } + } } /** Browser-tab cursor with constant retained bytes per thread. */ From 2842af4a9e83e43b7f2acc16b33c2cb5e8dd6b89 Mon Sep 17 00:00:00 2001 From: Agent Runtime Fixture Date: Sun, 27 Sep 2026 12:13:30 +0100 Subject: [PATCH 133/136] docs(performance): drop issue-scoped working notes --- .../issue-1760-followup-regression.md | 54 ----- .../issue-1760-performance-recovery.md | 200 ------------------ .../issue-1760-seven-task-evidence.md | 83 -------- docs/performance/message-list-baseline.md | 180 ---------------- .../tanstack-virtual-current-trial.md | 181 ---------------- .../vlist-react-adapter-prototype.md | 73 ------- 6 files changed, 771 deletions(-) delete mode 100644 docs/performance/issue-1760-followup-regression.md delete mode 100644 docs/performance/issue-1760-performance-recovery.md delete mode 100644 docs/performance/issue-1760-seven-task-evidence.md delete mode 100644 docs/performance/message-list-baseline.md delete mode 100644 docs/performance/tanstack-virtual-current-trial.md delete mode 100644 docs/performance/vlist-react-adapter-prototype.md diff --git a/docs/performance/issue-1760-followup-regression.md b/docs/performance/issue-1760-followup-regression.md deleted file mode 100644 index d8fb45d7a..000000000 --- a/docs/performance/issue-1760-followup-regression.md +++ /dev/null @@ -1,54 +0,0 @@ -# Follow-up completion regression, 2026-09-25 - -## Reproduction - -The owned Electron dev app at `bb1e241d` reproduced the reported failure with -real Codex `gpt-5.6-luna`. The first message completed. The second reply arrived -and the server persisted a completed turn in about three seconds, but the UI -still showed Thinking and Stop after 50 seconds. Public `agent.activeCount` was -zero while the renderer retained `runtimePhase: running` and a null execution ID. - -The earlier completion, Stop, reload, and seven-task benchmark checks did not -cover consecutive successful follow-ups. Those results did not establish that -this interaction worked. - -## Cause and correction - -Optimistic send clears the execution ID and marks the task running. Worker-owned -`turnStarted` supplies a new execution ID without optional file-effect metadata. -The renderer compared two empty file-effect IDs, incorrectly classified the -start as a duplicate, and skipped binding the execution. Terminal events then -failed the execution ownership check. - -The duplicate guard now also checks execution identity. It still preserves the -existing file-generation comparison, duplicate-start behavior, and stale-event -fence. No backend scheduling or persistence behavior changed in this correction. - -## Verification - -The same owned Electron app was cleanly relaunched with the renderer fix. -All task mutations used this worktree's `.dev/fixture-repo`. - -| Real Codex desktop action | Observed result | -| --- | --- | -| First prompt and short follow-up | Replies appeared and Send returned; follow-up reached idle in 5.063 seconds | -| Longer follow-up; switch away after text starts, then return before completion | Public active count stayed 1 before, during, and after navigation; streaming and Stop remained visible on return | -| Completion of the switched turn | Final marker appeared, active count became 0, and Stop disappeared | -| Another follow-up after navigation | Reply completed and remained idle for 46.552 seconds | -| Reload, reopen, and another follow-up | Saved reply restored; new reply completed normally | -| Durable public conversation | Five assistant messages and five distinct completed execution receipts, with every expected marker | -| Renderer errors and cleanup | No page errors; both owned reproduction tasks deleted and owned Electron stopped | - -The new regression tests failed before the fix. Afterward, all 109 tests across -follow-up ownership, event branches, task isolation, and turn persistence passed. -Web TypeScript checking and targeted Oxlint passed. Independent read-only review -found no actionable issue in the guard or tests. - -Evidence is retained under `.dev/verification/issue-1760-followup/` and uploaded -to PR #1768. The verification feature map now requires consecutive turns, -navigation during streaming, delayed-event observation, and a post-reload turn. - -This proves the reported Codex text-response workflow in Electron. It is not a -new performance benchmark, a Claude/Cursor live proof, or a file-change Review -proof. Separate source-review concerns about late MCP attribution and worker -file-summary publication remain outside this correction. diff --git a/docs/performance/issue-1760-performance-recovery.md b/docs/performance/issue-1760-performance-recovery.md deleted file mode 100644 index debb5d9b7..000000000 --- a/docs/performance/issue-1760-performance-recovery.md +++ /dev/null @@ -1,200 +0,0 @@ -# Issue 1760: performance recovery - -Measured on September 25, 2026, on Windows with Bun 1.4.1. This record follows -the [original comparison](issue-1760-seven-task-evidence.md). It preserves that -earlier result and reports the optimized runtime at `8a498f7a` separately. -The later `cc6f0ec7` changes a test's floating-point assertion only. - -The optimized version improves all five nonzero metric medians from the -initial PR. Its median largest logged pause remains zero. Four of six medians -also improve on the original implementation. Peak public event rate and sampled -memory still miss the original targets. The request to improve every original -metric is therefore not fully met. - -## Ten-run comparison - -Each cell contains the median, followed by the minimum and maximum in brackets. -Times are milliseconds. Lower is better except for public event rate. - -| Metric | Original implementation | Initial PR | Optimized PR | -| --- | ---: | ---: | ---: | -| Largest logged server pause | 6491 [4893, 8079] | 0 [0, 1096] | 0 [0, 0] | -| Per-run task completion p95 | 32931 [22705, 47756] | 72911 [70481, 75846] | 26053 [23928, 30139] | -| Peak public events per second | 231 [177, 270] | 147.5 [138, 156] | 205.5 [170, 219] | -| Maximum sampled server working set, MiB | 446.4 [367.5, 623.9] | 588.7 [538.2, 627.6] | 458.3 [430.9, 493.1] | -| Terminal creation | 746 [103, 25863] | 196 [166, 1008] | 175.6 [126.9, 608.9] | -| Model list | 29 [11, 133] | 35 [7, 476] | 19.6 [5.0, 117.9] | - -Against the initial PR, completion takes 64.3% less time, peak event rate is -39.3% higher, and sampled memory is 22.2% lower. Terminal creation takes 10.4% -less time and model listing takes 44.0% less time. - -Against the original implementation, completion takes 20.9% less time. -Peak event rate remains 11.0% lower and sampled memory remains 2.7% higher. -Those two remaining gaps are not evidence of a proven performance ceiling. - -## Workload and measurement limits - -The unchanged public harness starts seven direct Codex tasks and one terminal -in this worktree's fixture repository. Each provider fixture emits 120 tool -pairs. The benchmark uses the checked-in provider fixture, without upstream -model calls. The separate desktop checks below use the real Codex provider. - -- Completion is the p95 completion time among each run's seven tasks, then the - median of those ten values. It is not a pooled p95 across all 70 tasks. -- Event rate is the largest one-second bucket of public event arrivals. It is - not average processing throughput. -- Memory is the maximum of three server working-set samples, taken before - dispatch, after the turns, and after cleanup. It is not a continuous peak. - This clarifies the earlier document's shorter label, "peak server memory." -- A zero pause means no delay was logged by the default diagnostic. It does not - mean that the event loop had zero latency. -- Terminal and model timings are one request per run while tasks are active. -- The original baseline is `15fa7030ad09b5c2d083efeb2e0670b68277220c`. The initial - PR runtime is `0a73175d`. These historical cohorts used the same host and - workload at different times. Host load was not held constant. -- No tests or builds ran concurrently with this final cohort. An unrelated - server process consumed roughly one CPU core during it. Its contribution to - the remaining differences was not established. - -All ten optimized runs passed event ordering, duplicate and missing-event -checks, persisted conversation audits, and fixture-only cleanup. Each delivered -3466 public events. The maximum queue observations were seven pending commands, -one queued command per worker, and four in flight. The original baseline did -not record a comparable queue depth. - -## Every optimized run - -All rows had zero logged main-server pauses. Memory is MiB and times are ms. - -| Run | Completion p95 | Peak events/s | Sampled memory | Terminal | Models | -| --- | ---: | ---: | ---: | ---: | ---: | -| 1 | 29904.329 | 179 | 430.895 | 608.941 | 117.923 | -| 2 | 30139.381 | 187 | 453.777 | 325.490 | 16.305 | -| 3 | 29335.843 | 170 | 444.551 | 230.683 | 25.523 | -| 4 | 25944.837 | 209 | 454.813 | 139.852 | 27.889 | -| 5 | 24035.207 | 208 | 437.164 | 196.337 | 55.097 | -| 6 | 26161.430 | 216 | 472.516 | 171.915 | 5.037 | -| 7 | 29806.950 | 180 | 471.328 | 178.944 | 10.793 | -| 8 | 25148.070 | 203 | 493.098 | 126.890 | 18.474 | -| 9 | 24624.156 | 213 | 474.363 | 172.181 | 18.552 | -| 10 | 23927.607 | 219 | 461.770 | 145.625 | 20.682 | - -The local cohort manifest is `.dev/verification/issue-1760/final-cohort.json`. -Each receipt is under `.dev/verification/performance/seven-thread-live/`, in -the following directory, with filename `receipt.json`. These names identify -local artifacts; the table above is the committed reviewable result. - -```text -after-2026-09-25T21-10-23-813Z-4bf144ab-f4b5-49df-88d4-8c80a0d0b9d8 -after-2026-09-25T21-10-56-894Z-a115cc20-328a-4dea-a2b3-d31faffde650 -after-2026-09-25T21-11-30-031Z-c95b34d0-9f6d-44ed-b57a-804f8ff6d0db -after-2026-09-25T21-12-03-033Z-e8d1e249-9f15-453c-9d18-afd9581fac85 -after-2026-09-25T21-12-32-933Z-9ad255f1-fd1f-4b16-8291-56ef49b7f660 -after-2026-09-25T21-12-59-911Z-4e4ab32b-7281-4bdf-9219-125ae026753b -after-2026-09-25T21-13-28-694Z-5b62691c-9ce3-4d20-b2d9-2db090e395bd -after-2026-09-25T21-14-01-558Z-d7ca1563-f27f-4bf2-9884-f9d2559a47ed -after-2026-09-25T21-14-29-420Z-c22652c7-ec2a-4424-866d-682b5f8ddd60 -after-2026-09-25T21-14-56-624Z-c24b17b0-d971-494d-b3db-36f4a028f7cf -``` - -The public workload is reproducible with: - -```powershell -bun scripts/perf/seven-thread-live-harness.mjs --run --confirm-run --label after -``` - -## Changes supported by the measurements - -The first trace found repeated whole-thread loads during terminal writes. -Loading transaction-local terminal state reduced 1701 loads taking 18.312 s -to 66 loads taking 26 ms in the scoped comparison. Reusing consecutive -narrative message resolutions and prepared queries removed more repeated work. -The resolver preserves the original lookup order when multiple canonical -sources share a display message. - -The writer now groups already-ready independent appends within bounds of eight -operations, 512 KiB, and 16 ms. It adds no fill delay. Repeated executions and -control operations remain ordering barriers. Acknowledgments and publication -wait for the physical commit. A failed group rolls back and retries semantic -writes individually, without replaying provider actions. One operation can -exceed the time bound before the next boundary check. - -The remaining changes reduce allocation and startup work. Worker bundles omit -unused schemas. Legacy event workers start on their first event. Recovery -checkpoints copy changed tool records and calculate deltas before cloning. -Admission persists turn settings in one atomic row update. Worker collection -and guarded idle collection release completed execution memory. The execution -pool remains at four workers. Legacy providers now pay worker startup on their -first event. - -These changes were measured incrementally and checked with focused behavior -tests. The final ten-run cohort measures their combined effect. A single -exploratory run does not establish the isolated benefit of each change. - -Several alternatives were rejected. Eight workers did not recover completion -speed. A later three-worker trial reduced memory to 399.9 MiB but took 33.648 s -and reached only 148 events/s. A 1 ms group fill delay also reduced throughput. -Forced full collection during active critical memory pressure reduced memory -but harmed throughput. None of those experiments is the shipped configuration. - -## Cancellation and remaining startup pause - -The separate Stop-one run returned in 1103 ms. It cancelled only the selected -third task. All six peers completed, and all seven persisted audits passed. -The cancelled reconnect snapshot and cleanup also passed. No server pause was -logged. The receipt directory is: - -```text -after-stop-one-2026-09-25T21-15-34-504Z-a763a605-5150-484e-8fcc-7b80ce3738a4 -``` - -A separate run with `MCODE_SERVER_WORK_TRACE=1` passed the same correctness -checks. It recorded an 842 ms startup pause about 1.5 seconds into the run. -The work trace measured the same incident at 835.6 ms. Its window contained -seven publication samples, each below 0.34 ms, with no main-thread event -application or finalization samples. Later trace windows also contained no -event application or finalization work. This does not identify the startup -pause's cause. It remains covered by -[issue #1767](https://github.com/Mzeey-Empire/mcode/issues/1767). - -The traced run's receipt directory is: - -```text -after-2026-09-25T21-17-05-673Z-e750de8c-5e38-46e9-9a1f-1eb3732938fe -``` - -## Desktop checks and automated checks - -The owned Electron app used the freshly built server bundle and its separate -worktree-local database. Two new direct fixture tasks used real Codex with -GPT-5.6 Luna. The reply `ISSUE1760_RECOVERY_FINAL_PASS` survived reload and task -reopening. Both stopped turns retained their partial replies and "You stopped" -state after reload. The public conversation API independently returned the -persisted messages. Only the two recorded task IDs were deleted during cleanup. - -Five warm model-picker opens took 62.8 to 100.8 ms, with a median of 76.3 ms. -Five task switches took 372.7 to 608.2 ms, with a median of 416.0 ms. A terminal -reached its fixture PowerShell prompt in 1189 ms. PowerShell printed the expected -command marker. Command output timing is excluded because an earlier identical -marker was already present. These are current-build observations, not a matched -desktop speed comparison. - -The first Stop measurement took 4775 ms from the Playwright action call to the -"Stopping" state. It includes actionability waiting and did not measure final -idle restoration. A second check recorded the actual DOM click and restored -the idle composer after 591 ms, or 718 ms from the Playwright action call. -Both cancellations completed. These observations do not establish a Stop -latency distribution or prove that every desktop Stop meets the 2 s target. -Local captures and timings are under `.dev/verification/issue-1760/final-ui-*`. - -Focused checks cover transaction rollback, commit failure, peer isolation, -duplicate delivery, recovery bounds, cloning ownership, atomic settings, -legacy worker startup, and memory collection guards. Existing provider checks -cover Claude and Cursor adapters. There is no new live performance claim for -those providers. - -At `cc6f0ec7`, CI passed tests, lint, typecheck, build, and Linux canary -packaging. The earlier trace test failed on `220.00000000000006` versus `220`. -Its assertion now allows floating-point arithmetic error. Production timing -behavior did not change. diff --git a/docs/performance/issue-1760-seven-task-evidence.md b/docs/performance/issue-1760-seven-task-evidence.md deleted file mode 100644 index 9de4da6f7..000000000 --- a/docs/performance/issue-1760-seven-task-evidence.md +++ /dev/null @@ -1,83 +0,0 @@ -# Issue 1760: seven-task evidence - -Measured on Windows with Bun 1.4.1 and the public seven-task fixture harness. The -baseline was `15fa7030ad09b5c2d083efeb2e0670b68277220c`; the candidate -runtime was merge commit `0a73175d`. Each row is one sequential run with seven -Codex tasks and one terminal. All 20 runs passed the persisted reload audit, -had no missing, duplicate, or reordered events, and cleaned up their own -resources. The candidate uses four execution workers. The diagnostic-only fix -in `fe98cec3` does not change the default runtime path. - -`Stall` is the largest logged main-server event-loop delay during the run; zero -means none was logged. `Rate` is the largest one-second bucket of public event -arrivals. The baseline rate is derived from its saved arrival timestamps using -the candidate harness's bucket formula. `Memory` is peak server working set in -MiB. `Finish` is the p95 completion time among that run's seven tasks. Terminal -and models are the single RPC observation in each run. Times are milliseconds. - -| Run | Events | Stall | Rate/s | Memory | Finish | Terminal | Models | Queued/worker | -| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | -| Before 1 | 3458 | 7775 | 177 | 378.6 | 47756 | 24197 | 133 | unavailable | -| Before 2 | 3458 | 8079 | 245 | 623.9 | 29606 | 2605 | 40 | unavailable | -| Before 3 | 3458 | 6956 | 250 | 445.4 | 26953 | 25863 | 14 | unavailable | -| Before 4 | 3458 | 6483 | 270 | 447.3 | 25621 | 809 | 35 | unavailable | -| Before 5 | 3458 | 5871 | 259 | 460.8 | 24854 | 684 | 119 | unavailable | -| Before 6 | 3458 | 6322 | 200 | 367.5 | 36525 | 7115 | 128 | unavailable | -| Before 7 | 3458 | 7096 | 207 | 538.0 | 36256 | 133 | 24 | unavailable | -| Before 8 | 3459 | 6309 | 208 | 533.1 | 37795 | 146 | 11 | unavailable | -| Before 9 | 3459 | 6499 | 217 | 394.1 | 38664 | 117 | 16 | unavailable | -| Before 10 | 3459 | 4893 | 258 | 417.6 | 22705 | 103 | 24 | unavailable | -| After 1 | 3465 | 0 | 147 | 555.3 | 73301 | 200 | 18 | 1 | -| After 2 | 3466 | 0 | 138 | 538.2 | 75846 | 217 | 52 | 1 | -| After 3 | 3466 | 0 | 148 | 574.3 | 73599 | 176 | 9 | 1 | -| After 4 | 3466 | 0 | 149 | 576.4 | 75128 | 284 | 56 | 1 | -| After 5 | 3466 | 0 | 148 | 592.4 | 72390 | 205 | 36 | 1 | -| After 6 | 3466 | 0 | 142 | 590.1 | 72521 | 193 | 7 | 1 | -| After 7 | 3466 | 0 | 156 | 602.7 | 70955 | 166 | 49 | 1 | -| After 8 | 3466 | 0 | 141 | 627.6 | 75179 | 181 | 17 | 1 | -| After 9 | 3466 | 0 | 156 | 608.3 | 70481 | 175 | 34 | 1 | -| After 10 | 3466 | 1096 | 142 | 587.2 | 71338 | 1008 | 476 | 1 | - -| Metric | Before median (range) | After median (range) | -| --- | ---: | ---: | -| Largest server pause, ms | 6491 (4893–8079) | 0 (0–1096) | -| Peak public events/s | 231 (177–270) | 147.5 (138–156) | -| Peak server memory, MiB | 446.4 (367.5–623.9) | 588.7 (538.2–627.6) | -| Per-run finish p95, ms | 32931 (22705–47756) | 72911 (70481–75846) | -| Terminal creation, ms | 746 (103–25863) | 196 (166–1008) | -| Model list, ms | 29 (11–133) | 35 (7–476) | - -The candidate queue had at most seven pending commands and one queued command -per worker. The baseline harness did not record a comparable queue depth. -The change sharply reduced the observed long server pauses and kept the tested -controls within their budgets. It also reduced peak event throughput by about -36%, more than doubled completion p95, and raised median peak working set by -about 142 MiB. These are material costs, not an overall latency win. An -exploratory eight-worker run did not improve completion and used more memory; -the fixed pool remains at four. - -The stopped-task run durably cancelled only its selected execution, while the -six peers completed in order. Stop returned in 584 ms and terminal creation in -914 ms; cleanup passed. It recorded a 770 ms server pause 1.25 seconds after -startup. After run 10 recorded 1096 ms at 1.73 seconds, while provider sends -and terminal startup were active. Work tracing was off for these default runs, -so the exact operation causing either pause is unknown. They must not be -credited to, or ruled out from, a specific operation solely from timestamps. - -During follow-up, enabling the old `MCODE_SERVER_WORK_TRACE` also enabled it -inside the SQLite writer worker. That worker's pauses were incorrectly counted -as server-loop pauses. `fe98cec3` limits this trace to the main thread. Ten -additional seven-task runs with the corrected trace all passed, with largest -main-server delays of 312, 242, 272, 325, 471, 306, 244, 228, 316, and 430 ms. -Their traces recorded no main-thread event application or finalization work for -these worker-owned Codex turns. Those runs did not reproduce either default-run -pause above 500 ms; the intermittent startup pause is tracked in -[issue #1767](https://github.com/Mzeey-Empire/mcode/issues/1767). - -On the owned Electron app, a real Codex reply remained visible after reload; -the model picker opened in 58 ms, task switch in 1004 ms, terminal prompt in -2338 ms, and a PowerShell command printed in 269 ms. Stop cleared its button -in 119 ms. After reload the prompt remained, the Stop button was absent, and -SQLite showed a `Cancelled` turn with a `cancelled` ingest checkpoint. These -are individual desktop observations, not latency distributions. Claude and -Cursor adapter behavior was checked by focused tests, not live provider runs. diff --git a/docs/performance/message-list-baseline.md b/docs/performance/message-list-baseline.md deleted file mode 100644 index 4335da3b6..000000000 --- a/docs/performance/message-list-baseline.md +++ /dev/null @@ -1,180 +0,0 @@ -# Message List Performance Baseline - -## Scope - -This report records the final issue #1546 baseline from these raw artifacts only: - -- `.dev/verification/performance/issue-1546-profiling-final.json` -- `.dev/verification/performance/issue-1546-production-final.json` - -It records no candidate win and no dependency change. The pinned versions remain `@tanstack/react-virtual` 3.13.23 and `@tanstack/virtual-core` 3.13.23. - -## Run contract - -| Field | Value | -| --- | --- | -| Source revision | `250a153964347c7792b9a48ff39802c760540a48` | -| Source dirty | `true` in both artifacts and both runtimes | -| Viewport | 1440 x 1000 | -| Warmups | 1 | -| Raw samples | 7 per workload and runtime | -| Workload order | `message100`, `message1000`, `threadSwitch`, `streaming`, `messageListBehavior`, `denseNarrative`, `markdownShiki`, `panelTransitions` | - -Overall, profiling accepted 107 samples and rejected 5. Production accepted 110 samples and rejected 2. - -```powershell -$env:PLAYWRIGHT_BROWSERS_PATH = Join-Path (Resolve-Path '.dev') 'playwright-browsers'; -bun run perf:frontend -- --mode profiling --sample-count 7 --output .dev/verification/performance/issue-1546-profiling-final.json - -$env:PLAYWRIGHT_BROWSERS_PATH = Join-Path (Resolve-Path '.dev') 'playwright-browsers'; -bun run perf:frontend -- --mode production --sample-count 7 --output .dev/verification/performance/issue-1546-production-final.json -``` - -## Environment and runtime signals - -The artifacts record Windows `win32` release `10.0.26200`, an x64 12th Gen Intel(R) Core(TM) i7-12700 with 20 CPUs, and 68,373,008,384 bytes of host memory. They record Node 22.16.0, Electron 35.7.5, and Playwright 1.62.1. - -| Runtime | Browser identity | Reported device memory | Fixed acceleration field | -| --- | --- | ---: | --- | -| standalone-web | HeadlessChrome 151.0.7922.34 | 32 GiB | `null` | -| electron | Chrome 134.0.6998.205 in Electron 35.7.5 | 8 GiB | `false` | - -Profiling records no Chromium trace object for either runtime, so ResizeObserver, GC, layout, and paint trace values are unavailable there. The production Electron process signal is available and records `accelerationMode: "disabled"`; the standalone-web process signal is `null`. OS GPU counters are unavailable in both runtimes with the signal: `OS GPU counters are not available in the paired un-packaged runner`. - -## Behavior baseline - -All required MessageList checks passed except click-driven sticky jump to an initially unmounted row, which missed the 3.6s observation window in 5/14 profiling samples and 2/14 production samples; focused runs could pass, so this is load-sensitive current-baseline behavior, not a claimed migration result. - -The cache contract exercised by this baseline is: - -- A cache hit restores the saved reading anchor. -- Simulated cache eviction forgets scroll memory, holds the outgoing transcript during the cold load, then positions the restored transcript at the tail. - -Each cell gives accepted/rejected raw samples. All seven samples ran for every workload and runtime. - -| Workload | Profiling standalone-web | Profiling Electron | Production standalone-web | Production Electron | -| --- | ---: | ---: | ---: | ---: | -| message100 | 7/0 | 7/0 | 7/0 | 7/0 | -| message1000 | 7/0 | 7/0 | 7/0 | 7/0 | -| threadSwitch | 7/0 | 7/0 | 7/0 | 7/0 | -| streaming | 7/0 | 7/0 | 7/0 | 7/0 | -| messageListBehavior | 3/4 | 6/1 | 6/1 | 6/1 | -| denseNarrative | 7/0 | 7/0 | 7/0 | 7/0 | -| markdownShiki | 7/0 | 7/0 | 7/0 | 7/0 | -| panelTransitions | 7/0 | 7/0 | 7/0 | 7/0 | - -## Workload durations - -All values are milliseconds. The artifact summaries include accepted samples only, so the `messageListBehavior` summaries have three or six samples as shown by their accepted counts. - -### Profiling, standalone-web - -| Workload | Accepted/rejected | Min | Median | Max | -| --- | ---: | ---: | ---: | ---: | -| message100 | 7/0 | 120.3 | 127.6 | 181.2 | -| message1000 | 7/0 | 173.7 | 195.8 | 212.0 | -| threadSwitch | 7/0 | 96.3 | 112.7 | 137.6 | -| streaming | 7/0 | 16721.2 | 16737.8 | 16781.8 | -| messageListBehavior | 3/4 | 2664.8 | 2734.0 | 2739.7 | -| denseNarrative | 7/0 | 24.8 | 29.4 | 31.7 | -| markdownShiki | 7/0 | 3997.9 | 4063.1 | 4651.0 | -| panelTransitions | 7/0 | 56.0 | 58.3 | 65.8 | - -### Profiling, Electron - -| Workload | Accepted/rejected | Min | Median | Max | -| --- | ---: | ---: | ---: | ---: | -| message100 | 7/0 | 196.3 | 211.0 | 242.7 | -| message1000 | 7/0 | 208.9 | 213.5 | 223.1 | -| threadSwitch | 7/0 | 152.9 | 155.5 | 204.7 | -| streaming | 7/0 | 13435.2 | 13487.7 | 13948.9 | -| messageListBehavior | 6/1 | 3108.9 | 3365.7 | 3519.8 | -| denseNarrative | 7/0 | 34.8 | 40.7 | 44.6 | -| markdownShiki | 7/0 | 5687.2 | 5825.8 | 6204.3 | -| panelTransitions | 7/0 | 102.9 | 111.0 | 120.6 | - -### Production, standalone-web - -| Workload | Accepted/rejected | Min | Median | Max | -| --- | ---: | ---: | ---: | ---: | -| message100 | 7/0 | 75.6 | 109.5 | 124.9 | -| message1000 | 7/0 | 148.5 | 152.5 | 166.7 | -| threadSwitch | 7/0 | 106.8 | 111.7 | 148.5 | -| streaming | 7/0 | 16749.4 | 16774.7 | 16816.2 | -| messageListBehavior | 6/1 | 3181.4 | 3279.8 | 3319.7 | -| denseNarrative | 7/0 | 63.3 | 64.2 | 81.9 | -| markdownShiki | 7/0 | 5666.4 | 6134.6 | 7008.2 | -| panelTransitions | 7/0 | 84.5 | 94.1 | 95.9 | - -### Production, Electron - -| Workload | Accepted/rejected | Min | Median | Max | -| --- | ---: | ---: | ---: | ---: | -| message100 | 7/0 | 185.8 | 214.3 | 274.1 | -| message1000 | 7/0 | 159.0 | 185.3 | 217.4 | -| threadSwitch | 7/0 | 139.6 | 154.1 | 176.1 | -| streaming | 7/0 | 13438.8 | 13488.0 | 13901.5 | -| messageListBehavior | 6/1 | 2558.2 | 2574.0 | 2639.4 | -| denseNarrative | 7/0 | 42.3 | 45.5 | 51.4 | -| markdownShiki | 7/0 | 3940.5 | 4419.6 | 4665.5 | -| panelTransitions | 7/0 | 55.2 | 79.8 | 86.4 | - -## MessageList attribution - -This table records the narrow MessageList stages for `messageListBehavior` in profiling mode. Stage counts include every measured stage observation, not only the accepted workload samples. - -| Runtime | Narrative stage count | Narrative median | Narrative p95 | TanStack stage count | TanStack median | TanStack p95 | -| --- | ---: | ---: | ---: | ---: | ---: | ---: | -| standalone-web | 39 | 0.2 | 0.6 | 384 | 0.0 | 0.1 | -| electron | 78 | 0.5 | 2.5 | 782 | 0.0 | 0.1 | - -The React profiler is present for all 14 profiling `messageListBehavior` samples. It records 14 commits per runtime, and its actual and base duration minimum, median, and maximum are all 0.0 ms. React attribution is `null` for the corresponding production samples. - -Production Chromium trace attribution is present for `messageListBehavior`. The table gives min, median, and max across its seven raw samples. ResizeObserver callback trace duration is `null` in both runtimes. - -| Runtime | GC min/median/max | Layout min/median/max | Paint min/median/max | -| --- | ---: | ---: | ---: | -| standalone-web | 423.730 / 533.545 / 574.810 | 74.686 / 96.521 / 98.244 | 99.005 / 116.156 / 121.028 | -| electron | 879.900 / 923.831 / 997.533 | 97.836 / 112.967 / 118.839 | 106.737 / 114.805 / 118.057 | - -## Shiki attribution - -`codeToHtml` is the largest Shiki stage in every runtime and mode. Values below are medians in milliseconds. A dash means the artifact reports `null` for that stage. - -### Profiling Shiki medians - -| Runtime | Phase | Worker startup | Highlighter creation | Grammar load | Code to HTML | Worker delivery | React commit | HTML insertion | Total completion | -| --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | -| standalone-web | cold, n=7 | 0.5 | 1.1 | 3.0 | 494.1 | 7.0 | 1.3 | 0.0 | 574.9 | -| standalone-web | warm, n=63 | - | 0.0 | 0.0 | 372.3 | 8.0 | 0.8 | 0.0 | 2507.7 | -| electron | cold, n=7 | 3.9 | 1.6 | 4.9 | 559.4 | 3.0 | 1.5 | 0.0 | 726.9 | -| electron | warm, n=63 | - | 0.0 | 0.0 | 553.2 | 4.0 | 1.6 | 0.0 | 3534.7 | - -Profiling Shiki style and layout fields are `null`. - -### Production Shiki medians - -| Runtime | Phase | Worker startup | Highlighter creation | Grammar load | Code to HTML | Worker delivery | -| --- | --- | ---: | ---: | ---: | ---: | ---: | -| standalone-web | cold, n=7 | 5.0 | 1.7 | 4.8 | 616.1 | 7.0 | -| standalone-web | warm, n=63 | - | 0.0 | 0.0 | 562.2 | 10.0 | -| electron | cold, n=7 | 4.7 | 1.0 | 3.0 | 503.9 | 4.0 | -| electron | warm, n=63 | - | 0.0 | 0.0 | 386.6 | 4.0 | - -Production `reactCommit`, `htmlInsertion`, and `totalCompletion` fields are `null`. Production workload style and layout fields are available: - -| Runtime | Style min/median/max | Layout min/median/max | -| --- | ---: | ---: | -| standalone-web | 97.951 / 109.194 / 127.527 | 46.340 / 50.839 / 57.788 | -| electron | 101.389 / 113.301 / 127.111 | 45.503 / 48.114 / 54.967 | - -The response-byte fields are present. Cold samples have n=7 and warm samples have n=63 in every runtime and mode. Their median bytes are 32,485 and 32,485 for profiling standalone-web, 32,345 and 32,293 for profiling Electron, 32,310 and 32,345 for production standalone-web, and 32,485 and 32,485 for production Electron. - -## Limits - -- This is a baseline from a dirty source tree, not a before-and-after comparison. -- Rejected behavior samples remain in the raw artifacts but not in duration summaries. -- The standalone-web acceleration field is `null`; it does not establish an acceleration setting. -- The profiling artifacts contain no Chromium trace attribution. -- GPU counters are unavailable in the paired un-packaged runner. -- The artifacts do not provide Electron process measurements except the production Electron signal. diff --git a/docs/performance/tanstack-virtual-current-trial.md b/docs/performance/tanstack-virtual-current-trial.md deleted file mode 100644 index 0288d5c5f..000000000 --- a/docs/performance/tanstack-virtual-current-trial.md +++ /dev/null @@ -1,181 +0,0 @@ -# TanStack Virtual Current-Stable Trial - -## Decision - -Keep `@tanstack/react-virtual` 3.13.23 and `@tanstack/virtual-core` 3.13.23. -The 3.14.10 trial did not pass the MessageList behavior gate. The duration -samples therefore do not support an upgrade. - -This report records a reversible dependency trial for issue #1547. The trial -changed only `bun.lock` while it ran. The original lock file and installed -packages were restored before this report was added. - -## Candidate - -The release state was frozen at approximately `2026-08-25T14:59Z`. - -| Package | Candidate version | Published | Tarball integrity | Release commit | -| --- | --- | --- | --- | --- | -| `@tanstack/react-virtual` | 3.14.10 | 2026-08-18 15:06:28.045 UTC | `sha512-SRyoUbdFMRHuYXMijV5H4ZarQWpXkj3iANq8OFre+pybeVap8ZJjZ3Nz9bVjx4d8PfobVUQUdKyyyHYk3E+djw==` | `e9874f033c74afd3251eeb9f3e60b2530cc7ae88` | -| `@tanstack/virtual-core` | 3.17.8 | 2026-08-18 15:06:21.826 UTC | `sha512-BfEvehNpOT75r5Ksc5xW6NZuXujTfb7nlSEyVu4XHG3gdxNg1KqXruWbDewXOUaUYIo4oRbSfkjIajz4MAT8tA==` | `e9874f033c74afd3251eeb9f3e60b2530cc7ae88` | - -The candidate lock entries used the official tarballs: - -- `https://registry.npmjs.org/@tanstack/react-virtual/-/react-virtual-3.14.10.tgz` -- `https://registry.npmjs.org/@tanstack/virtual-core/-/virtual-core-3.17.8.tgz` - -Official sources: [React change log](https://github.com/TanStack/virtual/blob/main/packages/react-virtual/CHANGELOG.md), -[core change log](https://github.com/TanStack/virtual/blob/main/packages/virtual-core/CHANGELOG.md), -and [release commit](https://github.com/TanStack/virtual/commit/e9874f033c74afd3251eeb9f3e60b2530cc7ae88). - -The existing application range, `^3.13.23`, accepted 3.14.10. The candidate -lock resolved React Virtual 3.14.10 and its nested Virtual Core dependency -3.17.8. A frozen install and both candidate runner builds succeeded with no -product-source change or public API incompatibility. The candidate tree differed -from the clean baseline only in `bun.lock`. - -## Run contract - -All four runs used source revision `b04b632061f335e010c24de0b2edaf17e1360688`. -The runner used a 1440 by 1000 viewport, one warmup, and seven samples for each -workload and runtime. The workload order was `message100`, `message1000`, -`threadSwitch`, `streaming`, `messageListBehavior`, `denseNarrative`, -`markdownShiki`, and `panelTransitions`. - -All four artifacts record Playwright 1.62.1. Standalone web reported no -acceleration field. Electron reported disabled acceleration in every artifact. -The baseline artifacts were clean at `b04b6320`. The candidate artifacts had -`sourceDirty: true` because the temporary dependency resolution changed only -`bun.lock`. - -```powershell -bun run perf:frontend -- --mode profiling --sample-count 7 --output .dev/verification/performance/issue-1547-baseline-profiling.json -bun run perf:frontend -- --mode production --sample-count 7 --output .dev/verification/performance/issue-1547-baseline-production.json -bun run perf:frontend -- --mode profiling --sample-count 7 --output .dev/verification/performance/issue-1547-candidate-profiling.json -bun run perf:frontend -- --mode production --sample-count 7 --output .dev/verification/performance/issue-1547-candidate-production.json -``` - -Each command wrote its raw artifact but exited 1 when the runner rejected one -or more behavior samples. The exit status preserves the runner contract. It -does not discard accepted samples from the artifact. - -## Behavior results - -The fresh issue #1547 baseline records the known load-sensitive sticky jump. It -missed the 3.6-second observation window in six of 14 profiling samples and -two of 14 production samples. The accepted issue #1546 baseline recorded five -of 14 profiling misses and two of 14 production misses. This fresh rerun shows -load-sensitive variation. It does not replace or restate the accepted #1546 -artifacts. The candidate run did not prove parity: - -- Candidate profiling rejected all 56 Electron samples after a WebSocket - closure and connection refusal. Treat its Electron React data as invalid. -- Candidate production observed two standalone-web sticky jumps and five - Electron sticky jumps. The baseline production observed zero standalone-web - and two Electron sticky jumps. The candidate run therefore observed five - more sticky failures in total, including three more in Electron. -- Candidate production also observed two Electron streaming failures. Each - failed sample did not commit all 200 visible updates, did not keep the tail in - view, and moved a user who had left the tail. Baseline production accepted all - 14 streaming samples. - -The message-list behavior samples that reached their assertions still recorded -dynamic Markdown heights, older and newer anchors, cache-hit restoration, -cache-miss tail restoration, live-to-persisted identity, focus, and interactive -controls, resident thread switch, and text selection. The sticky jump targets -were initially unmounted. The failed sticky samples did not reach the required -transcript row within the observation window. - -| Mode | Runtime | Baseline accepted/rejected | Candidate accepted/rejected | Result | -| --- | --- | ---: | ---: | --- | -| Profiling | standalone web | 54/2 | 55/1 | Candidate sticky behavior remained load-sensitive. | -| Profiling | Electron | 52/4 | 0/56 | Candidate Electron run was invalid after socket failures. | -| Production | standalone web | 56/0 | 54/2 | Candidate run observed two sticky-jump rejections. | -| Production | Electron | 54/2 | 49/7 | Candidate run observed three more sticky and two streaming rejections. | -| Total | both runtimes | 216/8 | 158/66 | Candidate run did not prove parity. | - -The reported candidate profiling Electron socket errors are a runner -observation, not evidence that TanStack Virtual caused a product failure. They -still prevent that run from proving parity or a performance result. The -production behavior observations alone fail the candidate under the predeclared -behavior-parity gate. - -## Accepted-sample duration comparison - -These values are medians in milliseconds from accepted standalone-web samples. -They are descriptive only. Candidate behavior failed, and sample pools differ, -so no row supports a performance claim. - -### Profiling - -| Workload | Baseline | Candidate | -| --- | ---: | ---: | -| `message100` | 81.5 | 128.1 | -| `message1000` | 81.5 | 149.3 | -| `threadSwitch` | 59.3 | 123.2 | -| `streaming` | 16738.1 | 16745.3 | -| `messageListBehavior` | 3159.3 | 3162.3 | -| `denseNarrative` | 35.2 | 32.0 | -| `markdownShiki` | 8237.7 | 6095.4 | -| `panelTransitions` | 76.0 | 72.0 | - -### Production - -| Workload | Baseline | Candidate | -| --- | ---: | ---: | -| `message100` | 123.9 | 124.4 | -| `message1000` | 137.1 | 150.7 | -| `threadSwitch` | 114.3 | 127.1 | -| `streaming` | 16729.4 | 16720.6 | -| `messageListBehavior` | 2927.6 | 2986.6 | -| `denseNarrative` | 37.6 | 42.3 | -| `markdownShiki` | 5744.1 | 4425.8 | -| `panelTransitions` | 55.7 | 67.8 | - -## Attribution boundaries - -React data came from profiling builds. In accepted standalone-web -`messageListBehavior` samples, React actual duration had a median of 0.0 ms in -both baseline and candidate data. The narrow MessageList stages also had the -same median: narrative-item projection was 0.4 ms and `getVirtualItems()` was -0.0 ms. Their p95 values were 0.7 ms and 0.1 ms. The sample counts differed, -so these values do not establish an improvement. - -Chromium trace data came from production builds. Electron process CPU and -memory data came from production Electron samples. OS GPU counters were -unavailable in all four artifacts. The candidate did not pass behavior, and its -profiling Electron samples were invalid, so the Chromium and process signals -are recorded in the raw artifacts but are not compared as upgrade evidence. - -## Upstream review - -The candidate includes upstream work for scroll compensation above or across -the fold, dynamic `scrollToIndex`, DOM-node identity cleanup, end anchoring, -cached measurements, skipping redundant unchanged-offset scroll events, -allocation reduction, ResizeObserver and count-shrink safety, pending -observer-reset cancellation, and observer teardown state reset. - -React Virtual 3.14.0 also adds opt-in `directDomUpdates`. Mcode does not set -that option. The candidate build succeeded with the existing Mcode options and -showed no public API break. This trial did not edit product code or opt in to -new APIs. - -## Restoration and limits - -The temporary lockfile was restored byte-for-byte. Its saved and restored -SHA-256 value was `54D9671BAB3EC886AF5BBCA8FFB205C9978C44CEC6F47993AECF4CF4CB708825`. -A frozen install then resolved React Virtual and Virtual Core back to 3.13.23. -No dependency manifest or lockfile change remains. - -The candidate fails the behavior gate, so this trial cannot decide whether a -future version can improve Mcode. The profiling Electron socket failure also -limits attribution. A future trial needs a candidate that first passes the -full behavior matrix in both runtimes. Only then can matched accepted samples -support an upgrade decision. - -## Raw artifacts - -- `.dev/verification/performance/issue-1547-baseline-profiling.json` -- `.dev/verification/performance/issue-1547-baseline-production.json` -- `.dev/verification/performance/issue-1547-candidate-profiling.json` -- `.dev/verification/performance/issue-1547-candidate-production.json` diff --git a/docs/performance/vlist-react-adapter-prototype.md b/docs/performance/vlist-react-adapter-prototype.md deleted file mode 100644 index fa5238f53..000000000 --- a/docs/performance/vlist-react-adapter-prototype.md +++ /dev/null @@ -1,73 +0,0 @@ -# vlist React adapter prototype - -## Scope - -This development-only probe tests an architectural bridge between React and vlist. One React root portals logical rows into hosts that vlist owns. - -The bridge uses a vlist plugin with priority `100`. It performs these operations: - -1. `onCalculate` removes outgoing React rows before vlist changes the DOM. -2. `onCalculate` parks retained portal hosts when their logical rows move to new indices. -3. vlist updates and recycles its item shells. -4. `onCommit` moves parked hosts into their new shells and renders the visible React rows. -5. The adapter unmounts React before it destroys vlist. - -The probe uses `overscan: 0` and fixed 84-pixel rows. It does not change the production TanStack message list or compare performance. - -## Frozen baseline - -| Package | Version | Published | Source revision | -| --- | --- | --- | --- | -| `vlist` | 2.6.1 | 2026-07-15 | [`7c39284b964a4d4e7b67821d17a622cf599eaed8`](https://github.com/floor/vlist/tree/7c39284b964a4d4e7b67821d17a622cf599eaed8) | -| `vlist-react` | 2.6.0 | 2026-07-08 | [`4f8a94334f32386b4921dfd8384368506c24f8d3`](https://github.com/floor/vlist-react/tree/4f8a94334f32386b4921dfd8384368506c24f8d3) | - -The vlist-react wrapper creates a vanilla vlist instance in a React effect and updates its items. It does not define React-owned virtual rows. [Source](https://github.com/floor/vlist-react/blob/4f8a94334f32386b4921dfd8384368506c24f8d3/src/index.ts#L42-L104) - -## Browser observation - -The Electron renderer ran three profiling samples. The run wrote disposable raw output to `.dev/verification/performance/issue-1548-vlist-lifecycle-bridge-prepend-electron.json`. Repository policy keeps live verification artifacts under the ignored `.dev/verification/` directory. The raw file is not part of this document. See [ADR 0017](../adr/0017-separate-maintained-tests-from-disposable-verification.md). - -Run this command to create a new local artifact: - -```powershell -bun run perf:frontend -- --mode profiling --sample-count 3 --workload vlistLifecycle --runtime electron --output .dev/verification/performance/issue-1548-vlist-lifecycle-bridge-prepend-electron.json -``` - -All three samples returned this decision: - -```json -{"status":"accepted","candidateEligible":true,"reason":null} -``` - -The runner derives the decision from raw browser facts. It does not trust a summary from the probe. - -The lifecycle checks cover heterogeneous rows, local state, effects, refs, body portals, focus, controls, and host reuse. The checks also cover native scroll recycling and final disposal. - -The prepend check inserts two older rows above a visible anchor. In each sample, the anchor stayed at pixel `774`. Its edited draft stayed `edited-before-prepend`. Its effect and ref counts stayed at one. Its React portal host token stayed `portal-host-0`. - -The standalone Chromium process failed to start twice within its 180-second limit. Later Electron profiles also failed during app startup. Those failures occurred before probe code ran. - -The runner ignores timing fields for this workload. This probe makes no performance claim. - -## Gate matrix - -| Gate | Status | Evidence | -| --- | --- | --- | -| React portal lifecycle under vlist host reuse | Passed in Electron | Three samples show React cleanup before vlist detaches the old host. Native scroll also recycles pool shells. | -| Fixed-height prepend anchor and logical identity | Passed in Electron | Three samples keep the anchor position, draft, mount count, ref count, and portal host. | -| Dynamic-height measurement | Not run | The fixed-height prepend bridge does not prove the autosize path. [Source](https://github.com/floor/vlist/blob/7c39284b964a4d4e7b67821d17a622cf599eaed8/src/plugins/autosize/plugin.ts) | -| Tail follow and user-away posture | Not run | No browser artifact covers append behavior after the bridge change. | -| Sticky user-message jump | Not run | vlist sticky support covers group headers, not the MessageList contract. [Source](https://github.com/floor/vlist/blob/7c39284b964a4d4e7b67821d17a622cf599eaed8/src/plugins/groups/plugin.ts) | -| Selection and scroll-to-message | Not run | The probe covers focus and controls only. | -| Thread-switch anchor restore | Not run | No browser artifact covers per-thread anchor storage after the bridge change. | -| Arbitrary retained-row range | Does not fit the core model | Core rendering calculates one contiguous visible range plus overscan. It has no API for an unrelated retained range. [Source](https://github.com/floor/vlist/blob/7c39284b964a4d4e7b67821d17a622cf599eaed8/src/core/pipeline.ts#L225-L265) | -| Equivalent retained-state mechanism | Not run | The bridge can keep moved visible hosts, but it does not prove an off-screen retained-state design. | -| Production integration and comparison | Not run | The production TanStack route is unchanged. | - -## Architectural result - -The original direct-portal topology failed because vlist detached the portal target before React cleanup. The bridge changes that ownership handoff. React now removes outgoing rows before vlist updates its shells. - -Prepend needs more work than cleanup order. vlist keys rendered shells by index. The bridge parks retained portal hosts, shifts the pending fixed-height range, and restores each host under its new index. This keeps the React component mounted. - -The tested bridge removes the original lifecycle rejection. It also proves fixed-height prepend behavior in Electron. The candidate remains incomplete until the other behavior gates pass. From b121cfb1160143f279e5244d4067b1bfea22394f Mon Sep 17 00:00:00 2001 From: Agent Runtime Fixture Date: Sun, 27 Sep 2026 12:50:08 +0100 Subject: [PATCH 134/136] refactor(server): drop the CanonicalAgentEventSink compatibility facade --- .../skills/verify-mcode/scripts/runtime.mjs | 2 +- .../features/agents/__tests__/index.test.ts | 1 - ...ub.ts => canonical-agent-boundary-stub.ts} | 10 ++--- ...st.ts => canonical-agent-boundary.test.ts} | 38 ++++++++--------- .../canonical/canonical-agent-event-sink.ts | 9 ---- .../canonical/canonical-agent-event-store.ts | 2 +- .../canonical-agent-read-repository.ts | 2 +- ...nonical-codex-collaboration-coordinator.ts | 2 +- .../canonical-parent-turn-lifecycle.ts | 2 +- .../agents/composition/register-agents.ts | 4 -- .../legacy-conversation-migration.test.ts | 6 +-- apps/server/src/features/agents/index.ts | 3 -- .../agent-service-existing-worktree.test.ts | 4 +- .../__tests__/agent-service-gate.test.ts | 4 +- .../agent-service-goal-command.test.ts | 4 +- .../agent-service-narrative-persist.test.ts | 42 +++++++++---------- .../agent-service-plan-marker.test.ts | 4 +- .../agent-service-session-invalidated.test.ts | 4 +- .../agent-service-transient-retry.test.ts | 4 +- .../agent-service-turn-cleanup.test.ts | 10 ++--- .../agent-service-turn-started.test.ts | 6 +-- .../__tests__/turn-recovery-service.test.ts | 8 ++-- ...-assistant-text-checkpoint-service.test.ts | 4 +- .../__tests__/turn-diff-finalization.test.ts | 4 +- .../turns/__tests__/turn-finalizer.test.ts | 4 +- scripts/perf/subagent-detail-fixture.ts | 10 ++--- 26 files changed, 88 insertions(+), 105 deletions(-) rename apps/server/src/features/agents/canonical/__tests__/{canonical-agent-event-sink-stub.ts => canonical-agent-boundary-stub.ts} (75%) rename apps/server/src/features/agents/canonical/__tests__/{canonical-agent-event-sink.test.ts => canonical-agent-boundary.test.ts} (99%) delete mode 100644 apps/server/src/features/agents/canonical/canonical-agent-event-sink.ts diff --git a/.agents/skills/verify-mcode/scripts/runtime.mjs b/.agents/skills/verify-mcode/scripts/runtime.mjs index 00d5351f0..eb4979761 100644 --- a/.agents/skills/verify-mcode/scripts/runtime.mjs +++ b/.agents/skills/verify-mcode/scripts/runtime.mjs @@ -67,7 +67,7 @@ const FOCUSED_TEST_FILES = [ "src/features/agents/turns/__tests__/turn-event-sink.test.ts", "src/features/agents/turns/__tests__/turn-finalizer.test.ts", "src/features/agents/turns/__tests__/turn-runtime.test.ts", - "src/features/agents/canonical/__tests__/canonical-agent-event-sink.test.ts", + "src/features/agents/canonical/__tests__/canonical-agent-boundary.test.ts", "src/features/agents/collaboration/adapters/__tests__/codex-collaboration-event-adapter.test.ts", "src/features/providers/composition/__tests__/provider-event-ingress.test.ts", "src/features/projects/git/__tests__/git-repository-fetch.test.ts", diff --git a/apps/server/src/features/agents/__tests__/index.test.ts b/apps/server/src/features/agents/__tests__/index.test.ts index 57ac2dc9e..3619de974 100644 --- a/apps/server/src/features/agents/__tests__/index.test.ts +++ b/apps/server/src/features/agents/__tests__/index.test.ts @@ -8,7 +8,6 @@ describe("agents feature boundary", () => { "AgentPermissionService", "AgentService", "CanonicalAgentBoundary", - "CanonicalAgentEventSink", "DelegationTargetResolver", "GoalLifecycleService", "ParentAssistantTextCheckpointService", diff --git a/apps/server/src/features/agents/canonical/__tests__/canonical-agent-event-sink-stub.ts b/apps/server/src/features/agents/canonical/__tests__/canonical-agent-boundary-stub.ts similarity index 75% rename from apps/server/src/features/agents/canonical/__tests__/canonical-agent-event-sink-stub.ts rename to apps/server/src/features/agents/canonical/__tests__/canonical-agent-boundary-stub.ts index 853b84038..b2a7526f8 100644 --- a/apps/server/src/features/agents/canonical/__tests__/canonical-agent-event-sink-stub.ts +++ b/apps/server/src/features/agents/canonical/__tests__/canonical-agent-boundary-stub.ts @@ -1,13 +1,13 @@ -import type { CanonicalAgentEventSink } from "../../index.js"; +import type { CanonicalAgentBoundary } from "../../index.js"; import type { Database } from "bun:sqlite"; /** Creates an AgentService test seam that runs compatibility writes without canonical persistence. */ -export function createCanonicalAgentEventSinkStub( +export function createCanonicalAgentBoundaryStub( db: Pick, -): CanonicalAgentEventSink { +): CanonicalAgentBoundary { return { startParentTurn: ( - input: Parameters[0], + input: Parameters[0], ) => { db.transaction(input.projectUserMessage)(); return { @@ -26,5 +26,5 @@ export function createCanonicalAgentEventSinkStub( finishCanonicalChildTurn: () => null, recordProviderDiagnostic: () => undefined, recordCodexChildRoutingDiagnostic: () => false, - } as unknown as CanonicalAgentEventSink; + } as unknown as CanonicalAgentBoundary; } diff --git a/apps/server/src/features/agents/canonical/__tests__/canonical-agent-event-sink.test.ts b/apps/server/src/features/agents/canonical/__tests__/canonical-agent-boundary.test.ts similarity index 99% rename from apps/server/src/features/agents/canonical/__tests__/canonical-agent-event-sink.test.ts rename to apps/server/src/features/agents/canonical/__tests__/canonical-agent-boundary.test.ts index 2478066ba..54620ab03 100644 --- a/apps/server/src/features/agents/canonical/__tests__/canonical-agent-event-sink.test.ts +++ b/apps/server/src/features/agents/canonical/__tests__/canonical-agent-boundary.test.ts @@ -29,9 +29,9 @@ import { ACTIVE_TURN_WRITE_BATCH_LIMITS } from "../../../../runtime/persistence/ import { PARENT_ASSISTANT_TEXT_RETAINED_LIMITS } from "../../turns/parent-assistant-text-checkpoint-service.js"; import { CANONICAL_AGENT_CONTROL_EVENT_RESERVE, - CanonicalAgentEventSink, + CanonicalAgentBoundary, type CanonicalAgentEventDraft, -} from "../canonical-agent-event-sink.js"; +} from "../canonical-agent-boundary.js"; import { CodexCollaborationEventAdapter } from "../../collaboration/adapters/codex-collaboration-event-adapter.js"; import type { CodexCollaborationDurability } from "../../collaboration/codex-collaboration-durability.js"; @@ -117,7 +117,7 @@ function initialDrafts(): CanonicalAgentEventDraft[] { function seedUnfinishedCheckpointCount( db: Database, - sink: CanonicalAgentEventSink, + sink: CanonicalAgentBoundary, count: number, ): void { const messageRepo = new MessageRepo(db); @@ -171,7 +171,7 @@ function terminalDraft( } function startCanonicalParent( - sink: CanonicalAgentEventSink, + sink: CanonicalAgentBoundary, db: Database, approvalReview: { mode: "manual" | "automatic"; reason: string } = { mode: "manual", reason: "manual-requested" }, ): void { @@ -312,7 +312,7 @@ function executionIdForTurn(db: Database, turnId: string): string { return row.execution_id; } -function parentTerminalInput(db: Database, toolCount = 0): Parameters[0] { +function parentTerminalInput(db: Database, toolCount = 0): Parameters[0] { const messageRepo = new MessageRepo(db); const message = messageRepo.create(THREAD_ID, "assistant", "answer", 2, undefined, undefined, undefined, undefined, true); return { @@ -333,21 +333,21 @@ function parentTerminalInput(db: Database, toolCount = 0): Parameters { +describe("CanonicalAgentBoundary", () => { let db: Database; let published: ReturnType void>>; - let sink: CanonicalAgentEventSink; + let sink: CanonicalAgentBoundary; beforeEach(() => { db = openMemoryDatabase(); seedThread(db); published = vi.fn(); - sink = new CanonicalAgentEventSink(db, published); + sink = new CanonicalAgentBoundary(db, published); }); it("retains the resolved approval-review decision when a turn is read after reopening", () => { startCanonicalParent(sink, db, { mode: "automatic", reason: "automatic-review-available" }); - const reloaded = new CanonicalAgentEventSink(db, published); + const reloaded = new CanonicalAgentBoundary(db, published); expect(reloaded.loadTurnByExecution(EXECUTION_ID)).toMatchObject({ approvalReviewMode: "automatic", @@ -437,7 +437,7 @@ describe("CanonicalAgentEventSink", () => { it("reports deferred canonical delivery without undoing the durable commit", () => { const failingPublisher = vi.fn(() => { throw new Error("canonical delivery failed"); }); - const deferredSink = new CanonicalAgentEventSink(db, failingPublisher); + const deferredSink = new CanonicalAgentBoundary(db, failingPublisher); const result = deferredSink.commit({ threadId: THREAD_ID, @@ -557,7 +557,7 @@ describe("CanonicalAgentEventSink", () => { preparedSql.push(sql); return originalPrepare(sql); }) as Database["prepare"]; - const instrumentedSink = new CanonicalAgentEventSink(db, vi.fn()); + const instrumentedSink = new CanonicalAgentBoundary(db, vi.fn()); instrumentedSink.commit({ threadId: THREAD_ID, @@ -808,7 +808,7 @@ describe("CanonicalAgentEventSink", () => { userMessageId = message.id; return message; }, - } satisfies Parameters[0]; + } satisfies Parameters[0]; sink.startParentTurn(startInput); sink.startParentTurn(startInput); const finishInput = { @@ -855,7 +855,7 @@ describe("CanonicalAgentEventSink", () => { }], }; }, - } satisfies Parameters[0]; + } satisfies Parameters[0]; sink.finishParentTurn(finishInput); sink.finishParentTurn(finishInput); @@ -887,7 +887,7 @@ describe("CanonicalAgentEventSink", () => { let interruptPublication = false; let interrupted = false; const batchRows: number[] = []; - sink = new CanonicalAgentEventSink(db, (events) => { + sink = new CanonicalAgentBoundary(db, (events) => { batchRows.push(3 + events.reduce( (rows, event) => rows + (event.payload.type === "item.recorded" ? 2 : 1), 0, @@ -1067,7 +1067,7 @@ describe("CanonicalAgentEventSink", () => { seedThread(writerDb); let finishing = false; let mutated = false; - const writer = new CanonicalAgentEventSink(writerDb, () => { + const writer = new CanonicalAgentBoundary(writerDb, () => { if (!finishing || mutated) return; mutated = true; otherDb.prepare("UPDATE canonical_agent_threads SET roster_revision = 17 WHERE id = ?").run(THREAD_ID); @@ -1278,7 +1278,7 @@ describe("CanonicalAgentEventSink", () => { }, }], }); - const reloaded = new CanonicalAgentEventSink(db, vi.fn()); + const reloaded = new CanonicalAgentBoundary(db, vi.fn()); expect(committed.outcome).toBe("committed"); expect(duplicate.outcome).toBe("duplicate"); @@ -1856,7 +1856,7 @@ describe("CanonicalAgentEventSink", () => { }, }), "toolCall:spawn-generic-roster-source"); - const restoredSink = new CanonicalAgentEventSink(db, vi.fn()); + const restoredSink = new CanonicalAgentBoundary(db, vi.fn()); const row = restoredSink.loadSubagentRoster({ owningParentThreadId: THREAD_ID, limit: CANONICAL_SUBAGENT_ROSTER_MAX_CHILDREN, @@ -2539,7 +2539,7 @@ describe("CanonicalAgentEventSink", () => { }], payload: { projection: "codexCollaboration" }, }); - const restoredSink = new CanonicalAgentEventSink(db, vi.fn()); + const restoredSink = new CanonicalAgentBoundary(db, vi.fn()); expect(restoredSink.loadCollaborationActionBySourceProviderIdentity( delegation.childThread.id, childTurn.id, @@ -2921,7 +2921,7 @@ describe("CanonicalAgentEventSink", () => { outcome: "errored", })).toMatchObject({ status: "Completed" }); - const restored = new CanonicalAgentEventSink(db, vi.fn()); + const restored = new CanonicalAgentBoundary(db, vi.fn()); const recovered = restored.loadCodexChildDelegation(THREAD_ID, input.parentItemId); expect(recovered).toMatchObject({ childThread: { diff --git a/apps/server/src/features/agents/canonical/canonical-agent-event-sink.ts b/apps/server/src/features/agents/canonical/canonical-agent-event-sink.ts deleted file mode 100644 index d8ccbff9a..000000000 --- a/apps/server/src/features/agents/canonical/canonical-agent-event-sink.ts +++ /dev/null @@ -1,9 +0,0 @@ -import { injectable } from "tsyringe"; - -import { CanonicalAgentBoundary } from "./canonical-agent-boundary.js"; - -export * from "./canonical-agent-boundary.js"; - -/** Temporary compatibility façade for callers that have not yet moved to the named boundary. */ -@injectable() -export class CanonicalAgentEventSink extends CanonicalAgentBoundary {} diff --git a/apps/server/src/features/agents/canonical/canonical-agent-event-store.ts b/apps/server/src/features/agents/canonical/canonical-agent-event-store.ts index cded4bd95..6577b8595 100644 --- a/apps/server/src/features/agents/canonical/canonical-agent-event-store.ts +++ b/apps/server/src/features/agents/canonical/canonical-agent-event-store.ts @@ -13,7 +13,7 @@ import { import type { CanonicalAgentCommitResult, CanonicalAgentEventDraft, -} from "./canonical-agent-event-sink.js"; +} from "./canonical-agent-boundary.js"; import { decideCanonicalExecutionLifecycle } from "./canonical-execution-lifecycle.js"; /** Provider-neutral durable progress for one canonical execution. */ diff --git a/apps/server/src/features/agents/canonical/canonical-agent-read-repository.ts b/apps/server/src/features/agents/canonical/canonical-agent-read-repository.ts index da99e12c9..b9cb74689 100644 --- a/apps/server/src/features/agents/canonical/canonical-agent-read-repository.ts +++ b/apps/server/src/features/agents/canonical/canonical-agent-read-repository.ts @@ -12,7 +12,7 @@ import { type CanonicalAgentRevision, type Message, } from "@mcode/contracts"; -import type { CanonicalAgentCheckpoint } from "./canonical-agent-event-sink.js"; +import type { CanonicalAgentCheckpoint } from "./canonical-agent-boundary.js"; import { CanonicalConversationProjectionReader, type CanonicalConversationProjection, diff --git a/apps/server/src/features/agents/canonical/canonical-codex-collaboration-coordinator.ts b/apps/server/src/features/agents/canonical/canonical-codex-collaboration-coordinator.ts index 92787418d..02220e8a2 100644 --- a/apps/server/src/features/agents/canonical/canonical-codex-collaboration-coordinator.ts +++ b/apps/server/src/features/agents/canonical/canonical-codex-collaboration-coordinator.ts @@ -38,7 +38,7 @@ import type { CanonicalAgentCommitInput, CanonicalAgentCommitResult, CanonicalAgentEventDraft, -} from "./canonical-agent-event-sink.js"; +} from "./canonical-agent-boundary.js"; /** Generic persistence operations required by the Codex child-thread protocol. */ export interface CanonicalCodexCollaborationOperations { diff --git a/apps/server/src/features/agents/canonical/canonical-parent-turn-lifecycle.ts b/apps/server/src/features/agents/canonical/canonical-parent-turn-lifecycle.ts index b9d455de9..6b909b5b1 100644 --- a/apps/server/src/features/agents/canonical/canonical-parent-turn-lifecycle.ts +++ b/apps/server/src/features/agents/canonical/canonical-parent-turn-lifecycle.ts @@ -27,7 +27,7 @@ import type { CanonicalAgentCommitResult, CanonicalAgentEventDraft, CanonicalProviderContinuationInput, -} from "./canonical-agent-event-sink.js"; +} from "./canonical-agent-boundary.js"; /** Canonical alias for the parent-turn start durability input. */ export type CanonicalParentTurnStartInput = ParentTurnStartInput; diff --git a/apps/server/src/features/agents/composition/register-agents.ts b/apps/server/src/features/agents/composition/register-agents.ts index a8b7b8a69..d5ce7b35b 100644 --- a/apps/server/src/features/agents/composition/register-agents.ts +++ b/apps/server/src/features/agents/composition/register-agents.ts @@ -8,7 +8,6 @@ import { AgentPermissionService, AgentService, CanonicalAgentBoundary, - CanonicalAgentEventSink, ParentAssistantTextCheckpointService, publishCanonicalAgentEvents, TurnRecoveryService, @@ -155,9 +154,6 @@ export function registerAgentServices(container: DependencyContainer): void { container.register(SUBAGENT_LIFECYCLE_DURABILITY, { useFactory: (c) => c.resolve(CanonicalAgentBoundary), }); - container.register(CanonicalAgentEventSink, { - useFactory: (c) => c.resolve(CanonicalAgentBoundary) as CanonicalAgentEventSink, - }); container.register( ParentAssistantTextCheckpointService, { useClass: ParentAssistantTextCheckpointService }, diff --git a/apps/server/src/features/agents/conversation/migrations/__tests__/legacy-conversation-migration.test.ts b/apps/server/src/features/agents/conversation/migrations/__tests__/legacy-conversation-migration.test.ts index a1d1bc445..35ae8ae14 100644 --- a/apps/server/src/features/agents/conversation/migrations/__tests__/legacy-conversation-migration.test.ts +++ b/apps/server/src/features/agents/conversation/migrations/__tests__/legacy-conversation-migration.test.ts @@ -5,7 +5,7 @@ import * as NodeURL from "node:url"; import type { Database } from "bun:sqlite"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { openMemoryDatabase } from "../../../../../runtime/persistence/sqlite/database.js"; -import { CanonicalAgentEventSink } from "../../../canonical/canonical-agent-event-sink.js"; +import { CanonicalAgentBoundary } from "../../../canonical/canonical-agent-boundary.js"; import { LEGACY_CONVERSATION_MIGRATION_MAX_BYTES, LEGACY_CONVERSATION_MIGRATION_VERSION, @@ -60,7 +60,7 @@ describe("LegacyConversationMigration", () => { provenance: "native", }]), }); - const projection = new CanonicalAgentEventSink(db, vi.fn()) + const projection = new CanonicalAgentBoundary(db, vi.fn()) .loadConversationProjection("thread-v1", 10); expect(projection.messages).toEqual([ expect.objectContaining({ @@ -144,7 +144,7 @@ describe("LegacyConversationMigration", () => { WHERE thread_id = 'thread-child-v1' AND json_extract(payload_json, '$.projection') = 'toolCall' `).get()).toEqual({ count: 1 }); - const restoredSink = new CanonicalAgentEventSink(db, vi.fn()); + const restoredSink = new CanonicalAgentBoundary(db, vi.fn()); expect(restoredSink.loadThread("thread-child-v1")).toMatchObject({ parentThreadId: "thread-child-parent-v1", rootThreadId: "thread-child-parent-v1", diff --git a/apps/server/src/features/agents/index.ts b/apps/server/src/features/agents/index.ts index f0e5b315b..eebd966e8 100644 --- a/apps/server/src/features/agents/index.ts +++ b/apps/server/src/features/agents/index.ts @@ -16,9 +16,6 @@ export { SubagentLifecycleService } from "./collaboration/subagent-lifecycle-ser /** Full canonical agent system boundary used by the server composition roots. */ export { CanonicalAgentBoundary } from "./canonical/canonical-agent-boundary.js"; -/** Temporary compatibility façade for callers migrating to the named boundary. */ -export { CanonicalAgentEventSink } from "./canonical/canonical-agent-event-sink.js"; - /** Publishes canonical agent events for the server composition roots. */ export { publishCanonicalAgentEvents } from "./canonical/canonical-agent-boundary.js"; diff --git a/apps/server/src/features/agents/orchestration/__tests__/agent-service-existing-worktree.test.ts b/apps/server/src/features/agents/orchestration/__tests__/agent-service-existing-worktree.test.ts index abb2fe070..ec2048b00 100644 --- a/apps/server/src/features/agents/orchestration/__tests__/agent-service-existing-worktree.test.ts +++ b/apps/server/src/features/agents/orchestration/__tests__/agent-service-existing-worktree.test.ts @@ -10,7 +10,7 @@ import { WorkspaceRepo } from "../../../projects/persistence/workspace-repo.js"; import { MessageRepo } from "../../conversation/persistence/message-repo.js"; import { createAgentServiceForTest, goalLifecycleForAgentServiceTest } from "./agent-service-test-harness.js"; import { WorkspaceEnvironmentService } from "../../../projects/environment/workspace-environment-service.js"; -import { createCanonicalAgentEventSinkStub } from "../../canonical/__tests__/canonical-agent-event-sink-stub.js"; +import { createCanonicalAgentBoundaryStub } from "../../canonical/__tests__/canonical-agent-boundary-stub.js"; import type { GitService } from "../../../projects/index.js"; import type { ThreadService } from "../../../thread-control/index.js"; import { ParentAssistantTextCheckpointService } from "../../turns/parent-assistant-text-checkpoint-service.js"; @@ -128,7 +128,7 @@ function createAgentServiceHarness(automaticSetup?: {} as never, undefined, undefined, - createCanonicalAgentEventSinkStub(db), + createCanonicalAgentBoundaryStub(db), resolvedAutomaticSetup as never, undefined, undefined, diff --git a/apps/server/src/features/agents/orchestration/__tests__/agent-service-gate.test.ts b/apps/server/src/features/agents/orchestration/__tests__/agent-service-gate.test.ts index 3446fd76d..86817107c 100644 --- a/apps/server/src/features/agents/orchestration/__tests__/agent-service-gate.test.ts +++ b/apps/server/src/features/agents/orchestration/__tests__/agent-service-gate.test.ts @@ -3,7 +3,7 @@ import { describe, it, expect, vi, beforeEach } from "vitest"; import type { ApprovalReviewSupport, Thread, IProviderRegistry } from "@mcode/contracts"; import { supportsInternalThreadControl } from "../../turns/turn-admission-dispatch-coordinator.js"; import { createAgentServiceForTest } from "./agent-service-test-harness.js"; -import { createCanonicalAgentEventSinkStub } from "../../canonical/__tests__/canonical-agent-event-sink-stub.js"; +import { createCanonicalAgentBoundaryStub } from "../../canonical/__tests__/canonical-agent-boundary-stub.js"; import { NarrativeStore } from "../../conversation/narrative/narrative-store.js"; import { ParentAssistantTextCheckpointService } from "../../turns/parent-assistant-text-checkpoint-service.js"; import { ProviderAvailabilityService } from "../../../providers/availability/provider-availability-service.js"; @@ -222,7 +222,7 @@ function buildService({ undefined, threadControlMcp as never, undefined, - createCanonicalAgentEventSinkStub(db), + createCanonicalAgentBoundaryStub(db), ); return { svc, diff --git a/apps/server/src/features/agents/orchestration/__tests__/agent-service-goal-command.test.ts b/apps/server/src/features/agents/orchestration/__tests__/agent-service-goal-command.test.ts index d4836d0cc..9e84ccb08 100644 --- a/apps/server/src/features/agents/orchestration/__tests__/agent-service-goal-command.test.ts +++ b/apps/server/src/features/agents/orchestration/__tests__/agent-service-goal-command.test.ts @@ -19,7 +19,7 @@ import { waitForAgentServiceIngressForTest, wrapProviderEmitterForRuntimeEvents, } from "./agent-service-test-harness.js"; -import { createCanonicalAgentEventSinkStub } from "../../canonical/__tests__/canonical-agent-event-sink-stub.js"; +import { createCanonicalAgentBoundaryStub } from "../../canonical/__tests__/canonical-agent-boundary-stub.js"; import { NarrativeStore } from "../../conversation/narrative/narrative-store.js"; import { GoalLifecycleService } from "../../goals/goal-lifecycle-service.js"; import { ParentAssistantTextCheckpointService } from "../../turns/parent-assistant-text-checkpoint-service.js"; @@ -176,7 +176,7 @@ function buildService(db: Database) { undefined, undefined, undefined, - createCanonicalAgentEventSinkStub(db), + createCanonicalAgentBoundaryStub(db), undefined, undefined, undefined, diff --git a/apps/server/src/features/agents/orchestration/__tests__/agent-service-narrative-persist.test.ts b/apps/server/src/features/agents/orchestration/__tests__/agent-service-narrative-persist.test.ts index 1557663c3..a657e1878 100644 --- a/apps/server/src/features/agents/orchestration/__tests__/agent-service-narrative-persist.test.ts +++ b/apps/server/src/features/agents/orchestration/__tests__/agent-service-narrative-persist.test.ts @@ -26,14 +26,14 @@ import { streamAgentReliabilityTextForTest, waitForAgentServiceIngressForTest, } from "./agent-service-test-harness.js"; -import { createCanonicalAgentEventSinkStub } from "../../canonical/__tests__/canonical-agent-event-sink-stub.js"; +import { createCanonicalAgentBoundaryStub } from "../../canonical/__tests__/canonical-agent-boundary-stub.js"; import { NarrativeStore } from "../../conversation/narrative/narrative-store.js"; import { TaskPersistenceService } from "../../tasks/task-persistence-service.js"; import { ParentAssistantTextCheckpointService, PARENT_ASSISTANT_TEXT_RETAINED_LIMITS, } from "../../turns/parent-assistant-text-checkpoint-service.js"; -import { CanonicalAgentEventSink } from "../../canonical/canonical-agent-event-sink.js"; +import { CanonicalAgentBoundary } from "../../canonical/canonical-agent-boundary.js"; import { openMemoryDatabase } from "../../../../runtime/persistence/sqlite/database.js"; import { broadcast } from "../../../../application/transport/push.js"; import { isTurnScopedEvent } from "../../turns/turn-runtime.js"; @@ -145,7 +145,7 @@ function makeThread(overrides: Partial = {}): Thread { interface Built { service: AgentService; providerEmitter: NodeEvents.EventEmitter; - canonicalSink: CanonicalAgentEventSink; + canonicalSink: CanonicalAgentBoundary; db: Database; messageRepo: MessageRepo; thoughtBulk: ReturnType; @@ -160,7 +160,7 @@ interface Built { function build(options: { db?: Database; - canonicalSink?: CanonicalAgentEventSink; + canonicalSink?: CanonicalAgentBoundary; messageRepo?: MessageRepo; parentAssistantTextCheckpoints?: ParentAssistantTextCheckpointService; onProviderEvent?: (event: AgentEvent) => void; @@ -270,7 +270,7 @@ function build(options: { transaction: vi.fn((fn: Function) => fn), prepare: vi.fn(() => ({ run: vi.fn() })), } as unknown as Database); - const canonicalSink = options.canonicalSink ?? createCanonicalAgentEventSinkStub(db); + const canonicalSink = options.canonicalSink ?? createCanonicalAgentBoundaryStub(db); const parentAssistantTextCheckpoints = options.parentAssistantTextCheckpoints ?? new ParentAssistantTextCheckpointService(db); @@ -372,7 +372,7 @@ describe("AgentService narrative persistence", () => { db.prepare( "INSERT INTO threads (id, workspace_id, title, branch, provider, status, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", ).run(THREAD_ID, "ws-1", "Parent", "main", "claude", "active", now, now); - const canonicalSink = new CanonicalAgentEventSink(db, vi.fn()); + const canonicalSink = new CanonicalAgentBoundary(db, vi.fn()); const published: AgentEvent[] = []; const { providerEmitter, service } = build({ db, @@ -442,7 +442,7 @@ describe("AgentService narrative persistence", () => { db.prepare( "INSERT INTO threads (id, workspace_id, title, branch, provider, status, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", ).run(THREAD_ID, "ws-1", "Parent", "main", "claude", "active", now, now); - const canonicalSink = new CanonicalAgentEventSink(db, vi.fn()); + const canonicalSink = new CanonicalAgentBoundary(db, vi.fn()); const published: AgentEvent[] = []; const { providerEmitter, service } = build({ db, @@ -501,7 +501,7 @@ describe("AgentService narrative persistence", () => { db.prepare( "INSERT INTO threads (id, workspace_id, title, branch, provider, status, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", ).run(THREAD_ID, "ws-1", "Parent", "main", "claude", "active", now, now); - const canonicalSink = new CanonicalAgentEventSink(db, vi.fn()); + const canonicalSink = new CanonicalAgentBoundary(db, vi.fn()); const published: AgentEvent[] = []; let sqliteAvailable = false; const appendChunk = ParentAssistantTextCheckpointService.prototype.appendChunk; @@ -569,7 +569,7 @@ describe("AgentService narrative persistence", () => { db.prepare( "INSERT INTO threads (id, workspace_id, title, branch, provider, status, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", ).run(THREAD_ID, "ws-1", "Parent", "main", "claude", "active", now, now); - const canonicalSink = new CanonicalAgentEventSink(db, vi.fn()); + const canonicalSink = new CanonicalAgentBoundary(db, vi.fn()); const published: AgentEvent[] = []; const restoreChunks = ParentAssistantTextCheckpointService.prototype.restoreChunks; const restoreChunksSpy = vi.spyOn(ParentAssistantTextCheckpointService.prototype, "restoreChunks") @@ -635,7 +635,7 @@ describe("AgentService narrative persistence", () => { db.prepare( "INSERT INTO threads (id, workspace_id, title, branch, provider, status, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", ).run(THREAD_ID, "ws-1", "Parent", "main", "claude", "active", now, now); - const canonicalSink = new CanonicalAgentEventSink(db, vi.fn()); + const canonicalSink = new CanonicalAgentBoundary(db, vi.fn()); const restoreChunksSpy = vi.spyOn(ParentAssistantTextCheckpointService.prototype, "restoreChunks") .mockImplementation(() => { throw Object.assign(new Error("database locked"), { code: "SQLITE_BUSY" }); @@ -697,7 +697,7 @@ describe("AgentService narrative persistence", () => { db.prepare( "INSERT INTO threads (id, workspace_id, title, branch, provider, status, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", ).run(THREAD_ID, "ws-1", "Parent", "main", "claude", "active", now, now); - const canonicalSink = new CanonicalAgentEventSink(db, vi.fn()); + const canonicalSink = new CanonicalAgentBoundary(db, vi.fn()); const restoreChunksSpy = vi.spyOn(ParentAssistantTextCheckpointService.prototype, "restoreChunks") .mockImplementation(() => { throw Object.assign(new Error("database locked"), { code: "SQLITE_BUSY" }); @@ -767,7 +767,7 @@ describe("AgentService narrative persistence", () => { db.prepare( "INSERT INTO threads (id, workspace_id, title, branch, provider, status, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", ).run(THREAD_ID, "ws-1", "Parent", "main", "claude", "active", now, now); - const canonicalSink = new CanonicalAgentEventSink(db, vi.fn()); + const canonicalSink = new CanonicalAgentBoundary(db, vi.fn()); const journalService = new ParentAssistantTextCheckpointService( db, undefined, @@ -851,7 +851,7 @@ describe("AgentService narrative persistence", () => { db.prepare( "INSERT INTO threads (id, workspace_id, title, branch, provider, status, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", ).run(THREAD_ID, "ws-1", "Parent", "main", "claude", "active", now, now); - const canonicalSink = new CanonicalAgentEventSink(db, vi.fn()); + const canonicalSink = new CanonicalAgentBoundary(db, vi.fn()); const published: AgentEvent[] = []; const appendSpy = vi.spyOn(ParentAssistantTextCheckpointService.prototype, "appendChunk") .mockImplementation(() => { @@ -920,7 +920,7 @@ describe("AgentService narrative persistence", () => { db.prepare( "INSERT INTO threads (id, workspace_id, title, branch, provider, status, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", ).run(THREAD_ID, "ws-1", "Parent", "main", "claude", "active", now, now); - const canonicalSink = new CanonicalAgentEventSink(db, vi.fn()); + const canonicalSink = new CanonicalAgentBoundary(db, vi.fn()); const published: AgentEvent[] = []; const { providerEmitter, service, thoughtBulk, narrativeStore } = build({ db, @@ -990,7 +990,7 @@ describe("AgentService narrative persistence", () => { db.prepare( "INSERT INTO threads (id, workspace_id, title, branch, provider, status, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", ).run(THREAD_ID, "ws-1", "Parent", "main", "claude", "active", now, now); - const canonicalSink = new CanonicalAgentEventSink(db, vi.fn()); + const canonicalSink = new CanonicalAgentBoundary(db, vi.fn()); const observed: Array<{ type: string; persisted: string | undefined }> = []; const { providerEmitter, service } = build({ db, @@ -1047,7 +1047,7 @@ describe("AgentService narrative persistence", () => { db.prepare( "INSERT INTO threads (id, workspace_id, title, branch, provider, status, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", ).run(THREAD_ID, "ws-1", "Parent", "main", "claude", "active", now, now); - const canonicalSink = new CanonicalAgentEventSink(db, vi.fn()); + const canonicalSink = new CanonicalAgentBoundary(db, vi.fn()); const published: AgentEvent[] = []; const messageRepo = new SqliteMessageRepo(db); const { service } = build({ @@ -1599,7 +1599,7 @@ describe("AgentService narrative persistence", () => { db.prepare( "INSERT INTO threads (id, workspace_id, title, branch, provider, status, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", ).run(THREAD_ID, "ws-1", "Parent", "main", "claude", "active", now, now); - const canonicalSink = new CanonicalAgentEventSink(db, vi.fn()); + const canonicalSink = new CanonicalAgentBoundary(db, vi.fn()); const published: AgentEvent[] = []; const { service, providerEmitter, hookBulk, narrativeStore } = build({ db, @@ -1930,7 +1930,7 @@ describe("AgentService narrative persistence", () => { "INSERT INTO threads (id, workspace_id, title, branch, provider, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?)", ).run(THREAD_ID, "ws-1", "Parent", "main", "codex", now, now); const published = vi.fn(); - const canonicalSink = new CanonicalAgentEventSink(db, published); + const canonicalSink = new CanonicalAgentBoundary(db, published); const { providerEmitter, service, narrativeStore } = build({ db, canonicalSink }); const executionId = narrativeExecutionId(service); const messages = new SqliteMessageRepo(db); @@ -2093,7 +2093,7 @@ describe("AgentService narrative persistence", () => { db.prepare( "INSERT INTO threads (id, workspace_id, title, branch, provider, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?)", ).run(THREAD_ID, "ws-nested", "Parent", "main", "codex", now, now); - const canonicalSink = new CanonicalAgentEventSink(db, vi.fn()); + const canonicalSink = new CanonicalAgentBoundary(db, vi.fn()); const { providerEmitter, service } = build({ db, canonicalSink }); const executionId = narrativeExecutionId(service); const messages = new SqliteMessageRepo(db); @@ -2352,7 +2352,7 @@ describe("AgentService narrative persistence", () => { db.prepare( "INSERT INTO threads (id, workspace_id, title, branch, provider, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?)", ).run(THREAD_ID, "ws-failure", "Parent", "main", "codex", now, now); - const canonicalSink = new CanonicalAgentEventSink(db, vi.fn()); + const canonicalSink = new CanonicalAgentBoundary(db, vi.fn()); const { providerEmitter, service } = build({ db, canonicalSink }); const executionId = narrativeExecutionId(service); const messages = new SqliteMessageRepo(db); @@ -2432,7 +2432,7 @@ describe("AgentService narrative persistence", () => { db.prepare( "INSERT INTO threads (id, workspace_id, title, branch, provider, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?)", ).run(THREAD_ID, "ws-action", "Parent", "main", "codex", now, now); - const canonicalSink = new CanonicalAgentEventSink(db, vi.fn()); + const canonicalSink = new CanonicalAgentBoundary(db, vi.fn()); const { providerEmitter, service } = build({ db, canonicalSink }); const executionId = narrativeExecutionId(service); const messages = new SqliteMessageRepo(db); diff --git a/apps/server/src/features/agents/orchestration/__tests__/agent-service-plan-marker.test.ts b/apps/server/src/features/agents/orchestration/__tests__/agent-service-plan-marker.test.ts index 9deb40fab..ac20a5e45 100644 --- a/apps/server/src/features/agents/orchestration/__tests__/agent-service-plan-marker.test.ts +++ b/apps/server/src/features/agents/orchestration/__tests__/agent-service-plan-marker.test.ts @@ -10,7 +10,7 @@ import { MessageRepo } from "../../conversation/persistence/message-repo.js"; import { PlanQuestionAnswersRepo } from "../../planning/persistence/plan-question-answers-repo.js"; import { TurnSnapshotRepo } from "../../turns/persistence/turn-snapshot-repo.js"; import { createAgentServiceForTest } from "./agent-service-test-harness.js"; -import { createCanonicalAgentEventSinkStub } from "../../canonical/__tests__/canonical-agent-event-sink-stub.js"; +import { createCanonicalAgentBoundaryStub } from "../../canonical/__tests__/canonical-agent-boundary-stub.js"; import { NarrativeStore } from "../../conversation/narrative/narrative-store.js"; import { PlanQuestionService } from "../../planning/plan-question-service.js"; import { PlanTurnService } from "../../planning/plan-turn-service.js"; @@ -120,7 +120,7 @@ function buildService(db: Database) { undefined, undefined, undefined, - createCanonicalAgentEventSinkStub(db), + createCanonicalAgentBoundaryStub(db), ); const plans = new PlanTurnService( threadRepo, diff --git a/apps/server/src/features/agents/orchestration/__tests__/agent-service-session-invalidated.test.ts b/apps/server/src/features/agents/orchestration/__tests__/agent-service-session-invalidated.test.ts index 37793d64e..27213272d 100644 --- a/apps/server/src/features/agents/orchestration/__tests__/agent-service-session-invalidated.test.ts +++ b/apps/server/src/features/agents/orchestration/__tests__/agent-service-session-invalidated.test.ts @@ -19,7 +19,7 @@ import { ToolCallRecordRepo } from "../../tools/persistence/tool-call-record-rep import { TurnSnapshotRepo } from "../../turns/persistence/turn-snapshot-repo.js"; import { AgentService } from "../agent-service.js"; import { createAgentServiceForTest, startAgentServiceIngressForTest, wrapProviderEmitterForRuntimeEvents } from "./agent-service-test-harness.js"; -import { createCanonicalAgentEventSinkStub } from "../../canonical/__tests__/canonical-agent-event-sink-stub.js"; +import { createCanonicalAgentBoundaryStub } from "../../canonical/__tests__/canonical-agent-boundary-stub.js"; import { NarrativeStore } from "../../conversation/narrative/narrative-store.js"; import { ParentAssistantTextCheckpointService } from "../../turns/parent-assistant-text-checkpoint-service.js"; import type { GitService } from "../../../projects/index.js"; @@ -147,7 +147,7 @@ describe("AgentService clears sdk_session_id on session invalidation", () => { assertUsable: vi.fn(), } as unknown as ProviderAvailabilityService; providerEventIngress = new ProviderEventIngress(); - const canonicalSink = createCanonicalAgentEventSinkStub(db); + const canonicalSink = createCanonicalAgentBoundaryStub(db); Object.assign(canonicalSink, { recordNativeCursor: vi.fn(() => true) }); svc = createAgentServiceForTest( diff --git a/apps/server/src/features/agents/orchestration/__tests__/agent-service-transient-retry.test.ts b/apps/server/src/features/agents/orchestration/__tests__/agent-service-transient-retry.test.ts index f79ab4cec..fb83c7ffe 100644 --- a/apps/server/src/features/agents/orchestration/__tests__/agent-service-transient-retry.test.ts +++ b/apps/server/src/features/agents/orchestration/__tests__/agent-service-transient-retry.test.ts @@ -10,7 +10,7 @@ import { wrapProviderEmitterForRuntimeEvents, } from "./agent-service-test-harness.js"; import { publishParentProviderEvent } from "../../events/provider-event-publication.js"; -import { createCanonicalAgentEventSinkStub } from "../../canonical/__tests__/canonical-agent-event-sink-stub.js"; +import { createCanonicalAgentBoundaryStub } from "../../canonical/__tests__/canonical-agent-boundary-stub.js"; import { ThreadControlMutationReservationService } from "../../../thread-control/index.js"; import { NarrativeStore } from "../../conversation/narrative/narrative-store.js"; import { ParentAssistantTextCheckpointService } from "../../turns/parent-assistant-text-checkpoint-service.js"; @@ -219,7 +219,7 @@ function buildService(): { undefined, threadControlMcp as never, mutationReservations, - createCanonicalAgentEventSinkStub(db), + createCanonicalAgentBoundaryStub(db), ); return { diff --git a/apps/server/src/features/agents/orchestration/__tests__/agent-service-turn-cleanup.test.ts b/apps/server/src/features/agents/orchestration/__tests__/agent-service-turn-cleanup.test.ts index 30de5b646..1a65b16ef 100644 --- a/apps/server/src/features/agents/orchestration/__tests__/agent-service-turn-cleanup.test.ts +++ b/apps/server/src/features/agents/orchestration/__tests__/agent-service-turn-cleanup.test.ts @@ -40,8 +40,8 @@ import { waitForAgentServiceIngressForTest, wrapProviderEmitterForRuntimeEvents, } from "./agent-service-test-harness.js"; -import { createCanonicalAgentEventSinkStub } from "../../canonical/__tests__/canonical-agent-event-sink-stub.js"; -import { CanonicalAgentEventSink } from "../../canonical/canonical-agent-event-sink.js"; +import { createCanonicalAgentBoundaryStub } from "../../canonical/__tests__/canonical-agent-boundary-stub.js"; +import { CanonicalAgentBoundary } from "../../canonical/canonical-agent-boundary.js"; import { NarrativeStore } from "../../conversation/narrative/narrative-store.js"; import { ParentAssistantTextCheckpointService } from "../../turns/parent-assistant-text-checkpoint-service.js"; import { broadcast } from "../../../../application/transport/push.js"; @@ -340,7 +340,7 @@ function buildService( undefined, undefined, mutationReservations, - createCanonicalAgentEventSinkStub(db), + createCanonicalAgentBoundaryStub(db), undefined, undefined, ); @@ -1460,7 +1460,7 @@ describe("AgentService Ended finalization", () => { stopSession: ReturnType; interruptChildTurn: ReturnType; }; - let canonicalSink: CanonicalAgentEventSink; + let canonicalSink: CanonicalAgentBoundary; let canonicalEvents: CanonicalAgentEventEnvelope[]; let service: AgentService; let pendingPlanOutputs: Map; @@ -1526,7 +1526,7 @@ describe("AgentService Ended finalization", () => { listAnsweredForThread: vi.fn(() => []), } as unknown as PlanQuestionAnswersRepo; - canonicalSink = new CanonicalAgentEventSink(db, (events) => { + canonicalSink = new CanonicalAgentBoundary(db, (events) => { canonicalEvents.push(...events); }); pendingPlanOutputs = new Map(); diff --git a/apps/server/src/features/agents/orchestration/__tests__/agent-service-turn-started.test.ts b/apps/server/src/features/agents/orchestration/__tests__/agent-service-turn-started.test.ts index 4b32248a3..9232da236 100644 --- a/apps/server/src/features/agents/orchestration/__tests__/agent-service-turn-started.test.ts +++ b/apps/server/src/features/agents/orchestration/__tests__/agent-service-turn-started.test.ts @@ -17,7 +17,7 @@ import { ToolCallRecordRepo } from "../../tools/persistence/tool-call-record-rep import { TurnSnapshotRepo } from "../../turns/persistence/turn-snapshot-repo.js"; import { AgentService } from "../agent-service.js"; import { createAgentServiceForTest, startAgentServiceIngressForTest, wrapProviderEmitterForRuntimeEvents } from "./agent-service-test-harness.js"; -import { CanonicalAgentEventSink } from "../../canonical/canonical-agent-event-sink.js"; +import { CanonicalAgentBoundary } from "../../canonical/canonical-agent-boundary.js"; import { NarrativeStore } from "../../conversation/narrative/narrative-store.js"; import { ParentAssistantTextCheckpointService } from "../../turns/parent-assistant-text-checkpoint-service.js"; import type { GitService } from "../../../projects/index.js"; @@ -43,7 +43,7 @@ describe("AgentService.sendMessage emits TurnStarted", () => { let toolCallRecordRepo: ToolCallRecordRepo; let turnSnapshotRepo: TurnSnapshotRepo; let svc: AgentService; - let canonicalSink: CanonicalAgentEventSink; + let canonicalSink: CanonicalAgentBoundary; let providerStub: NodeEvents.EventEmitter & Partial & { sendTurn: ReturnType; }; @@ -123,7 +123,7 @@ describe("AgentService.sendMessage emits TurnStarted", () => { assertUsable: vi.fn(), } as unknown as ProviderAvailabilityService; - canonicalSink = new CanonicalAgentEventSink(db, vi.fn()); + canonicalSink = new CanonicalAgentBoundary(db, vi.fn()); svc = createAgentServiceForTest( threadRepo, workspaceRepo, diff --git a/apps/server/src/features/agents/recovery/__tests__/turn-recovery-service.test.ts b/apps/server/src/features/agents/recovery/__tests__/turn-recovery-service.test.ts index 33cf7bfd8..604f93050 100644 --- a/apps/server/src/features/agents/recovery/__tests__/turn-recovery-service.test.ts +++ b/apps/server/src/features/agents/recovery/__tests__/turn-recovery-service.test.ts @@ -9,9 +9,9 @@ import { openMemoryDatabase } from "../../../../runtime/persistence/sqlite/datab import { MessageRepo } from "../../conversation/persistence/message-repo.js"; import { ThreadRepo } from "../../../thread-control/persistence/thread-repo.js"; import { - CanonicalAgentEventSink, + CanonicalAgentBoundary, type CanonicalAgentEventPublisher, -} from "../../canonical/canonical-agent-event-sink.js"; +} from "../../canonical/canonical-agent-boundary.js"; import { ParentAssistantTextCheckpointService } from "../../turns/parent-assistant-text-checkpoint-service.js"; import { NarrativeStore } from "../../conversation/narrative/narrative-store.js"; import { ToolCallRecordRepo } from "../../tools/persistence/tool-call-record-repo.js"; @@ -29,7 +29,7 @@ const EXECUTION_ID = "00000000-0000-4000-8000-000000000015"; describe("TurnRecoveryService", () => { let db: Database; - let sink: CanonicalAgentEventSink; + let sink: CanonicalAgentBoundary; let threadRepo: ThreadRepo; let messageRepo: MessageRepo; let defaultCheckpoints: ParentAssistantTextCheckpointService; @@ -45,7 +45,7 @@ describe("TurnRecoveryService", () => { "INSERT INTO threads (id, workspace_id, title, branch, provider, status, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", ).run(THREAD_ID, "workspace-recovery", "Recovery", "main", "codex", "active", NOW, NOW); published = vi.fn(); - sink = new CanonicalAgentEventSink(db, published); + sink = new CanonicalAgentBoundary(db, published); threadRepo = new ThreadRepo(db); messageRepo = new MessageRepo(db); defaultCheckpoints = new ParentAssistantTextCheckpointService(db); diff --git a/apps/server/src/features/agents/turns/__tests__/parent-assistant-text-checkpoint-service.test.ts b/apps/server/src/features/agents/turns/__tests__/parent-assistant-text-checkpoint-service.test.ts index 7d86998bd..2fa03d885 100644 --- a/apps/server/src/features/agents/turns/__tests__/parent-assistant-text-checkpoint-service.test.ts +++ b/apps/server/src/features/agents/turns/__tests__/parent-assistant-text-checkpoint-service.test.ts @@ -5,7 +5,7 @@ import * as NodePath from "node:path"; import type { Database } from "bun:sqlite"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { openMemoryDatabase } from "../../../../runtime/persistence/sqlite/database.js"; -import { CanonicalAgentEventSink } from "../../canonical/canonical-agent-event-sink.js"; +import { CanonicalAgentBoundary } from "../../canonical/canonical-agent-boundary.js"; import { MessageRepo } from "../../conversation/persistence/message-repo.js"; import { ParentAssistantTextCheckpointQueue, @@ -30,7 +30,7 @@ function seedParentAssistantTurn(db: Database): void { "INSERT INTO threads (id, workspace_id, title, branch, provider, status, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", ).run(THREAD_ID, "workspace-1522", "Durability", "main", "claude", "active", NOW, NOW); const messages = new MessageRepo(db); - new CanonicalAgentEventSink(db, () => {}).startParentTurn({ + new CanonicalAgentBoundary(db, () => {}).startParentTurn({ thread: { id: THREAD_ID, workspaceId: "workspace-1522", providerId: "claude", createdAt: NOW }, turnId: TURN_ID, executionId: EXECUTION_ID, diff --git a/apps/server/src/features/agents/turns/__tests__/turn-diff-finalization.test.ts b/apps/server/src/features/agents/turns/__tests__/turn-diff-finalization.test.ts index da3f0c489..9add876b7 100644 --- a/apps/server/src/features/agents/turns/__tests__/turn-diff-finalization.test.ts +++ b/apps/server/src/features/agents/turns/__tests__/turn-diff-finalization.test.ts @@ -10,7 +10,7 @@ import { ToolCallRecordRepo } from "../../tools/persistence/tool-call-record-rep import { ThreadRepo } from "../../../thread-control/persistence/thread-repo.js"; import { SnapshotService } from "../../../projects/diffs/snapshots/snapshot-service.js"; import { RealGitExecutor } from "../../../projects/git/execution/real-git-executor.js"; -import { CanonicalAgentEventSink } from "../../canonical/canonical-agent-event-sink.js"; +import { CanonicalAgentBoundary } from "../../canonical/canonical-agent-boundary.js"; import { TurnDiffRepo } from "../persistence/turn-diff-repo.js"; import { TurnSnapshotRepo } from "../persistence/turn-snapshot-repo.js"; import { TurnDiffService } from "../turn-diff-service.js"; @@ -34,7 +34,7 @@ describe("native diff terminal persistence", () => { function harness(canonical: boolean) { const messages = new MessageRepo(db); const narrative = new NarrativeStore(messages, new ToolCallRecordRepo(db), new ThoughtSegmentRepo(db), new HookExecutionRepo(db)); - const sink = canonical ? new CanonicalAgentEventSink(db, () => {}) : undefined; + const sink = canonical ? new CanonicalAgentBoundary(db, () => {}) : undefined; if (sink) sink.startParentTurn({ thread: { id: identity.threadId, workspaceId: "ws-1", providerId: "codex", createdAt: new Date().toISOString() }, turnId: identity.turnId, executionId: identity.turnExecutionId, permissionMode: "supervised", providerIdentities: [], diff --git a/apps/server/src/features/agents/turns/__tests__/turn-finalizer.test.ts b/apps/server/src/features/agents/turns/__tests__/turn-finalizer.test.ts index 1e96bdbd9..fc0bf1f33 100644 --- a/apps/server/src/features/agents/turns/__tests__/turn-finalizer.test.ts +++ b/apps/server/src/features/agents/turns/__tests__/turn-finalizer.test.ts @@ -16,7 +16,7 @@ import { TurnSnapshotRepo } from "../persistence/turn-snapshot-repo.js"; import type { TurnOutcome } from "../turn-outcome.js"; import type { TurnFileTracker } from "../turn-file-tracker.js"; import { broadcast } from "../../../../application/transport/push.js"; -import { CanonicalAgentEventSink } from "../../canonical/canonical-agent-event-sink.js"; +import { CanonicalAgentBoundary } from "../../canonical/canonical-agent-boundary.js"; import { ParentAssistantTextCheckpointService } from "../parent-assistant-text-checkpoint-service.js"; vi.mock("../../../../application/transport/push.js", () => ({ broadcast: vi.fn() })); @@ -355,7 +355,7 @@ describe("TurnFinalizer canonical commit recovery", () => { thoughtRepo, hookRepo, ); - const sink = new CanonicalAgentEventSink(db, vi.fn()); + const sink = new CanonicalAgentBoundary(db, vi.fn()); sink.startParentTurn({ thread: { id: THREAD, diff --git a/scripts/perf/subagent-detail-fixture.ts b/scripts/perf/subagent-detail-fixture.ts index 75db4a4d0..790591272 100644 --- a/scripts/perf/subagent-detail-fixture.ts +++ b/scripts/perf/subagent-detail-fixture.ts @@ -1,7 +1,7 @@ import "../../apps/server/node_modules/reflect-metadata/Reflect.js"; import * as NodeFS from "node:fs"; import * as NodePath from "node:path"; -import { CanonicalAgentEventSink } from "../../apps/server/src/features/agents/canonical/canonical-agent-event-sink.js"; +import { CanonicalAgentBoundary } from "../../apps/server/src/features/agents/canonical/canonical-agent-boundary.js"; import { MessageRepo } from "../../apps/server/src/features/agents/conversation/persistence/message-repo.js"; import { ToolCallRecordRepo } from "../../apps/server/src/features/agents/tools/persistence/tool-call-record-repo.js"; import { WorkspaceRepo } from "../../apps/server/src/features/projects/persistence/workspace-repo.js"; @@ -58,7 +58,7 @@ function databasePath(repoRoot: string): string { return requested; } -function recordCompletedBashCalls(sink: CanonicalAgentEventSink, childThreadId: string, nativeTurnId: string): void { +function recordCompletedBashCalls(sink: CanonicalAgentBoundary, childThreadId: string, nativeTurnId: string): void { for (let index = 0; index < COMPLETED_TOOL_COUNT; index += 1) { const nativeItemId = `fixture-completed-${index}`; const isError = index === 124; @@ -68,13 +68,13 @@ function recordCompletedBashCalls(sink: CanonicalAgentEventSink, childThreadId: } } -function recordActiveBashCalls(sink: CanonicalAgentEventSink, childThreadId: string, nativeTurnId: string): void { +function recordActiveBashCalls(sink: CanonicalAgentBoundary, childThreadId: string, nativeTurnId: string): void { for (let index = 0; index < ACTIVE_TOOL_COUNT; index += 1) { sink.recordCodexChildItem({ childThreadId, nativeTurnId, nativeItemId: `fixture-active-${index}`, eventKey: "started", kind: "tool-call", payload: { projection: "codexChildToolCall", toolName: "Bash", toolInput: { command: `printf fixture-active-${index}` } } }); } } -function verifyFixtureRecovery(sink: CanonicalAgentEventSink, parentThreadId: string, childThreadId: string): void { +function verifyFixtureRecovery(sink: CanonicalAgentBoundary, parentThreadId: string, childThreadId: string): void { const recovery = sink.recoverThread(childThreadId, { conversationRevision: 0, rosterRevision: 0 }); const canonicalItems = recovery.mode === "snapshot" ? Object.values(recovery.snapshot.state.items) : []; const narrativeItemCount = canonicalItems.filter((item) => item.kind !== "message").length; @@ -126,7 +126,7 @@ function seed(repoRoot: string, path: string, dbPath: string): Descriptor { const fixturePath = NodePath.join(repoRoot, ".dev", "fixture-repo"); const workspace = workspaces.findByPath(fixturePath) ?? workspaces.create("fixture-repo", fixturePath); const parent = threads.create(workspace.id, MARKER, "local", "main", false, "codex"); - const sink = new CanonicalAgentEventSink(db, () => {}); + const sink = new CanonicalAgentBoundary(db, () => {}); const parentTurnId = crypto.randomUUID(); const parentExecutionId = crypto.randomUUID(); const parentItemId = "toolCall:fixture-subagent"; From acfb860dc46309503f225cf8cc860e8ae6f1fe68 Mon Sep 17 00:00:00 2001 From: Agent Runtime Fixture Date: Sun, 27 Sep 2026 13:04:11 +0100 Subject: [PATCH 135/136] refactor(server): share execution identity and lease matching --- .../canonical-execution-semantic-writer.ts | 22 +++++-------------- .../execution/execution-mailbox-owner.ts | 5 +---- .../execution/execution-mailbox-protocol.ts | 14 ++++++++++++ .../execution-provider-event-ownership.ts | 11 +--------- .../execution-worker-file-evidence.ts | 5 ++--- .../agents/execution/execution-worker-port.ts | 19 +++++----------- .../provider-execution-event-state.ts | 4 ++-- 7 files changed, 31 insertions(+), 49 deletions(-) diff --git a/apps/server/src/features/agents/canonical/canonical-execution-semantic-writer.ts b/apps/server/src/features/agents/canonical/canonical-execution-semantic-writer.ts index 8cfed789a..5a4973fbf 100644 --- a/apps/server/src/features/agents/canonical/canonical-execution-semantic-writer.ts +++ b/apps/server/src/features/agents/canonical/canonical-execution-semantic-writer.ts @@ -20,6 +20,7 @@ import { z } from "zod"; import { ACTIVE_TURN_WRITE_BATCH_LIMITS } from "../../../runtime/persistence/sqlite/bounded-write-batches.js"; import { ThreadRepo } from "../../thread-control/persistence/thread-repo.js"; +import { sameExecution, sameLease } from "../execution/execution-mailbox-protocol.js"; import type { ExecutionIdentity, ExecutionLease } from "../execution/execution-mailbox-protocol.js"; import type { ExecutionProviderCommitReceipt, @@ -1600,8 +1601,7 @@ function validAssistantTextInput( function validAssistantTextEntry(input: ParentAssistantTextCheckpointInput | undefined, execution: ExecutionIdentity): boolean { if (!input) return false; - return input.executionId === execution.executionId && input.threadId === execution.threadId - && input.turnId === execution.turnId && typeof input.text === "string" + return sameExecution(input, execution) && typeof input.text === "string" && Buffer.byteLength(input.text, "utf8") > 0 && Number.isSafeInteger(input.sequence) && input.sequence > 0; } @@ -1621,9 +1621,7 @@ function validFinish( mutation: Extract, ): boolean { return mutation.outcome === mutation.input.outcome - && mutation.input.threadId === operation.execution.threadId - && mutation.input.turnId === operation.execution.turnId - && mutation.input.executionId === operation.execution.executionId; + && sameExecution(mutation.input, operation.execution); } function unchangedTextEvent(event: ExecutionLivePublicationIntent["event"]): boolean { @@ -1655,8 +1653,7 @@ function validTerminalProviderEvent( input: Extract["providerEvent"], ): boolean { return input === undefined || Boolean(input.phase && input.phase.length <= 64 && input.events.length > 0 - && input.events.every((event) => event.routing.threadId === execution.threadId - && event.routing.turnId === execution.turnId && event.routing.executionId === execution.executionId)); + && input.events.every((event) => sameExecution(event.routing, execution))); } function sameTerminalEvent( @@ -1703,8 +1700,7 @@ function nextHead(head: SemanticHead, operation: ExecutionSemanticOperation): bo return !head.terminal && head.execution.threadId === operation.execution.threadId && head.execution.turnId === operation.execution.turnId && head.ordinal + 1 === operation.ordinal - && lease.ownerEpoch === candidate.ownerEpoch && lease.workerIndex === candidate.workerIndex - && lease.workerGeneration === candidate.workerGeneration && lease.leaseId === candidate.leaseId; + && sameLease(lease, candidate); } function validLostExecutionInput(input: LostExecutionInterruption, operation: ExecutionSemanticOperation): boolean { @@ -1713,13 +1709,7 @@ function validLostExecutionInput(input: LostExecutionInterruption, operation: Ex } function sameExecutionAndLease(head: SemanticHead, input: Pick): boolean { - return head.execution.threadId === input.execution.threadId - && head.execution.turnId === input.execution.turnId - && head.execution.executionId === input.execution.executionId - && head.lease.ownerEpoch === input.lease.ownerEpoch - && head.lease.workerIndex === input.lease.workerIndex - && head.lease.workerGeneration === input.lease.workerGeneration - && head.lease.leaseId === input.lease.leaseId; + return sameExecution(head.execution, input.execution) && sameLease(head.lease, input.lease); } function workerLossOperation(input: LostExecutionInterruption): ExecutionSemanticOperation { diff --git a/apps/server/src/features/agents/execution/execution-mailbox-owner.ts b/apps/server/src/features/agents/execution/execution-mailbox-owner.ts index c10872910..5ca45bbc8 100644 --- a/apps/server/src/features/agents/execution/execution-mailbox-owner.ts +++ b/apps/server/src/features/agents/execution/execution-mailbox-owner.ts @@ -1,5 +1,6 @@ import type { DataOnlyParentTurnStartInput } from "../canonical/canonical-parent-turn-write.js"; import type { ExecutionParentStartContext } from "./provider-execution-event-state.js"; +import { sameExecution } from "./execution-mailbox-protocol.js"; import type { ExecutionIdentity, ExecutionLease, ExecutionMailboxCompletion } from "./execution-mailbox-protocol.js"; import type { ExecutionMailboxScheduler, ExecutionRecoveryReceipt } from "./execution-mailbox-scheduler.js"; import type { ExecutionWorkCommand, ExecutionWorkerResult } from "./execution-worker-handler.js"; @@ -127,7 +128,3 @@ function requireReply(completion: ExecutionMailboxCompletion { postMessage(request: ExecutionWorkerRequest): void; terminate(): void; } + +/** Identity fields carried by inbound provider routing, where the turn may be unresolved. */ +export type RoutedExecutionIdentity = Omit & { readonly turnId?: string | undefined }; + +/** Two executions match only when every durable identity field agrees. */ +export function sameExecution(left: RoutedExecutionIdentity, right: ExecutionIdentity): boolean { + return left.threadId === right.threadId && left.turnId === right.turnId && left.executionId === right.executionId; +} + +/** Two leases match only when the epoch, worker incarnation, and lease id all agree. */ +export function sameLease(left: ExecutionLease, right: ExecutionLease): boolean { + return left.ownerEpoch === right.ownerEpoch && left.workerIndex === right.workerIndex + && left.workerGeneration === right.workerGeneration && left.leaseId === right.leaseId; +} diff --git a/apps/server/src/features/agents/execution/execution-provider-event-ownership.ts b/apps/server/src/features/agents/execution/execution-provider-event-ownership.ts index c421034b5..a3557a43b 100644 --- a/apps/server/src/features/agents/execution/execution-provider-event-ownership.ts +++ b/apps/server/src/features/agents/execution/execution-provider-event-ownership.ts @@ -1,3 +1,4 @@ +import { sameExecution } from "./execution-mailbox-protocol.js"; import type { ExecutionIdentity } from "./execution-mailbox-protocol.js"; import type { ExecutionMailboxOwner } from "./execution-mailbox-owner.js"; import type { ProviderEventCommitReceipt } from "@mcode/providers"; @@ -42,11 +43,6 @@ export class ExecutionProviderEventOwnership implements ProviderEventOwnership { this.prepareCommand = prepare; } - /** Activate another provider only after its worker live effects are installed. */ - enableProvider(providerId: string): void { - this.workerProviders.add(providerId); - } - /** Admit one provider attempt only while the exact mailbox still owns the thread. */ async bind(execution: ExecutionIdentity, deliveryAttempt: number): Promise { this.requireOwner(execution, deliveryAttempt); @@ -172,8 +168,3 @@ export class ExecutionProviderEventOwnership implements ProviderEventOwnership { await this.onCommitted?.(execution, result); } } - -function sameExecution(left: ExecutionIdentity | undefined, right: ExecutionIdentity): boolean { - return left?.threadId === right.threadId && left.turnId === right.turnId - && left.executionId === right.executionId; -} diff --git a/apps/server/src/features/agents/execution/execution-worker-file-evidence.ts b/apps/server/src/features/agents/execution/execution-worker-file-evidence.ts index 17644b4a8..a0e3ad04b 100644 --- a/apps/server/src/features/agents/execution/execution-worker-file-evidence.ts +++ b/apps/server/src/features/agents/execution/execution-worker-file-evidence.ts @@ -6,6 +6,7 @@ import { RealGitExecutor } from "../../projects/git/execution/real-git-executor. import { SnapshotService } from "../../projects/diffs/snapshots/snapshot-service.js"; import { TurnFileTracker, type CapturedToolUseObservation, type FileTurnHandoff } from "../turns/turn-file-tracker.js"; import type { PreparedExecutionFileEvidence } from "../turns/turn-execution-file-evidence.js"; +import { sameExecution } from "./execution-mailbox-protocol.js"; import type { ExecutionIdentity } from "./execution-mailbox-protocol.js"; import { prepareFrozenExecutionFileEvidence, @@ -132,9 +133,7 @@ export class ExecutionWorkerFileEvidence { } private matches(active: ActiveFileExecution, execution: ExecutionIdentity, deliveryAttempt: number): boolean { - return active.execution.threadId === execution.threadId - && active.execution.turnId === execution.turnId - && active.execution.executionId === execution.executionId + return sameExecution(active.execution, execution) && active.deliveryAttempt === deliveryAttempt; } diff --git a/apps/server/src/features/agents/execution/execution-worker-port.ts b/apps/server/src/features/agents/execution/execution-worker-port.ts index f37eeb8a7..acf7702d8 100644 --- a/apps/server/src/features/agents/execution/execution-worker-port.ts +++ b/apps/server/src/features/agents/execution/execution-worker-port.ts @@ -1,3 +1,4 @@ +import { sameExecution, sameLease } from "./execution-mailbox-protocol.js"; import type { ExecutionWorkerPort, ExecutionWorkerReply, @@ -196,24 +197,14 @@ function defaultWorkerUrl(): URL { function matchesOperation(request: Request | undefined, operation: ExecutionSemanticOperation): boolean { return request !== undefined && request.ordinal === operation.ordinal - && request.execution.threadId === operation.execution.threadId - && request.execution.turnId === operation.execution.turnId - && request.execution.executionId === operation.execution.executionId - && request.lease.leaseId === operation.lease.leaseId - && request.lease.ownerEpoch === operation.lease.ownerEpoch - && request.lease.workerGeneration === operation.lease.workerGeneration - && request.lease.workerIndex === operation.lease.workerIndex + && sameExecution(request.execution, operation.execution) + && sameLease(request.lease, operation.lease) && operation.operationId === `${request.lease.leaseId}:${request.ordinal}`; } function matchesReply(request: Request | undefined, reply: Reply): boolean { return request !== undefined && request.requestId === reply.requestId && request.ordinal === reply.ordinal - && request.execution.threadId === reply.execution.threadId - && request.execution.turnId === reply.execution.turnId - && request.execution.executionId === reply.execution.executionId - && request.lease.leaseId === reply.lease.leaseId - && request.lease.ownerEpoch === reply.lease.ownerEpoch - && request.lease.workerGeneration === reply.lease.workerGeneration - && request.lease.workerIndex === reply.lease.workerIndex; + && sameExecution(request.execution, reply.execution) + && sameLease(request.lease, reply.lease); } diff --git a/apps/server/src/features/agents/execution/provider-execution-event-state.ts b/apps/server/src/features/agents/execution/provider-execution-event-state.ts index 0550b928b..94f50221d 100644 --- a/apps/server/src/features/agents/execution/provider-execution-event-state.ts +++ b/apps/server/src/features/agents/execution/provider-execution-event-state.ts @@ -3,6 +3,7 @@ import type { ProviderEventDraft } from "@mcode/providers"; import { processProviderEventWorkerTask } from "../../providers/composition/provider-event-worker-protocol.js"; import { CodexLiveEventReducer, type CodexPlanFeature, type SyntheticTerminalInput } from "./codex-live-event-reducer.js"; import { CodexLiveEventEffects, type PreparedCodexLiveEvent } from "./codex-live-event-effects.js"; +import { sameExecution } from "./execution-mailbox-protocol.js"; import type { ExecutionIdentity } from "./execution-mailbox-protocol.js"; import { OtherProviderLiveEventEffects, type OtherProviderRuntimeIntent } from "./provider-live-event-effects.js"; @@ -112,8 +113,7 @@ export class ProviderExecutionEventState { } private matchesExecution(draft: ProviderEventDraft): boolean { - return draft.routing.threadId === this.execution.threadId && draft.routing.turnId === this.execution.turnId - && draft.routing.executionId === this.execution.executionId; + return sameExecution(draft.routing, this.execution); } } From dc5092174917b95db155fa5a43749513a74fe8c2 Mon Sep 17 00:00:00 2001 From: Agent Runtime Fixture Date: Sun, 27 Sep 2026 13:04:20 +0100 Subject: [PATCH 136/136] refactor(server): collapse duplicated worker result and effect shapes --- .../execution/execution-mailbox-scheduler.ts | 19 ++-- .../execution/execution-worker-handler.ts | 99 +++++-------------- .../execution/worker-owned-turn-runtime.ts | 14 ++- 3 files changed, 38 insertions(+), 94 deletions(-) diff --git a/apps/server/src/features/agents/execution/execution-mailbox-scheduler.ts b/apps/server/src/features/agents/execution/execution-mailbox-scheduler.ts index a6dcb95f4..d18a5b3b7 100644 --- a/apps/server/src/features/agents/execution/execution-mailbox-scheduler.ts +++ b/apps/server/src/features/agents/execution/execution-mailbox-scheduler.ts @@ -1,5 +1,6 @@ import * as NodeCrypto from "node:crypto"; +import { sameExecution, sameLease } from "./execution-mailbox-protocol.js"; import type { ExecutionIdentity, ExecutionLease, @@ -19,6 +20,7 @@ export const EXECUTION_CONTROL_KINDS = [ "checkpoint", "effect-result", "provider-outcome", "stage-terminal", "finalize", "release", "finish-from-state", "finish-live-event", "post-terminal-event", ] as const; +/** A command kind that must still settle after Stop begins draining the mailbox. */ export type ExecutionControlKind = typeof EXECUTION_CONTROL_KINDS[number]; const CONTROL_KINDS: ReadonlySet = new Set(EXECUTION_CONTROL_KINDS); @@ -73,7 +75,7 @@ export interface ExecutionLostAssignment { /** Writer evidence that a lost worker's execution can release its thread and slot. */ export type ExecutionRecoveryReceipt = | { readonly kind: "committed"; readonly operationId: string; readonly durableRevision: number } - | { readonly kind: "conflict"; readonly operationId: string; readonly recoveryState: "not-started" | "already-terminal" }; + | { readonly kind: "conflict"; readonly operationId: string; readonly recoveryState?: "not-started" | "already-terminal" }; /** The host supplies durable owner epochs and a worker factory for each fixed slot. */ export interface ExecutionMailboxOptions { @@ -83,10 +85,12 @@ export interface ExecutionMailboxOptions void; } +/** The result of binding a turn attempt to a worker slot before its start is sent. */ export type ExecutionClaim = | { readonly kind: "claimed"; readonly lease: ExecutionLease } | { readonly kind: "thread-busy" | "worker-unavailable" | "shutdown" }; +/** The result of admitting one command; an admitted command resolves through its completion promise. */ export type ExecutionAdmission = | { readonly kind: "admitted"; readonly ordinal: number; readonly completion: Promise> } | { readonly kind: "stale-execution" | "overloaded" | "invalid-size" | "stop-already-requested" | "stopping" | "shutdown" }; @@ -281,7 +285,7 @@ export class ExecutionMailboxScheduler, Result> | undefined { const assignment = this.byThread.get(execution.threadId); - if (!assignment || !sameIdentity(assignment.execution, execution) || !sameLease(assignment.lease, lease)) return undefined; + if (!assignment || !sameExecution(assignment.execution, execution) || !sameLease(assignment.lease, lease)) return undefined; return assignment; } @@ -413,15 +417,6 @@ export class ExecutionMailboxScheduler(request: ExecutionWorkerRequest, reply: ExecutionWorkerReply): boolean { return request.requestId === reply.requestId && request.ordinal === reply.ordinal - && sameIdentity(request.execution, reply.execution) && sameLease(request.lease, reply.lease); + && sameExecution(request.execution, reply.execution) && sameLease(request.lease, reply.lease); } function validIdentity(execution: ExecutionIdentity): boolean { diff --git a/apps/server/src/features/agents/execution/execution-worker-handler.ts b/apps/server/src/features/agents/execution/execution-worker-handler.ts index 0da87b271..d4c007c59 100644 --- a/apps/server/src/features/agents/execution/execution-worker-handler.ts +++ b/apps/server/src/features/agents/execution/execution-worker-handler.ts @@ -17,6 +17,7 @@ import type { import type { ParentNarrativeRecoveryCommit } from "../turns/parent-turn-durability.js"; import type { TaskToolWriteIntent } from "../tasks/task-tool-intent-reducer.js"; import type { PlanPersistenceReady } from "../planning/plan-execution-state.js"; +import { sameExecution, sameLease } from "./execution-mailbox-protocol.js"; import type { ExecutionIdentity, ExecutionLease, @@ -38,21 +39,7 @@ export type ExecutionWorkCommand = | { readonly kind: "event"; readonly phase: string; readonly nativeCursor: unknown | null; readonly events: readonly ProviderEventDraft[]; readonly parentLive?: ParentLiveEffects; readonly terminalInput?: DataOnlyParentTurnFinishInput; readonly deliveryAttempt?: number; readonly capturedFileObservation?: CapturedToolUseObservation | null; readonly frozenFileEvidence?: FrozenExecutionFileEvidence; readonly livePublication?: readonly ExecutionLivePublicationIntent[] } | { readonly kind: "assistant-text"; readonly inputs: readonly ParentAssistantTextCheckpointInput[] } | { readonly kind: "narrative-delta"; readonly input: ParentNarrativeRecoveryCommit } - | { - readonly kind: "live-event"; - readonly text: - | { readonly kind: "unchanged" } - | { readonly kind: "append"; readonly inputs: readonly ParentAssistantTextCheckpointInput[] } - | { readonly kind: "reclassify"; readonly expectedText: string } - | { readonly kind: "promote"; readonly input: ParentAssistantTextCheckpointInput }; - readonly narrative?: ParentNarrativeRecoveryCommit; - readonly taskIntents?: readonly TaskToolWriteIntent[]; - readonly systemIntents?: readonly CodexSystemWriterIntent[]; - readonly message?: DataOnlyParentLiveMessageInput; - readonly planQuestions?: readonly PlanQuestion[]; - readonly planOutput?: PlanPersistenceReady; - readonly publication: ExecutionLivePublicationIntent; - } + | ({ readonly kind: "live-event"; readonly publication: ExecutionLivePublicationIntent } & ParentLiveEffects) | { readonly kind: "checkpoint"; readonly phase: string; readonly nativeCursor: unknown | null } | { readonly kind: "effect-result"; readonly effectId: string; readonly settled: boolean } | { readonly kind: "provider-outcome"; readonly outcome: TurnOutcome } @@ -64,7 +51,19 @@ export type ExecutionWorkCommand = | { readonly kind: "release"; readonly recovery?: ExecutionRecoveryReceipt }; /** Parent effects prepared from the same provider event as its canonical draft. */ -export type ParentLiveEffects = Omit, "kind" | "publication">; +export interface ParentLiveEffects { + readonly text: + | { readonly kind: "unchanged" } + | { readonly kind: "append"; readonly inputs: readonly ParentAssistantTextCheckpointInput[] } + | { readonly kind: "reclassify"; readonly expectedText: string } + | { readonly kind: "promote"; readonly input: ParentAssistantTextCheckpointInput }; + readonly narrative?: ParentNarrativeRecoveryCommit; + readonly taskIntents?: readonly TaskToolWriteIntent[]; + readonly systemIntents?: readonly CodexSystemWriterIntent[]; + readonly message?: DataOnlyParentLiveMessageInput; + readonly planQuestions?: readonly PlanQuestion[]; + readonly planOutput?: PlanPersistenceReady; +} /** The only effects admitted after an execution's terminal projection has committed. */ export interface PostTerminalEffects { @@ -94,16 +93,7 @@ export interface ExecutionSemanticOperation { | { readonly kind: "append-events"; readonly phase: string; readonly nativeCursor: unknown | null; readonly events: readonly ProviderEventDraft[]; readonly parentLive?: ParentLiveEffects } | { readonly kind: "append-assistant-text"; readonly inputs: readonly ParentAssistantTextCheckpointInput[] } | { readonly kind: "narrative-delta"; readonly input: ParentNarrativeRecoveryCommit } - | { - readonly kind: "live-event"; - readonly text: Extract["text"]; - readonly narrative?: ParentNarrativeRecoveryCommit; - readonly taskIntents?: readonly TaskToolWriteIntent[]; - readonly systemIntents?: readonly CodexSystemWriterIntent[]; - readonly message?: DataOnlyParentLiveMessageInput; - readonly planQuestions?: readonly PlanQuestion[]; - readonly planOutput?: PlanPersistenceReady; - } + | ({ readonly kind: "live-event" } & ParentLiveEffects) | { readonly kind: "checkpoint"; readonly phase: string; readonly nativeCursor: unknown | null } | { readonly kind: "stop-requested"; readonly requestId: string; readonly lastAdmittedOrdinal: number } | { readonly kind: "effect-result"; readonly effectId: string; readonly settled: boolean } @@ -176,14 +166,11 @@ export type ExecutionParentEventResult = Pick & { readonly parentEvent?: ExecutionParentEventResult }) | { readonly kind: "released" } | { readonly kind: "rejected"; readonly reason: "no-execution" | "stale-execution" | "out-of-order" | "invalid-transition" | "invalid-event-routing" | "invalid-text-routing" | "invalid-narrative-routing" | "invalid-stop-watermark" | "writer-conflict" | "writer-failure" }; type WorkerCommand = ExecutionMailboxCommand; -const METADATA_MUTATIONS: ReadonlySet = new Set([ - "checkpoint", "effect-result", "provider-outcome", "stage-terminal", -]); interface ExecutionState { readonly execution: ExecutionIdentity; @@ -293,8 +280,7 @@ export class ExecutionWorkerHandler { state.phase = "finalized"; if (state.fileAttempt !== undefined) this.files.retire(state.execution, state.fileAttempt); } - const result = committedResult(receipt); - return prepared.parentEvent ? { ...result, parentEvent: prepared.parentEvent } : result; + return prepared.parentEvent ? { ...receipt, parentEvent: prepared.parentEvent } : receipt; } private prepareRequest(request: ExecutionWorkerRequest, state: ExecutionState): PreparedExecutionRequest | undefined { @@ -432,9 +418,7 @@ export class ExecutionWorkerHandler { phase: "running", durableRevision: receipt.durableRevision, }); - return started - ? { ...committedResult(receipt), parentEvent: started } - : committedResult(receipt); + return started ? { ...receipt, parentEvent: started } : receipt; } private admissionStart( @@ -529,7 +513,7 @@ function syntheticTerminalInput( ): SyntheticTerminalInput | undefined { if (state.phase !== "running" && state.phase !== "stopping") return undefined; if (command.input.outcome !== command.outcome || command.input.providerId !== state.providerId - || !sameIdentity(command.input, state.execution)) return undefined; + || !sameExecution(command.input, state.execution)) return undefined; switch (command.outcome) { case "cancelled": return { outcome: "cancelled" }; case "interrupted": return { outcome: "interrupted" }; @@ -556,18 +540,11 @@ function metadataOrTextMutation( execution: ExecutionIdentity, state: ExecutionState, ): ExecutionSemanticOperation["mutation"] | undefined { - if (isMetadataMutation(command)) return command; + if (command.kind !== "assistant-text") return command; return state.phase === "running" && validTextRouting(command.inputs, execution) ? { kind: "append-assistant-text", inputs: command.inputs } : undefined; } -function isMetadataMutation(command: WorkerCommand): command is Extract< - WorkerCommand, - { readonly kind: "checkpoint" | "effect-result" | "provider-outcome" | "stage-terminal" } -> { - return METADATA_MUTATIONS.has(command.kind); -} - function eventMutation( command: Extract, execution: ExecutionIdentity, @@ -637,7 +614,7 @@ function commandRejection( request: ExecutionWorkerRequest, state: ExecutionState, ): Extract["reason"] | null { - if (!sameIdentity(state.execution, request.execution) || !sameLease(state.lease, request.lease)) return "stale-execution"; + if (!sameExecution(state.execution, request.execution) || !sameLease(state.lease, request.lease)) return "stale-execution"; if (validRecoveryRelease(request.command, state.lease)) return null; if (request.ordinal !== state.nextOrdinal) return "out-of-order"; return null; @@ -684,14 +661,12 @@ function invalidLiveEventReason( } function validEventRouting(events: readonly ProviderEventDraft[], execution: ExecutionIdentity): boolean { - return events.length > 0 && events.every((event) => event.routing.threadId === execution.threadId - && event.routing.turnId === execution.turnId && event.routing.executionId === execution.executionId); + return events.length > 0 && events.every((event) => sameExecution(event.routing, execution)); } function validTextRouting(inputs: readonly ParentAssistantTextCheckpointInput[], execution: ExecutionIdentity): boolean { return Array.isArray(inputs) && inputs.length > 0 && inputs.every((input) => input - && input.threadId === execution.threadId - && input.turnId === execution.turnId && input.executionId === execution.executionId); + && sameExecution(input, execution)); } function validLiveEventRouting( @@ -787,9 +762,7 @@ function validFinishInput( ): boolean { return command.outcome === command.input.outcome && command.input.providerId === providerId - && command.input.threadId === execution.threadId - && command.input.turnId === execution.turnId - && command.input.executionId === execution.executionId; + && sameExecution(command.input, execution); } function operationFor( @@ -831,25 +804,3 @@ function isDurableReceipt( return receipt.kind === "committed" && receipt.operationId === operationId(request) && Number.isSafeInteger(receipt.durableRevision) && receipt.durableRevision >= previousRevision; } - -function committedResult(receipt: Extract): Extract { - return { - kind: "committed", operationId: receipt.operationId, durableRevision: receipt.durableRevision, - ...(receipt.providerCommit ? { providerCommit: receipt.providerCommit } : {}), - ...(receipt.providerEvents ? { providerEvents: receipt.providerEvents } : {}), - ...(receipt.assistantTextCheckpoint ? { assistantTextCheckpoint: receipt.assistantTextCheckpoint } : {}), - ...(receipt.livePublication ? { livePublication: receipt.livePublication } : {}), - ...(receipt.planQuestions ? { planQuestions: receipt.planQuestions } : {}), - ...(receipt.planOutput ? { planOutput: receipt.planOutput } : {}), - ...(receipt.terminalPersistence ? { terminalPersistence: receipt.terminalPersistence } : {}), - }; -} - -function sameIdentity(left: ExecutionIdentity, right: ExecutionIdentity): boolean { - return left.threadId === right.threadId && left.turnId === right.turnId && left.executionId === right.executionId; -} - -function sameLease(left: ExecutionLease, right: ExecutionLease): boolean { - return left.ownerEpoch === right.ownerEpoch && left.workerIndex === right.workerIndex - && left.workerGeneration === right.workerGeneration && left.leaseId === right.leaseId; -} diff --git a/apps/server/src/features/agents/execution/worker-owned-turn-runtime.ts b/apps/server/src/features/agents/execution/worker-owned-turn-runtime.ts index 6c3367a83..32d4e3cc0 100644 --- a/apps/server/src/features/agents/execution/worker-owned-turn-runtime.ts +++ b/apps/server/src/features/agents/execution/worker-owned-turn-runtime.ts @@ -9,7 +9,7 @@ import { ExecutionMailboxOwner } from "./execution-mailbox-owner.js"; import { ExecutionMailboxScheduler, type ExecutionMailboxLimits } from "./execution-mailbox-scheduler.js"; import { ExecutionProviderEventOwnership } from "./execution-provider-event-ownership.js"; import { ExecutionThreadWorkerPort } from "./execution-worker-port.js"; -import type { ExecutionIdentity } from "./execution-mailbox-protocol.js"; +import type { ExecutionIdentity, ExecutionLease } from "./execution-mailbox-protocol.js"; import { ExecutionWorkerLossCoordinator, workerLossIncidentId } from "./execution-worker-loss-coordinator.js"; import type { ExecutionWorkCommand, ExecutionWorkerResult } from "./execution-worker-handler.js"; @@ -84,18 +84,16 @@ export class WorkerOwnedTurnRuntime { } } - private async recoverRejectedOwned(execution: ExecutionIdentity, lease: NonNullable>["lease"]): Promise { + private async recoverRejectedOwned(execution: ExecutionIdentity, lease: ExecutionLease): Promise { const receipt = await this.writerPort.interruptWorkerLoss({ execution, lease, reason: "The provider event could not be durably applied to this execution.", recoveryIncidentId: workerLossIncidentId({ execution, lease }), }); - const recovery = receipt.kind === "committed" ? receipt - : receipt.recoveryState === "already-terminal" - ? { ...receipt, recoveryState: "already-terminal" as const } - : null; - if (!recovery) throw new Error("Rejected execution has no durable recovery evidence"); - await this.owner.releaseRecovered(execution, recovery); + if (receipt.kind !== "committed" && receipt.recoveryState !== "already-terminal") { + throw new Error("Rejected execution has no durable recovery evidence"); + } + await this.owner.releaseRecovered(execution, receipt); this.onRecovered?.(execution); }